ttftkit 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ttftkit/hardware.py ADDED
@@ -0,0 +1,270 @@
1
+ """GPU 显存 / 利用率采样(v0.2)。
2
+
3
+ 只做三件事,且每件都要求「采不到就说采不到」:
4
+ 1. 解析 nvidia-smi / rocm-smi 的真实输出——坏输出一律报错,静默跳过等于把故障算成正常;
5
+ 2. 在测量窗口内按固定间隔采样,聚合出 baseline(窗口起点)、peak、delta;
6
+ 3. 采样器自己出错时按次数累计并不中断压测,但错误会如实出现在硬件块里。
7
+
8
+ 不做的事:不采集 ECC / 温度 / 功耗(DCGM 的活),也不做 Prometheus 导出(见 docs/roadmap.md)。
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import shutil
14
+ import subprocess
15
+ import threading
16
+ from dataclasses import dataclass
17
+
18
+ NVIDIA_QUERY = [
19
+ "--query-gpu=name,driver_version,memory.total,memory.used,utilization.gpu",
20
+ "--format=csv,noheader,nounits",
21
+ ]
22
+ COMMAND_TIMEOUT_S = 10.0
23
+ MB_PER_BYTE = 1024.0 * 1024.0
24
+
25
+
26
+ @dataclass(frozen=True, slots=True)
27
+ class GpuSample:
28
+ name: str
29
+ driver: str
30
+ mem_total_mb: float
31
+ mem_used_mb: float
32
+ utilization_pct: float | None = None
33
+
34
+
35
+ def _number(text: str, what: str) -> float:
36
+ try:
37
+ value = float(str(text).strip())
38
+ except (TypeError, ValueError):
39
+ raise ValueError(f"invalid {what}: {text!r}") from None
40
+ if value != value or value in (float("inf"), float("-inf")):
41
+ raise ValueError(f"invalid {what}: {text!r} (not finite)")
42
+ if value < 0:
43
+ raise ValueError(f"invalid {what}: {text!r} (negative)")
44
+ return value
45
+
46
+
47
+ def _optional_number(text: str, what: str) -> float | None:
48
+ raw = str(text).strip()
49
+ if raw in ("", "[N/A]", "N/A"):
50
+ return None
51
+ return _number(raw, what)
52
+
53
+
54
+ def parse_nvidia_smi(text: str) -> GpuSample:
55
+ """解析 `--query-gpu=name,driver_version,memory.total,memory.used,utilization.gpu` 的 CSV。"""
56
+ for line in str(text).splitlines():
57
+ if not line.strip():
58
+ continue
59
+ fields = [f.strip() for f in line.split(",")]
60
+ if len(fields) != 5:
61
+ raise ValueError(f"nvidia-smi line has {len(fields)} fields, expected 5: {line!r}")
62
+ name, driver, total, used, util = fields
63
+ return GpuSample(
64
+ name=name,
65
+ driver=driver,
66
+ mem_total_mb=_number(total, "memory.total"),
67
+ mem_used_mb=_number(used, "memory.used"),
68
+ utilization_pct=_optional_number(util, "utilization.gpu"),
69
+ )
70
+ raise ValueError("nvidia-smi produced no usable line")
71
+
72
+
73
+ def parse_rocm_smi(text: str) -> GpuSample:
74
+ """解析 `rocm-smi --showmeminfo vram --showuse --json` 的输出(字节转 MB)。"""
75
+ try:
76
+ data = json.loads(text)
77
+ except json.JSONDecodeError as exc:
78
+ raise ValueError(f"rocm-smi output is not JSON: {exc.msg}") from None
79
+ if not isinstance(data, dict):
80
+ raise ValueError("rocm-smi output must be a JSON object")
81
+ for key, card in data.items():
82
+ if not (isinstance(key, str) and key.startswith("card") and isinstance(card, dict)):
83
+ continue
84
+ total_b = card.get("VRAM Total Memory (B)")
85
+ used_b = card.get("VRAM Total Used Memory (B)")
86
+ if total_b is None or used_b is None:
87
+ raise ValueError(f"{key}: missing VRAM Total Memory / Used Memory fields")
88
+ return GpuSample(
89
+ name=str(card.get("Card Series") or card.get("Card series") or "unknown"),
90
+ driver=str(card.get("Driver version") or "unknown"),
91
+ mem_total_mb=_number(total_b, "VRAM Total Memory (B)") / MB_PER_BYTE,
92
+ mem_used_mb=_number(used_b, "VRAM Total Used Memory (B)") / MB_PER_BYTE,
93
+ utilization_pct=_optional_number(card.get("GPU use (%)", ""), "GPU use (%)"),
94
+ )
95
+ raise ValueError("rocm-smi output has no 'card*' entry")
96
+
97
+
98
+ class CommandSampler:
99
+ """跑一条外部命令并解析成 GpuSample(默认按 nvidia-smi CSV 解析)。"""
100
+
101
+ def __init__(self, argv, *, source="command", parser=parse_nvidia_smi):
102
+ if not argv:
103
+ raise ValueError("sampler command must not be empty")
104
+ self.argv = list(argv)
105
+ self.source = source
106
+ self._parser = parser
107
+
108
+ def sample(self) -> GpuSample:
109
+ proc = subprocess.run(
110
+ self.argv, capture_output=True, text=True, timeout=COMMAND_TIMEOUT_S, check=False
111
+ )
112
+ if proc.returncode != 0:
113
+ raise OSError(f"{self.source} exited with {proc.returncode}")
114
+ return self._parser(proc.stdout)
115
+
116
+
117
+ class RocmSampler(CommandSampler):
118
+ def __init__(self, argv):
119
+ super().__init__(argv, source="rocm-smi", parser=parse_rocm_smi)
120
+
121
+
122
+ def command_sampler(argv) -> CommandSampler:
123
+ return CommandSampler(argv)
124
+
125
+
126
+ def detect_sampler() -> CommandSampler | RocmSampler | None:
127
+ """探测本机可用的采样器;都没有就返回 None(调用方据此声明「未采样」)。"""
128
+ nvidia = shutil.which("nvidia-smi")
129
+ if nvidia:
130
+ return CommandSampler([nvidia, *NVIDIA_QUERY], source="nvidia-smi")
131
+ rocm = shutil.which("rocm-smi")
132
+ if rocm:
133
+ return RocmSampler([rocm, "--showmeminfo", "vram", "--showuse", "--json"])
134
+ return None
135
+
136
+
137
+ class Collector:
138
+ """在测量窗口内按间隔采样;线程只做采样,聚合在 block() 里一次算清。"""
139
+
140
+ def __init__(self, sampler, *, interval_s: float = 0.25, max_series: int = 120,
141
+ stop_timeout_s: float | None = None):
142
+ if interval_s <= 0:
143
+ raise ValueError("sample interval must be positive")
144
+ if sampler is None:
145
+ # 没有采样器就没有窗口起点,block() 也无从写 source:这种 Collector 是误用,
146
+ # 建的时候就说清楚(「不采样」的正确写法是根本不建 Collector)
147
+ raise ValueError("Collector needs a sampler; pass sampler=None by not creating one")
148
+ if isinstance(max_series, bool) or not isinstance(max_series, int) or max_series < 2:
149
+ # 降采样公式用 max_series-1 做步长:0 → IndexError,1 → ZeroDivisionError
150
+ raise ValueError("max_series must be an integer >= 2")
151
+ self.sampler = sampler
152
+ self.interval_s = interval_s
153
+ self.max_series = max_series
154
+ # 停线程的等待上限:线程卡在采样器里时不能无限等,但也不能装作它已经停了
155
+ self.stop_timeout_s = interval_s * 4 + 1.0 if stop_timeout_s is None else stop_timeout_s
156
+ if stop_timeout_s is not None and stop_timeout_s <= 0:
157
+ raise ValueError("stop timeout must be positive")
158
+ self._values: list[float] = []
159
+ self._util: list[float] = []
160
+ self._total: float | None = None
161
+ self._name: str | None = None
162
+ self._driver: str | None = None
163
+ self._errors = 0
164
+ self._first_error: str | None = None
165
+ self._truncated = False
166
+ self._lock = threading.Lock()
167
+ self._stop = threading.Event()
168
+ self._thread: threading.Thread | None = None
169
+
170
+ def sample_once(self) -> bool:
171
+ try:
172
+ sample = self.sampler.sample()
173
+ except Exception as exc: # 采样失败不能拖垮压测,但必须被记下来
174
+ with self._lock:
175
+ self._errors += 1
176
+ if self._first_error is None:
177
+ self._first_error = type(exc).__name__
178
+ return False
179
+ with self._lock:
180
+ self._values.append(float(sample.mem_used_mb))
181
+ self._total = float(sample.mem_total_mb)
182
+ self._name = sample.name
183
+ self._driver = sample.driver
184
+ if sample.utilization_pct is not None:
185
+ self._util.append(float(sample.utilization_pct))
186
+ return True
187
+
188
+ def start(self) -> None:
189
+ if self.sampler is None:
190
+ return
191
+ self.sample_once() # 先同步采一次,作为窗口起点 baseline
192
+ self._thread = threading.Thread(target=self._loop, daemon=True)
193
+
194
+ self._thread.start()
195
+
196
+ def _loop(self) -> None:
197
+ while not self._stop.wait(self.interval_s):
198
+ self.sample_once()
199
+
200
+ def stop(self) -> dict | None:
201
+ self._stop.set()
202
+ if self._thread is not None:
203
+ self._thread.join(timeout=self.stop_timeout_s)
204
+ if self._thread.is_alive():
205
+ # 线程卡在采样器里:切不断,就如实记一笔——否则峰值会被低报而报告看不出来
206
+ with self._lock:
207
+ self._errors += 1
208
+ self._truncated = True
209
+ if self._first_error is None:
210
+ self._first_error = "SamplerBusyAtStop"
211
+ self._thread = None
212
+ return self.block()
213
+
214
+ def _series(self, values: list[float]) -> list[float]:
215
+ if len(values) <= self.max_series:
216
+ return list(values)
217
+ stride = (len(values) - 1) / (self.max_series - 1)
218
+ picked = [values[round(i * stride)] for i in range(self.max_series)]
219
+ picked[-1] = values[-1] # 末点是窗口终点,降采样不能把它扔掉
220
+ return picked
221
+
222
+ def block(self) -> dict:
223
+ """聚合出硬件块。只要尝试过采样就返回块——一次都没采到也要返回,
224
+ 把 sampler_errors / error 落盘,否则「GPU 监控坏了」会表现为「没要求采样」。"""
225
+ with self._lock:
226
+ values = list(self._values)
227
+ utils = list(self._util)
228
+ total, name, driver = self._total, self._name, self._driver
229
+ errors, first_error, truncated = self._errors, self._first_error, self._truncated
230
+ baseline = values[0] if values else None
231
+ peak = max(values) if values else None
232
+ return {
233
+ "source": self.sampler.source,
234
+ "name": name,
235
+ "driver": driver,
236
+ "mem_total_mb": total,
237
+ "mem_used_baseline_mb": baseline,
238
+ "mem_used_peak_mb": peak,
239
+ # 单样本时「增长」无从谈起:delta 必须是 null,不能是 0(0 会被读成「测到没涨」)
240
+ "mem_used_delta_mb": round(peak - baseline, 3) if len(values) >= 2 else None,
241
+ "utilization_peak_pct": max(utils) if utils else None,
242
+ "samples": len(values),
243
+ "interval_s": self.interval_s,
244
+ "sampler_errors": errors,
245
+ "error": first_error,
246
+ "truncated": truncated,
247
+ "mem_used_series_mb": self._series(values),
248
+ }
249
+
250
+
251
+ def fingerprint(block: dict | None) -> str | None:
252
+ """硬件指纹:显存总量不同的两块卡,性能数字不可直接比较。
253
+
254
+ 拿不到设备名或总量时返回 None(不是字符串 "none")——"none" 是 truthy,
255
+ 会让「两侧都没采样」被当成「同一块卡」而绕过可比性检查。
256
+ """
257
+ if not isinstance(block, dict):
258
+ return None
259
+ name, total = block.get("name"), block.get("mem_total_mb")
260
+ if not isinstance(name, str) or not name.strip():
261
+ return None
262
+ if isinstance(total, bool) or not isinstance(total, (int, float)) or total <= 0:
263
+ return None
264
+ return f"{name}/{total:g}MB"
265
+
266
+
267
+ def is_monotonic(series) -> bool:
268
+ """严格单调上升(验收「注入已知泄漏」用;平票不算上升)。"""
269
+ return all(b > a for a, b in zip(series, series[1:], strict=False))
270
+
ttftkit/metrics.py ADDED
@@ -0,0 +1,157 @@
1
+ """汇总统计:纯函数,输入请求记录,输出可 JSON 序列化的字典。
2
+
3
+ 设计约束:
4
+ - 分位数用线性插值(与 numpy 默认 method="linear" 一致),便于和别人的报告对齐。
5
+ - 缺数据返回 None,绝不返回 0 冒充。
6
+ - 吞吐的分母是**测量墙钟时长**,不是各请求延迟之和。
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import math
11
+ from collections.abc import Iterable, Sequence
12
+
13
+ from .models import RequestRecord
14
+
15
+
16
+ def percentile(values: Sequence[float], q: float) -> float:
17
+ """线性插值分位数。q 取 0~1。"""
18
+ if not values:
19
+ raise ValueError("percentile of empty sequence")
20
+ if not 0.0 <= q <= 1.0:
21
+ raise ValueError(f"q must be in [0, 1], got {q}")
22
+ ordered = sorted(values)
23
+ if len(ordered) == 1:
24
+ return float(ordered[0])
25
+ pos = q * (len(ordered) - 1)
26
+ low = math.floor(pos)
27
+ high = math.ceil(pos)
28
+ if low == high:
29
+ return float(ordered[low])
30
+ weight = pos - low
31
+ return float(ordered[low] + (ordered[high] - ordered[low]) * weight)
32
+
33
+
34
+ def distribution(values: Iterable[float]) -> dict | None:
35
+ """一列数值的分布。空输入 → None(无证据,不是 0)。"""
36
+ data = [float(v) for v in values if v is not None]
37
+ if not data:
38
+ return None
39
+ mean = sum(data) / len(data)
40
+ if len(data) > 1:
41
+ variance = sum((v - mean) ** 2 for v in data) / (len(data) - 1) # 样本方差
42
+ stdev = math.sqrt(variance)
43
+ else:
44
+ stdev = 0.0
45
+ return {
46
+ "count": len(data),
47
+ "mean": mean,
48
+ "min": min(data),
49
+ "max": max(data),
50
+ "p50": percentile(data, 0.50),
51
+ "p90": percentile(data, 0.90),
52
+ "p95": percentile(data, 0.95),
53
+ "p99": percentile(data, 0.99),
54
+ "stdev": stdev,
55
+ "cv": (stdev / mean) if mean else None,
56
+ }
57
+
58
+
59
+ def _degradation_ratio(records: list[RequestRecord]) -> float | None:
60
+ """后 1/4 请求的平均延迟 ÷ 前 1/4,用于发现「跑久了变慢」。"""
61
+ ordered = [r for r in records if r.ok]
62
+ if len(ordered) < 4:
63
+ return None
64
+ quarter = max(1, len(ordered) // 4)
65
+ head = ordered[:quarter]
66
+ tail = ordered[-quarter:]
67
+ head_mean = sum(r.latency_ms for r in head) / len(head)
68
+ tail_mean = sum(r.latency_ms for r in tail) / len(tail)
69
+ if head_mean <= 0:
70
+ return None
71
+ return tail_mean / head_mean
72
+
73
+
74
+ def summarize(
75
+ records: Sequence[RequestRecord],
76
+ seconds: float,
77
+ *,
78
+ slo_ttft_ms: float | None = None,
79
+ slo_tpot_ms: float | None = None,
80
+ hardware: dict | None = None,
81
+ ) -> dict:
82
+ """把一组请求记录汇总成报告里的 `summary` 段。
83
+
84
+ seconds: 测量窗口的墙钟时长(秒),必须有限且 > 0。
85
+ hardware: 采集到的硬件块(v0.2+);未采样时传 None,对应指标一律为 null 而非 0。
86
+ """
87
+ if not isinstance(seconds, (int, float)) or isinstance(seconds, bool):
88
+ raise ValueError(f"measurement window must be a number, got {seconds!r}")
89
+ if not math.isfinite(seconds) or seconds <= 0:
90
+ raise ValueError(f"measurement window must be finite and > 0, got {seconds!r}")
91
+
92
+ total = len(records)
93
+ ok_records = [r for r in records if r.ok]
94
+ with_tokens = [r for r in ok_records if r.output_tokens is not None]
95
+
96
+ output_tokens_per_s: float | None
97
+ if not ok_records:
98
+ output_tokens_per_s = None
99
+ elif len(with_tokens) != len(ok_records):
100
+ output_tokens_per_s = None # 部分成功请求缺计数 → 总体吞吐不可信
101
+ else:
102
+ output_tokens_per_s = sum(r.output_tokens for r in with_tokens) / seconds
103
+
104
+ goodput = None
105
+ if slo_ttft_ms is not None or slo_tpot_ms is not None:
106
+ goodput = sum(1 for r in ok_records if _meets(r, slo_ttft_ms, slo_tpot_ms)) / seconds
107
+
108
+ return {
109
+ "requests": total,
110
+ "successful_requests": len(ok_records),
111
+ "error_rate": ((total - len(ok_records)) / total) if total else 0.0,
112
+ "requests_per_s": len(ok_records) / seconds,
113
+ "output_tokens_per_s": output_tokens_per_s,
114
+ "token_usage_coverage": (len(with_tokens) / len(ok_records)) if ok_records else 0.0,
115
+ "goodput_per_s": goodput,
116
+ "ttft_ms": distribution(r.ttft_ms for r in ok_records),
117
+ "tpot_ms": distribution(r.tpot_ms for r in ok_records),
118
+ "latency_ms": distribution(r.latency_ms for r in ok_records),
119
+ "decode_ms": distribution(r.decode_ms for r in ok_records),
120
+ "chunk_gap_ms": distribution(gap for r in ok_records for gap in r.chunk_intervals_ms),
121
+ "degradation_ratio": _degradation_ratio(ok_records),
122
+ "mem_used_peak_mb": _hw_metric(hardware, "mem_used_peak_mb"),
123
+ "mem_used_delta_mb": _hw_metric(hardware, "mem_used_delta_mb"),
124
+ "gpu_util_peak_pct": _hw_metric(hardware, "utilization_peak_pct"),
125
+ "errors": _error_counts(records),
126
+ }
127
+
128
+
129
+ def _hw_metric(hardware: dict | None, key: str) -> float | None:
130
+ """硬件指标缺席时必须是 null:0 会被下游误读成「测得为 0」。"""
131
+ if not isinstance(hardware, dict):
132
+ return None
133
+ value = hardware.get(key)
134
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
135
+ return None
136
+ if not math.isfinite(value):
137
+ return None
138
+ return float(value)
139
+
140
+
141
+ def _meets(record: RequestRecord, slo_ttft_ms: float | None, slo_tpot_ms: float | None) -> bool:
142
+ if slo_ttft_ms is not None and (record.ttft_ms is None or record.ttft_ms > slo_ttft_ms):
143
+ return False
144
+ return not (
145
+ slo_tpot_ms is not None and (record.tpot_ms is None or record.tpot_ms > slo_tpot_ms)
146
+ )
147
+
148
+
149
+ def _error_counts(records: Sequence[RequestRecord]) -> dict:
150
+ counts: dict[str, int] = {}
151
+ for r in records:
152
+ if r.ok:
153
+ continue
154
+ # 错误串本身已是固定分类(不含服务端取值),保留完整分类以不丢异常类型
155
+ key = (r.error or "unknown").strip()[:80]
156
+ counts[key] = counts.get(key, 0) + 1
157
+ return counts
ttftkit/models.py ADDED
@@ -0,0 +1,45 @@
1
+ """数据契约:一次请求的原始测量记录。
2
+
3
+ 这里只有数据和派生属性,不做任何 IO,也不做汇总统计(那在 metrics.py)。
4
+ 所有时间单位都是毫秒,除 `started_at` 用 perf_counter 秒。
5
+ """
6
+ from __future__ import annotations
7
+
8
+ from dataclasses import dataclass
9
+
10
+
11
+ @dataclass(frozen=True, slots=True)
12
+ class RequestRecord:
13
+ """一次流式请求的测量结果。
14
+
15
+ 字段口径(不要混用):
16
+ - latency_ms: 发请求到流结束的墙钟时间
17
+ - ttft_ms: 发请求到**首个非空 content 事件**的时间(含连接与排队)
18
+ - decode_ms: 首个内容事件到最后一个内容事件的时间差
19
+ - content_chunks: 收到多少个带内容的 SSE 事件(**不是 token 数**)
20
+ - output_tokens: 服务端 usage 报告的 completion_tokens;缺失时 None(不是 0)
21
+ """
22
+
23
+ index: int
24
+ ok: bool
25
+ latency_ms: float
26
+ ttft_ms: float | None = None
27
+ decode_ms: float | None = None
28
+ content_chunks: int = 0
29
+ output_tokens: int | None = None
30
+ prompt_tokens: int | None = None
31
+ chunk_intervals_ms: tuple[float, ...] = ()
32
+ error: str | None = None
33
+ started_at: float = 0.0
34
+
35
+ @property
36
+ def tpot_ms(self) -> float | None:
37
+ """平均每 token 解码时间 = decode_ms / (completion_tokens - 1)。
38
+
39
+ 少于 2 个 token、缺 usage、或只有一个内容事件时无定义 → None。
40
+ """
41
+ if not self.ok or self.output_tokens is None or self.output_tokens < 2:
42
+ return None
43
+ if self.decode_ms is None or self.content_chunks < 2:
44
+ return None
45
+ return self.decode_ms / (self.output_tokens - 1)
ttftkit/report.py ADDED
@@ -0,0 +1,176 @@
1
+ """报告渲染:同一份 JSON 生成 Markdown,避免两处口径漂移。"""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ from pathlib import Path
6
+
7
+
8
+ def _fmt(value, digits=1, suffix=""):
9
+ if value is None:
10
+ return "n/a"
11
+ if isinstance(value, bool):
12
+ return str(value)
13
+ if isinstance(value, (int, float)):
14
+ return f"{value:,.{digits}f}{suffix}"
15
+ return str(value)
16
+
17
+
18
+ _DIST_FIELDS = ("mean", "p50", "p95", "p99", "count")
19
+
20
+
21
+ def _dist_line(name, dist, unit="ms"):
22
+ if dist is None:
23
+ return f"| {name} | n/a | n/a | n/a | n/a | n/a |"
24
+ if not isinstance(dist, dict):
25
+ raise ValueError(f"{name} distribution must be an object or null")
26
+ missing = [f for f in _DIST_FIELDS if f not in dist]
27
+ if missing:
28
+ raise ValueError(f"{name} distribution missing {', '.join(missing)}")
29
+ return (
30
+ f"| {name} | {_fmt(dist['mean'])}{unit} | {_fmt(dist['p50'])}{unit} "
31
+ f"| {_fmt(dist['p95'])}{unit} | {_fmt(dist['p99'])}{unit} | {dist['count']} |"
32
+ )
33
+
34
+
35
+ def _mapping(value, what):
36
+ if not isinstance(value, dict):
37
+ raise ValueError(f"report field {what!r} must be an object")
38
+ return value
39
+
40
+
41
+ def markdown(result: dict) -> str:
42
+ if not isinstance(result, dict):
43
+ raise ValueError("report must be a JSON object")
44
+ config = _mapping(result.get("config", {}), "config")
45
+ summary = _mapping(result.get("summary", {}), "summary")
46
+ records = result.get("records", [])
47
+ if not isinstance(records, list):
48
+ raise ValueError("report field 'records' must be a list")
49
+ sha = config.get("prompt_sha256")
50
+ sha_text = f"{sha[:16]}…" if isinstance(sha, str) and sha else "?"
51
+ lines = [
52
+ "# ttftkit 性能报告",
53
+ "",
54
+ f"- 模型:`{config.get('model', '?')}`",
55
+ f"- 并发:{config.get('concurrency', '?')},请求数:{config.get('requests', '?')},"
56
+ f"max_tokens:{config.get('max_tokens', '?')},warmup:{config.get('warmup', 0)}",
57
+ f"- 协议:{config.get('protocol', '?')},部署 profile:`{config.get('profile', '?')}`",
58
+ f"- prompt sha256:`{sha_text}`(原文不入报告)",
59
+ f"- 测量窗口:{_fmt(result.get('duration_s'), 3)} s",
60
+ "",
61
+ "## 汇总",
62
+ "",
63
+ "| 指标 | 数值 |",
64
+ "| --- | --- |",
65
+ f"| 成功/总请求 | {summary.get('successful_requests', 0)} / {summary.get('requests', 0)} |",
66
+ f"| 错误率 | {_fmt(summary.get('error_rate'), 4)} |",
67
+ f"| 请求吞吐 | {_fmt(summary.get('requests_per_s'), 3)} req/s |",
68
+ f"| token 吞吐 | {_fmt(summary.get('output_tokens_per_s'), 2)} tok/s |",
69
+ f"| token 计数覆盖率 | {_fmt(summary.get('token_usage_coverage'), 3)} |",
70
+ # goodput 只有在给了 SLO(--slo-ttft/--slo-tpot)时才有定义,否则不展示以免误导。
71
+ *([f"| goodput | {_fmt(summary.get('goodput_per_s'), 3)} req/s |"]
72
+ if summary.get("goodput_per_s") is not None else []),
73
+ f"| 退化比(后1/4 ÷ 前1/4 延迟) | {_fmt(summary.get('degradation_ratio'), 3)} |",
74
+ "",
75
+ ]
76
+ hardware = result.get("hardware")
77
+ if hardware is not None and not isinstance(hardware, dict):
78
+ raise ValueError("report field 'hardware' must be an object or null")
79
+ if isinstance(hardware, dict) and not hardware:
80
+ raise ValueError("report field 'hardware' is an empty object")
81
+
82
+ def _mb(value) -> str:
83
+ return "n/a" if value is None else f"{_fmt(value)} MB"
84
+
85
+ lines += ["## 硬件(GPU 采样)", ""]
86
+ if hardware is None:
87
+ lines += [
88
+ "未采样:本报告不含显存 / 利用率证据(`--gpu none`,或未探测到 nvidia-smi / rocm-smi)。",
89
+ "",
90
+ ]
91
+ elif not hardware.get("samples"):
92
+ # 采样器在、却一次都没采到:这是坏掉的监控,不是「没有硬件」,必须显式说出来。
93
+ lines += [
94
+ f"采样失败:采样器已探测到(`{hardware.get('source', '?')}`),"
95
+ f"但 {hardware.get('sampler_errors', 0)} 次采样全部失败"
96
+ f"(首个:`{hardware.get('error')}`)。本报告**没有**显存 / 利用率证据。",
97
+ "",
98
+ ]
99
+ else:
100
+ util = hardware.get("utilization_peak_pct")
101
+ util_text = "n/a" if util is None else f"{_fmt(util)}%"
102
+ delta = hardware.get("mem_used_delta_mb")
103
+ delta_text = "n/a(采样不足 2 次,算不出增长)" if delta is None else _mb(delta)
104
+ lines += [
105
+ f"- 来源:`{hardware.get('source', '?')}`,设备:{hardware.get('name', '?')},"
106
+ f"驱动:{hardware.get('driver', '?')}",
107
+ f"- 显存:起点 {_mb(hardware.get('mem_used_baseline_mb'))} / "
108
+ f"峰值 {_mb(hardware.get('mem_used_peak_mb'))} "
109
+ f"(总量 {_mb(hardware.get('mem_total_mb'))})",
110
+ f"- 窗口内增长:{delta_text},峰值利用率:{util_text}",
111
+ f"- 采样 {hardware.get('samples', 0)} 次(间隔 {hardware.get('interval_s')} s),"
112
+ f"采样器错误 {hardware.get('sampler_errors', 0)} 次",
113
+ "",
114
+ ]
115
+ if hardware.get("truncated"):
116
+ lines += [
117
+ "> 采样线程在 `stop()` 超时后仍在运行(`SamplerBusyAtStop`):"
118
+ "峰值与增长可能被**低报**,这里只覆盖已收上来的样本。",
119
+ "",
120
+ ]
121
+ if hardware.get("sampler_errors"):
122
+ lines += [
123
+ f"> 采样器曾失败 {hardware['sampler_errors']} 次(首个:`{hardware.get('error')}`)。"
124
+ "显存数字只覆盖采样成功的时刻,不能当作全程证据。",
125
+ "",
126
+ ]
127
+ lines += [
128
+ "## 延迟分布(单位:毫秒)",
129
+ "",
130
+ "| 指标 | 均值 | p50 | p95 | p99 | 样本数 |",
131
+ "| --- | --- | --- | --- | --- | --- |",
132
+ _dist_line("TTFT", summary.get("ttft_ms")),
133
+ _dist_line("TPOT", summary.get("tpot_ms")),
134
+ _dist_line("请求总延迟", summary.get("latency_ms")),
135
+ _dist_line("解码段", summary.get("decode_ms")),
136
+ _dist_line("SSE 块间隔", summary.get("chunk_gap_ms")),
137
+ "",
138
+ "> TPOT = 解码段 ÷ (服务端 completion_tokens − 1)。SSE 块间隔只是传输层观测,"
139
+ "**不是**逐 token ITL;一个块可能含多个 token。",
140
+ "",
141
+ ]
142
+ errors = summary.get("errors")
143
+ if errors is None:
144
+ errors = {}
145
+ if not isinstance(errors, dict):
146
+ raise ValueError("summary field 'errors' must be an object")
147
+ if errors:
148
+ lines += ["## 失败分类", "", "| 错误 | 次数 |", "| --- | --- |"]
149
+ lines += [f"| `{k}` | {v} |" for k, v in sorted(errors.items(), key=lambda kv: -kv[1])]
150
+ lines.append("")
151
+ if records:
152
+ lines += ["## 逐请求", "", "| # | 成功 | TTFT(ms) | 解码(ms) | 块数 | tokens | 错误 |",
153
+ "| --- | --- | --- | --- | --- | --- | --- |"]
154
+ for r in records[:50]:
155
+ if not isinstance(r, dict):
156
+ raise ValueError("each record must be an object")
157
+ lines.append(
158
+ f"| {r.get('index')} | {'是' if r.get('ok') else '否'} "
159
+ f"| {_fmt(r.get('ttft_ms'))} | {_fmt(r.get('decode_ms'))} "
160
+ f"| {r.get('content_chunks')} | {_fmt(r.get('output_tokens'), 0)} "
161
+ f"| {r.get('error') or ''} |"
162
+ )
163
+ if len(records) > 50:
164
+ lines.append(f"| … | | | | | | 其余 {len(records) - 50} 条见 JSON |")
165
+ lines.append("")
166
+ return "\n".join(lines)
167
+
168
+
169
+ def write(result: dict, path: str | Path) -> tuple[Path, Path]:
170
+ """写 JSON 与同名 Markdown,返回两个路径。"""
171
+ json_path = Path(path)
172
+ json_path.parent.mkdir(parents=True, exist_ok=True)
173
+ json_path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
174
+ md_path = json_path.with_suffix(".md")
175
+ md_path.write_text(markdown(result), encoding="utf-8")
176
+ return json_path, md_path