ttftkit 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ttftkit/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """ttftkit: OpenAI 兼容推理服务的性能测量与回归门禁。"""
2
+
3
+ __version__ = "0.2.1"
ttftkit/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
ttftkit/cli.py ADDED
@@ -0,0 +1,159 @@
1
+ """命令行入口:run / gate / report。退出码 0=通过 1=不通过 2=用法或输入错误。"""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import asyncio
6
+ import json
7
+ import math
8
+ import os
9
+ import shlex
10
+ import sys
11
+ from pathlib import Path
12
+
13
+ from . import __version__
14
+ from .gate import evaluate
15
+ from .hardware import command_sampler, detect_sampler
16
+ from .report import markdown, write
17
+ from .runner import BenchmarkFailed, run_benchmark
18
+
19
+ DEFAULT_PROMPT = "Explain in one paragraph why KV cache memory grows with context length."
20
+
21
+
22
+ def _sampler_for(args):
23
+ """--gpu 三态:none 明确不采样;command 用给定命令;auto 探测本机工具。"""
24
+ if args.gpu == "none":
25
+ return None
26
+ if args.gpu == "command":
27
+ if not args.gpu_command:
28
+ raise ValueError("--gpu command requires --gpu-command")
29
+ argv = shlex.split(args.gpu_command)
30
+ if not argv:
31
+ raise ValueError("--gpu-command must not be empty")
32
+ return command_sampler(argv)
33
+ return detect_sampler()
34
+
35
+
36
+ def _load(path: str | Path) -> dict:
37
+ data = json.loads(Path(path).read_text(encoding="utf-8"))
38
+ if not isinstance(data, dict):
39
+ raise ValueError(f"{path}: expected a JSON object")
40
+ return data
41
+
42
+
43
+ def cmd_run(args) -> int:
44
+ if not math.isfinite(args.sample_interval) or args.sample_interval <= 0:
45
+ raise ValueError(f"--sample-interval must be finite and positive, got {args.sample_interval!r}")
46
+ sampler = _sampler_for(args)
47
+ result = asyncio.run(
48
+ run_benchmark(
49
+ endpoint=args.endpoint,
50
+ model=args.model,
51
+ prompt=args.prompt,
52
+ requests=args.requests,
53
+ concurrency=args.concurrency,
54
+ max_tokens=args.max_tokens,
55
+ warmup=args.warmup,
56
+ timeout_s=args.timeout,
57
+ profile=args.profile,
58
+ api_key=args.api_key,
59
+ slo_ttft_ms=args.slo_ttft,
60
+ slo_tpot_ms=args.slo_tpot,
61
+ sampler=sampler,
62
+ sample_interval_s=args.sample_interval,
63
+ )
64
+ )
65
+ json_path, md_path = write(result, args.output)
66
+ summary = result["summary"]
67
+ failed = summary["requests"] - summary["successful_requests"]
68
+ print(f"报告:{json_path} {md_path}")
69
+ tps = summary["output_tokens_per_s"]
70
+ tps_text = "n/a" if tps is None else f"{round(tps, 2)}"
71
+ print(
72
+ f"成功 {summary['successful_requests']}/{summary['requests']},"
73
+ f"错误率 {summary['error_rate']:.3f},token 吞吐 {tps_text}"
74
+ )
75
+ if failed:
76
+ print(f"ttftkit: {failed} 个请求失败,详见报告中的失败分类", file=sys.stderr)
77
+ return 1
78
+ return 0
79
+
80
+
81
+ def cmd_gate(args) -> int:
82
+ candidate = _load(args.report)
83
+ policy = _load(args.policy)
84
+ baseline = _load(args.baseline) if args.baseline else None
85
+ verdict = evaluate(candidate, policy, baseline)
86
+ for check in verdict["checks"]:
87
+ observed = check.get("observed")
88
+ observed_text = "n/a" if observed is None else f"{observed:g}"
89
+ print(
90
+ f"[{check['status']:4}] {check['metric']:<24} "
91
+ f"观测={observed_text:<12} {check.get('reason', '')}"
92
+ )
93
+ for reason in verdict["reasons"]:
94
+ stream = sys.stderr if verdict["exit_code"] else sys.stdout
95
+ print(f"ttftkit: {reason}", file=stream)
96
+ print(f"结论:exit={verdict['exit_code']}(0=通过 1=不通过 2=配置/可比性错误)")
97
+ return verdict["exit_code"]
98
+
99
+
100
+ def cmd_report(args) -> int:
101
+ print(markdown(_load(args.report)))
102
+ return 0
103
+
104
+
105
+ def build_parser() -> argparse.ArgumentParser:
106
+ parser = argparse.ArgumentParser(prog="ttftkit", description="LLM 推理服务性能测量与回归门禁")
107
+ # 版本号只有 __init__ 一个真相源;报 bug 时用户第一条信息就是这个
108
+ parser.add_argument("--version", action="version", version=f"ttftkit {__version__}")
109
+ sub = parser.add_subparsers(dest="command", required=True)
110
+
111
+ run = sub.add_parser("run", help="对 OpenAI 兼容端点做一次测量")
112
+ run.add_argument("--endpoint", required=True, help="完整 URL,如 http://host:8000/v1/chat/completions")
113
+ run.add_argument("--model", required=True)
114
+ run.add_argument("--prompt", default=DEFAULT_PROMPT)
115
+ run.add_argument("--requests", type=int, default=8)
116
+ run.add_argument("--concurrency", type=int, default=1)
117
+ run.add_argument("--max-tokens", type=int, default=32)
118
+ run.add_argument("--warmup", type=int, default=0)
119
+ run.add_argument("--timeout", type=float, default=60.0, help="单请求超时(秒,含整条 SSE 流的整体预算)")
120
+ run.add_argument("--profile", default="unspecified", help="部署标识,如 rtx4090-vllm-0.6.3")
121
+ run.add_argument("--slo-ttft", type=float, default=None, help="TTFT SLO(ms), 给出后算 goodput")
122
+ run.add_argument("--slo-tpot", type=float, default=None, help="TPOT SLO(ms), 给出后算 goodput")
123
+ run.add_argument("--output", default="results/run.json")
124
+ run.add_argument("--api-key", default=os.environ.get("TTFTKIT_API_KEY"))
125
+ run.add_argument("--gpu", choices=("auto", "none", "command"), default="auto",
126
+ help="显存采样:auto=探测 nvidia-smi/rocm-smi,none=不采样,command=用 --gpu-command")
127
+ run.add_argument("--gpu-command", default=None,
128
+ help="自定义采样命令(须输出 nvidia-smi CSV 格式),配合 --gpu command")
129
+ run.add_argument("--sample-interval", type=float, default=0.25,
130
+ help="采样间隔(秒),默认 0.25")
131
+ run.set_defaults(func=cmd_run)
132
+
133
+ gate = sub.add_parser("gate", help="按策略判定一份报告能否放行")
134
+ gate.add_argument("report")
135
+ gate.add_argument("--policy", required=True)
136
+ gate.add_argument("--baseline", default=None)
137
+ gate.set_defaults(func=cmd_gate)
138
+
139
+ report = sub.add_parser("report", help="把 JSON 报告渲染为 Markdown")
140
+ report.add_argument("report")
141
+ report.set_defaults(func=cmd_report)
142
+ return parser
143
+
144
+
145
+ def main(argv: list[str] | None = None) -> int:
146
+ args = build_parser().parse_args(argv)
147
+ try:
148
+ return args.func(args)
149
+ except BenchmarkFailed as exc:
150
+ # 基础设施/服务端失败 → 「本轮不通过」(1),不是用法错误(2)
151
+ print(f"ttftkit: error: {exc}", file=sys.stderr)
152
+ return 1
153
+ except (ValueError, OSError) as exc:
154
+ print(f"ttftkit: error: {exc}", file=sys.stderr)
155
+ return 2
156
+
157
+
158
+ if __name__ == "__main__":
159
+ sys.exit(main())
ttftkit/client.py ADDED
@@ -0,0 +1,173 @@
1
+ """OpenAI 兼容端点的流式测量客户端。
2
+
3
+ 只负责「一次请求怎么测」,并发编排在 runner.py。
4
+ 所有时间用 time.perf_counter()(单调时钟),不用 time.time()(会被 NTP 调整)。
5
+
6
+ 对服务端协议的容忍度:只接受 `data: {...}` 的 SSE 事件与 `[DONE]`。
7
+ 任何形状不对的事件(不是对象、usage 不是非负整数、finish_reason 不是正常结束)
8
+ 都判为**失败请求**并记录原因——压测工具悄悄跳过坏数据,等于把故障算成性能。
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import asyncio
13
+ import json
14
+ import time
15
+
16
+ import httpx
17
+
18
+ from .models import RequestRecord
19
+
20
+ DEFAULT_PATH = "/v1/chat/completions"
21
+ SUCCESS_FINISH_REASONS = frozenset({"stop", "length"})
22
+
23
+
24
+ def _fail(index: int, t0: float, error: str, latency_ms: float | None = None) -> RequestRecord:
25
+ latency = latency_ms if latency_ms is not None else (time.perf_counter() - t0) * 1000
26
+ return RequestRecord(index=index, ok=False, latency_ms=latency, error=error, started_at=t0)
27
+
28
+
29
+ def _count(value, what: str) -> int:
30
+ """usage 里的计数必须是非负整数(bool 不算)。
31
+
32
+ 错误串只带字段名,绝不内插服务端回传的取值——那些值可能回显 prompt 或凭据。
33
+ """
34
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
35
+ raise ValueError(f"invalid usage field: {what}")
36
+ return value
37
+
38
+
39
+ async def measure_request(
40
+ client: httpx.AsyncClient,
41
+ payload: dict,
42
+ *,
43
+ index: int = 0,
44
+ timeout_s: float = 60.0,
45
+ path: str = DEFAULT_PATH,
46
+ ) -> RequestRecord:
47
+ """发一次流式请求并记录计时。
48
+
49
+ 失败一律返回 ok=False + error 字符串,不抛异常(编排层不该被单点失败打断)。
50
+
51
+ 整请求总时限用 `asyncio.wait_for` 兜底:httpx 的 timeout 是「单次读超时」,
52
+ 每收到一块就重置,服务端若慢速滴流且**永不发换行**,`aiter_lines()` 不会产出任何行,
53
+ 循环里的 deadline 检查永远执行不到——只有外层的总预算能拦住这种挂起。
54
+ """
55
+ t0 = time.perf_counter()
56
+ try:
57
+ return await asyncio.wait_for(
58
+ _measure_once(client, payload, index=index, t0=t0, timeout_s=timeout_s, path=path),
59
+ timeout=timeout_s,
60
+ )
61
+ except asyncio.TimeoutError:
62
+ return _fail(index, t0, "timeout")
63
+
64
+
65
+ async def _measure_once(
66
+ client: httpx.AsyncClient,
67
+ payload: dict,
68
+ *,
69
+ index: int,
70
+ t0: float,
71
+ timeout_s: float,
72
+ path: str,
73
+ ) -> RequestRecord:
74
+ first_t: float | None = None
75
+ last_t: float | None = None
76
+ content_chunks = 0
77
+ intervals: list[float] = []
78
+ output_tokens: int | None = None
79
+ prompt_tokens: int | None = None
80
+ finish_reason: str | None = None
81
+ done_seen = False
82
+
83
+ try:
84
+ async with client.stream(
85
+ "POST", path, json=payload, timeout=httpx.Timeout(timeout_s)
86
+ ) as response:
87
+ if response.status_code >= 400:
88
+ # 只记状态码:响应体可能回显 prompt 或凭据,不进报告
89
+ return _fail(index, t0, f"HTTP {response.status_code}")
90
+
91
+ async for line in response.aiter_lines():
92
+ if not line.startswith("data:"):
93
+ continue # 空行、注释行(`: keep-alive`)跳过
94
+ data = line[5:].strip()
95
+ if data == "[DONE]":
96
+ done_seen = True
97
+ break
98
+ try:
99
+ event = json.loads(data)
100
+ except json.JSONDecodeError as exc:
101
+ # 只报位置,不回显原始文本(可能含服务端控制字符/敏感串)
102
+ return _fail(index, t0, f"invalid json at line {exc.lineno} column {exc.colno}")
103
+
104
+ try:
105
+ if not isinstance(event, dict):
106
+ return _fail(index, t0, "invalid event: not an object")
107
+ now = time.perf_counter()
108
+ usage = event.get("usage") or {}
109
+ if usage:
110
+ if usage.get("completion_tokens") is not None:
111
+ output_tokens = _count(usage["completion_tokens"], "completion_tokens")
112
+ if usage.get("prompt_tokens") is not None:
113
+ prompt_tokens = _count(usage["prompt_tokens"], "prompt_tokens")
114
+
115
+ choices = event.get("choices")
116
+ if choices is None:
117
+ choices = []
118
+ if not isinstance(choices, list):
119
+ return _fail(index, t0, "invalid event: choices is not a list")
120
+ if not choices:
121
+ continue # 只带 usage 的收尾块
122
+ choice = choices[0]
123
+ if not isinstance(choice, dict):
124
+ return _fail(index, t0, "invalid event: choice is not an object")
125
+ delta = choice.get("delta")
126
+ if delta is not None and not isinstance(delta, dict):
127
+ return _fail(index, t0, "invalid event: delta is not an object")
128
+ finish = choice.get("finish_reason")
129
+ if finish:
130
+ finish_reason = str(finish)
131
+ content = (delta or {}).get("content")
132
+ if content is not None and not isinstance(content, str):
133
+ # 多模态数组、数字等都不是「文本内容块」,判失败而不是当一个 chunk
134
+ return _fail(index, t0, "invalid event: content is not a string")
135
+ if content:
136
+ content_chunks += 1
137
+ if first_t is None:
138
+ first_t = now
139
+ else:
140
+ intervals.append((now - last_t) * 1000)
141
+ last_t = now
142
+ except (AttributeError, TypeError, KeyError, IndexError) as exc:
143
+ return _fail(index, t0, f"invalid event: {type(exc).__name__}")
144
+ except ValueError as exc:
145
+ return _fail(index, t0, str(exc))
146
+ except httpx.TimeoutException:
147
+ return _fail(index, t0, "timeout")
148
+ except httpx.HTTPError as exc:
149
+ # 协议层异常文本里可能夹带服务端字节(如非法的 chunk 头),只保留异常类名
150
+ return _fail(index, t0, f"http error: {type(exc).__name__}")
151
+
152
+ latency_ms = (time.perf_counter() - t0) * 1000
153
+ if content_chunks == 0:
154
+ return _fail(index, t0, "empty content", latency_ms)
155
+ if finish_reason is not None and finish_reason not in SUCCESS_FINISH_REASONS:
156
+ # 同理不内插服务端取值,只报「非正常结束」这一事实
157
+ return _fail(index, t0, "unsuccessful finish_reason", latency_ms)
158
+ if finish_reason is None and not done_seen:
159
+ return _fail(index, t0, "stream ended without finish_reason", latency_ms)
160
+
161
+ assert first_t is not None and last_t is not None # content_chunks > 0 保证
162
+ return RequestRecord(
163
+ index=index,
164
+ ok=True,
165
+ latency_ms=latency_ms,
166
+ ttft_ms=(first_t - t0) * 1000,
167
+ decode_ms=(last_t - first_t) * 1000,
168
+ content_chunks=content_chunks,
169
+ output_tokens=output_tokens,
170
+ prompt_tokens=prompt_tokens,
171
+ chunk_intervals_ms=tuple(intervals),
172
+ started_at=t0,
173
+ )