ttftkit 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ttftkit-0.2.1/LICENSE +202 -0
- ttftkit-0.2.1/PKG-INFO +195 -0
- ttftkit-0.2.1/README.md +173 -0
- ttftkit-0.2.1/pyproject.toml +51 -0
- ttftkit-0.2.1/setup.cfg +4 -0
- ttftkit-0.2.1/src/ttftkit/__init__.py +3 -0
- ttftkit-0.2.1/src/ttftkit/__main__.py +5 -0
- ttftkit-0.2.1/src/ttftkit/cli.py +159 -0
- ttftkit-0.2.1/src/ttftkit/client.py +173 -0
- ttftkit-0.2.1/src/ttftkit/gate.py +476 -0
- ttftkit-0.2.1/src/ttftkit/hardware.py +270 -0
- ttftkit-0.2.1/src/ttftkit/metrics.py +157 -0
- ttftkit-0.2.1/src/ttftkit/models.py +45 -0
- ttftkit-0.2.1/src/ttftkit/report.py +176 -0
- ttftkit-0.2.1/src/ttftkit/runner.py +88 -0
- ttftkit-0.2.1/src/ttftkit.egg-info/PKG-INFO +195 -0
- ttftkit-0.2.1/src/ttftkit.egg-info/SOURCES.txt +34 -0
- ttftkit-0.2.1/src/ttftkit.egg-info/dependency_links.txt +1 -0
- ttftkit-0.2.1/src/ttftkit.egg-info/entry_points.txt +2 -0
- ttftkit-0.2.1/src/ttftkit.egg-info/requires.txt +8 -0
- ttftkit-0.2.1/src/ttftkit.egg-info/top_level.txt +1 -0
- ttftkit-0.2.1/tests/test_baseline_policy.py +59 -0
- ttftkit-0.2.1/tests/test_cli.py +109 -0
- ttftkit-0.2.1/tests/test_cli_hardware.py +51 -0
- ttftkit-0.2.1/tests/test_client.py +74 -0
- ttftkit-0.2.1/tests/test_client_hardening.py +93 -0
- ttftkit-0.2.1/tests/test_gate.py +79 -0
- ttftkit-0.2.1/tests/test_gate_failclosed.py +160 -0
- ttftkit-0.2.1/tests/test_gate_hardware.py +246 -0
- ttftkit-0.2.1/tests/test_hardware.py +219 -0
- ttftkit-0.2.1/tests/test_invalid.py +46 -0
- ttftkit-0.2.1/tests/test_models.py +53 -0
- ttftkit-0.2.1/tests/test_package.py +33 -0
- ttftkit-0.2.1/tests/test_report.py +131 -0
- ttftkit-0.2.1/tests/test_runner_hardware.py +53 -0
- ttftkit-0.2.1/tests/test_shadow_gate.py +74 -0
ttftkit-0.2.1/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
ttftkit-0.2.1/PKG-INFO
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ttftkit
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: LLM serving benchmark harness and release gate: TTFT / TPOT / throughput / stability, CI-ready
|
|
5
|
+
Author: aidless
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/aidless/ttftkit
|
|
8
|
+
Project-URL: Issues, https://github.com/aidless/ttftkit/issues
|
|
9
|
+
Keywords: llm,inference,benchmark,ttft,tpot,vllm,sglang,performance,ci
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Topic :: System :: Benchmark
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: httpx>=0.27
|
|
16
|
+
Provides-Extra: prom
|
|
17
|
+
Requires-Dist: prometheus-client>=0.20; extra == "prom"
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
20
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# ttftkit
|
|
24
|
+
|
|
25
|
+
[](https://github.com/aidless/ttftkit/actions/workflows/ci.yml)
|
|
26
|
+
|
|
27
|
+
对 OpenAI 兼容推理端点做**可回归的性能测量**:TTFT / TPOT / 吞吐 / 稳定性 → JSON + Markdown 报告 → **发布门禁**(退出码 0/1/2,可直接接 CI)。
|
|
28
|
+
|
|
29
|
+
压测器不缺,缺的是「这次比上次慢了多少、慢到什么程度不许发」的**准入判定**。ttftkit 只做这件事,并且把每个数字的口径写在代码和文档里。
|
|
30
|
+
|
|
31
|
+
v0.2 加了两块:**GPU 显存/利用率采样**(`--gpu auto|none|command`,验收标准是「注入已知泄漏必须看得见单调上涨」)与**影子门禁**(先在 CI 里只留读数、不拦人,观察误报后再收紧)。想按 JD 一条条练,见 [`docs/labs.md`](docs/labs.md)。
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install -e .
|
|
35
|
+
ttftkit run --endpoint http://127.0.0.1:8000/v1/chat/completions \
|
|
36
|
+
--model Qwen/Qwen2.5-7B-Instruct --requests 32 --concurrency 8 \
|
|
37
|
+
--max-tokens 128 --profile rtx4090-vllm-0.6.3 --output results/run.json
|
|
38
|
+
ttftkit gate results/run.json --policy configs/policy.json --baseline baselines/main.json
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## 为什么不用现成的
|
|
42
|
+
|
|
43
|
+
| 工具 | 强项 | ttftkit 的定位 |
|
|
44
|
+
| --- | --- | --- |
|
|
45
|
+
| `vllm bench serve` | 打满单机吞吐,指标全 | 引擎自带,换 SGLang/TGI 就要换一套命令 |
|
|
46
|
+
| `aiperf` / `genai-perf` | 大规模、贴近生产流量 | 重;判定仍要自己写脚本 |
|
|
47
|
+
| `llmperf` | 简单可用 | 无基线对比、无门禁语义 |
|
|
48
|
+
|
|
49
|
+
ttftkit 刻意做小:**只测一个端点、只做闭环并发、只输出可比对的报告**,然后把「能不能发」变成退出码。生态地图见 [`docs/ecosystem.md`](docs/ecosystem.md)。
|
|
50
|
+
|
|
51
|
+
## 指标口径(容易测错的地方)
|
|
52
|
+
|
|
53
|
+
| 指标 | 定义 | 坑 |
|
|
54
|
+
| --- | --- | --- |
|
|
55
|
+
| TTFT | 发出请求 → 首个非空 `delta.content` 事件 | 包含连接、排队、prefill;不是纯 prefill 时间 |
|
|
56
|
+
| TPOT | `decode_ms / (completion_tokens − 1)` | 用服务端 `usage` 的 token 数,**不是 SSE 块数**;缺 usage 时是 `null`,不是 0 |
|
|
57
|
+
| token 吞吐 | `Σ completion_tokens / 测量墙钟时长` | 分母是墙钟,不是各请求延迟之和;任一成功请求缺计数则整体 `null` |
|
|
58
|
+
| 错误率 | 失败请求 / 全部请求 | 失败请求不参与延迟统计,但**一定要进分母** |
|
|
59
|
+
| 稳定性 | `latency_ms.cv`、`degradation_ratio`(后 1/4 ÷ 前 1/4 延迟) | 后者专抓跑久了变慢(显存泄漏、KV 碎片、降频) |
|
|
60
|
+
| 显存增长 | `mem_used_peak_mb − mem_used_baseline_mb` | 没采到样时是 `null`,**不是 0**;`0 MB` 会被读成「测到了、没占显存」 |
|
|
61
|
+
|
|
62
|
+
一个 SSE 块可能含多个 token,所以块间隔只作为传输层观测记录(`chunk_gap_ms`),**不叫 ITL**。详细推导与对齐说明见 [`docs/metrics.md`](docs/metrics.md)。
|
|
63
|
+
|
|
64
|
+
## 门禁
|
|
65
|
+
|
|
66
|
+
策略文件支持两种规则,可同时使用:
|
|
67
|
+
|
|
68
|
+
```json
|
|
69
|
+
{
|
|
70
|
+
"limits": { "ttft_ms.p95": 800, "tpot_ms.p95": 60, "error_rate": 0.0 },
|
|
71
|
+
"regressions": { "ttft_ms.p95": 10, "output_tokens_per_s": 5 }
|
|
72
|
+
}
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
- `limits`:绝对上限/下限,越界即 FAIL。
|
|
76
|
+
- `regressions`:相对基线允许的退化百分比(延迟类越小越好、吞吐类越大越好,方向内置)。
|
|
77
|
+
- 退出码:`0` 通过(可能带 WARN 贴边提示)|`1` 有规则不通过、失败数超过预算(默认 0)、或缺指标|`2` 策略/报告无效,或两次测量不可比。
|
|
78
|
+
|
|
79
|
+
**fail-closed 的地方**(这些都是踩过的坑,不是设计洁癖):
|
|
80
|
+
|
|
81
|
+
- 指标缺失、`NaN`、`inf`、字符串 `"100"`、布尔 `true` → 不通过,绝不当 0 或当通过。
|
|
82
|
+
- 空策略 `{}`、未知指标名(拼错)→ 配置错误(`2`),否则等于什么也没检查。
|
|
83
|
+
- 相对比较要求 `protocol / model / concurrency / max_tokens / requests / prompt_sha256 / profile` 完全一致,否则两次不是同一个实验。
|
|
84
|
+
- 报告里 `requests` 为 0 或 `successful_requests > requests` → 无证据可判。
|
|
85
|
+
- 分布指标的 `count` 少于成功请求数(例如一半请求缺 usage 导致 TPOT 只算了一半)→ 不通过。
|
|
86
|
+
- **基线也要可判定**:baseline 缺 `successful_requests`、`requests=0`、或分布 `count` 不可用 → 配置错误(`2`),不能拿 1 个样本当基准。
|
|
87
|
+
- 失败预算:默认任何失败都拦;若策略写了 `limits.error_rate`,则按该预算判定(预算内的小比例失败放行)。
|
|
88
|
+
|
|
89
|
+
策略写法与取舍见 [`docs/gate.md`](docs/gate.md)。
|
|
90
|
+
|
|
91
|
+
## 显存与 GPU 采样(v0.2)
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
# auto:探测 nvidia-smi → rocm-smi,都没有就不采样(报告里明写「未采样」)
|
|
95
|
+
ttftkit run --endpoint ... --model ... --gpu auto
|
|
96
|
+
# 自定义采样命令(stdout 须是 nvidia-smi 的 CSV 格式)
|
|
97
|
+
ttftkit run --endpoint ... --model ... --gpu command \
|
|
98
|
+
--gpu-command "nvidia-smi --query-gpu=name,driver_version,memory.total,memory.used,utilization.gpu --format=csv,noheader,nounits"
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
报告新增 `hardware` 段(来源、设备、驱动、`mem_used_baseline_mb` / `mem_used_peak_mb` / `mem_used_delta_mb`、峰值利用率、采样次数与错误数),
|
|
102
|
+
summary 里对应三个标量,可直接写进门禁:
|
|
103
|
+
|
|
104
|
+
```json
|
|
105
|
+
{ "limits": { "mem_used_delta_mb": 512 }, "regressions": { "mem_used_peak_mb": 10 } }
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
显存的相对回归要求两侧报告 `config.hardware` 指纹一致(`"设备名/总量MB"`),否则判**不可比**(退出码 `2`)——
|
|
109
|
+
24 GB 卡上的数字不能拿去和 6 GB 卡比。细节、三层验证与边界见 [`docs/hardware.md`](docs/hardware.md)。
|
|
110
|
+
|
|
111
|
+
## 基线来自实测,不是拍的
|
|
112
|
+
|
|
113
|
+
`baselines/` 里是同一台机器(Ryzen 5800H、CPU-only、ollama `qwen2.5-coder:0.5b`)连跑 5 次的结果,
|
|
114
|
+
`baselines/main.json` 取其中 TTFT p50 最接近中位数的那一次。5 次之间的自变异写在 [`baselines/README.md`](baselines/README.md):
|
|
115
|
+
TTFT p95 极差 **143.3%**、p50 37.8%、吞吐 46.6% —— 所以门槛不能按「退化 10% 就拦」拍,`configs/policy.baseline.json` 的阈值就是从这张表推出来的。
|
|
116
|
+
|
|
117
|
+
CI 里先用**影子模式**(`scripts/shadow_gate.py` 不带 `--strict` 时恒返回 0,只打印逐条 PASS/FAIL/WARN),
|
|
118
|
+
观察一段时间误报率再改 `--strict` 上强制门。
|
|
119
|
+
|
|
120
|
+
## 接 CI
|
|
121
|
+
|
|
122
|
+
```yaml
|
|
123
|
+
- name: Benchmark
|
|
124
|
+
run: |
|
|
125
|
+
ttftkit run --endpoint "$ENDPOINT" --model "$MODEL" --requests 32 --concurrency 8 \
|
|
126
|
+
--max-tokens 128 --profile "$PROFILE" --output results/run.json
|
|
127
|
+
- name: Release gate
|
|
128
|
+
run: ttftkit gate results/run.json --policy configs/policy.json --baseline baselines/main.json
|
|
129
|
+
# 或者先用影子门禁:只记录读数,不阻断
|
|
130
|
+
- name: Shadow gate (non-blocking)
|
|
131
|
+
continue-on-error: true
|
|
132
|
+
run: python scripts/shadow_gate.py --report results/run.json --baseline baselines/main.json --policy configs/policy.baseline.json
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
`run` 自身在**有请求失败时返回 1**,所以即使不配 `gate` 也不会把坏结果当成功。`gate` 则按策略里的失败预算判(默认 0,即任何失败都不放行)。
|
|
136
|
+
|
|
137
|
+
## 一次真实运行(本机 CPU,仅演示链路)
|
|
138
|
+
|
|
139
|
+
Ryzen 5800H / 无 GPU / ollama `qwen2.5-coder:0.5b`,3 请求、并发 1、`max_tokens=16`:
|
|
140
|
+
|
|
141
|
+
| 指标 | 值 |
|
|
142
|
+
| --- | --- |
|
|
143
|
+
| TTFT p50 | 84.6 ms |
|
|
144
|
+
| TPOT 均值 | 40.7 ms/token |
|
|
145
|
+
| token 吞吐 | 23.25 tok/s |
|
|
146
|
+
| token 计数覆盖率 | 1.0 |
|
|
147
|
+
| 错误率 | 0 |
|
|
148
|
+
|
|
149
|
+
这是**链路可用性演示**,不是任何模型的性能结论:样本 3 个、CPU 推理、无 GPU 参与。
|
|
150
|
+
|
|
151
|
+
> **复算**:上表每个数字都来自 `results/smoke-ollama.json`(schema 里 `summary` 段),
|
|
152
|
+
> 用 `python3 -c "import json;d=json.load(open('results/smoke-ollama.json'));print(d['summary']['ttft_ms']['p50'], d['summary']['tpot_ms']['mean'], d['summary']['output_tokens_per_s'])"`
|
|
153
|
+
> 即可逐项重取(预期 84.61 / 40.72 / 23.25)。触发该次运行的命令见同目录 `results/README.md`。
|
|
154
|
+
> 注意:这是 CPU + 3 样本的一次演示,**不构成任何模型性能结论**。
|
|
155
|
+
|
|
156
|
+
## 证明什么 / 不证明什么
|
|
157
|
+
|
|
158
|
+
**证明(全部有源码 + 测试 + 可复算证据支撑)**
|
|
159
|
+
|
|
160
|
+
- 这套测量与门禁链路**是闭环、可运行、可判定的**:`run` → 报告 → `gate` → 退出码 `0/1/2`,
|
|
161
|
+
在真实本地 SSE 服务上跑通(`tests/` 含真实本地 SSE 服务用例)。
|
|
162
|
+
- 门禁**fail-closed**:指标缺失/`NaN`/`inf`/字符串/布尔 → 不通过;空策略/未知指标名 → 配置错误(`2`);
|
|
163
|
+
相对比较要求协议/模型/并发/token/请求数/prompt 哈希/指纹完全一致,否则不可比。
|
|
164
|
+
- 已知退化**会被拦住**:`scripts/red_check.py` 逐条回退 32 处修复,对应用例全部变红(32/32),
|
|
165
|
+
证明这些判定不是摆设。
|
|
166
|
+
- 硬件块**自洽且可溯源**:`delta == peak − baseline`、负数/超设备总量 → 配置错误、
|
|
167
|
+
`config.hardware` 必须等于用块重算的指纹(防改块绕过可比性门禁)。
|
|
168
|
+
- 基线与阈值**来自同机 5 次实测**,不是拍的:`baselines/README.md` 给出 5 次全表 + 复现命令,
|
|
169
|
+
`tests/test_baseline_policy.py` 验证 5 次真实跨跑全 PASS、人为退化全 FAIL(双向,非装饰)。
|
|
170
|
+
|
|
171
|
+
**不证明(这些是边界,不要过度解读)**
|
|
172
|
+
|
|
173
|
+
- **真 GPU 读数尚未实测**:本机没有 NVIDIA 用户态库/设备,采样链路只用假采样器(`scripts/fake_gpu.py` 注入已知泄漏)验证过三层。真卡(nvidia-smi / rocm-smi)读数需要在云 GPU 上跑通再写进本 README,不发布未实测数字。
|
|
174
|
+
- 不构成**任何模型的性能结论**:README 里的演示样本(CPU、3 请求、ollama 0.5b)只证明链路可用,不证明模型快慢。
|
|
175
|
+
- 不做**模型横向比较**:单机、单模型、固定配置,换了配置/机器就不是同一个实验(可比性门禁会判 `2`)。
|
|
176
|
+
- 显存是**设备级 + 轮询**:同卡其他进程的变化会混进 `delta`,间隔内的单次尖峰可能被漏掉(见 [`docs/hardware.md`](docs/hardware.md))。
|
|
177
|
+
- 闭环固定并发,**不模拟泊松到达或真实流量分布**;要那个用 `aiperf`。
|
|
178
|
+
- TPOT 是**客户端近似**:同一 SSE 块内的 token 时间无法拆分。
|
|
179
|
+
- **不做分布式多机压测**,不测第三方付费 API 的绝对性能(网络抖动会污染结果)。
|
|
180
|
+
|
|
181
|
+
## 开发
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
python3 -m pytest -q # 全量测试(含真实本地 SSE 服务)
|
|
185
|
+
python3 -m ruff check src tests scripts
|
|
186
|
+
python3 scripts/red_check.py # RED 反证:把修复回退后,对应测试必须变红
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
`red_check.py` 是变异检查的自动化版本:在临时副本上逐条回退修复(TTFT 取值时机、门禁边界、吞吐分母、token 口径、坏 `choices`、总时限、错误串回显、非字符串 `content`、残缺报告、warmup 退出码、`count=0` 带分位数、显存 `null`→`0`、硬件可比性、影子门禁退出码、label 恒 PASS、利用率方向、stop 截断、单样本 delta、零样本块、指纹 `none`、证据链不一致、影子读坏输入、指纹溯源、截断读数、物理不可能值、Collector 参数),跑对应用例并断言它**失败**。当前 32/32 全部变红——修复若只是摆设,这一步会变绿并报错。
|
|
190
|
+
|
|
191
|
+
按 JD 练习的路径见 [`docs/labs.md`](docs/labs.md)(8 个实验,每个都有可复跑的通过判据)。
|
|
192
|
+
|
|
193
|
+
## License
|
|
194
|
+
|
|
195
|
+
Apache-2.0
|
ttftkit-0.2.1/README.md
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# ttftkit
|
|
2
|
+
|
|
3
|
+
[](https://github.com/aidless/ttftkit/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
对 OpenAI 兼容推理端点做**可回归的性能测量**:TTFT / TPOT / 吞吐 / 稳定性 → JSON + Markdown 报告 → **发布门禁**(退出码 0/1/2,可直接接 CI)。
|
|
6
|
+
|
|
7
|
+
压测器不缺,缺的是「这次比上次慢了多少、慢到什么程度不许发」的**准入判定**。ttftkit 只做这件事,并且把每个数字的口径写在代码和文档里。
|
|
8
|
+
|
|
9
|
+
v0.2 加了两块:**GPU 显存/利用率采样**(`--gpu auto|none|command`,验收标准是「注入已知泄漏必须看得见单调上涨」)与**影子门禁**(先在 CI 里只留读数、不拦人,观察误报后再收紧)。想按 JD 一条条练,见 [`docs/labs.md`](docs/labs.md)。
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install -e .
|
|
13
|
+
ttftkit run --endpoint http://127.0.0.1:8000/v1/chat/completions \
|
|
14
|
+
--model Qwen/Qwen2.5-7B-Instruct --requests 32 --concurrency 8 \
|
|
15
|
+
--max-tokens 128 --profile rtx4090-vllm-0.6.3 --output results/run.json
|
|
16
|
+
ttftkit gate results/run.json --policy configs/policy.json --baseline baselines/main.json
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## 为什么不用现成的
|
|
20
|
+
|
|
21
|
+
| 工具 | 强项 | ttftkit 的定位 |
|
|
22
|
+
| --- | --- | --- |
|
|
23
|
+
| `vllm bench serve` | 打满单机吞吐,指标全 | 引擎自带,换 SGLang/TGI 就要换一套命令 |
|
|
24
|
+
| `aiperf` / `genai-perf` | 大规模、贴近生产流量 | 重;判定仍要自己写脚本 |
|
|
25
|
+
| `llmperf` | 简单可用 | 无基线对比、无门禁语义 |
|
|
26
|
+
|
|
27
|
+
ttftkit 刻意做小:**只测一个端点、只做闭环并发、只输出可比对的报告**,然后把「能不能发」变成退出码。生态地图见 [`docs/ecosystem.md`](docs/ecosystem.md)。
|
|
28
|
+
|
|
29
|
+
## 指标口径(容易测错的地方)
|
|
30
|
+
|
|
31
|
+
| 指标 | 定义 | 坑 |
|
|
32
|
+
| --- | --- | --- |
|
|
33
|
+
| TTFT | 发出请求 → 首个非空 `delta.content` 事件 | 包含连接、排队、prefill;不是纯 prefill 时间 |
|
|
34
|
+
| TPOT | `decode_ms / (completion_tokens − 1)` | 用服务端 `usage` 的 token 数,**不是 SSE 块数**;缺 usage 时是 `null`,不是 0 |
|
|
35
|
+
| token 吞吐 | `Σ completion_tokens / 测量墙钟时长` | 分母是墙钟,不是各请求延迟之和;任一成功请求缺计数则整体 `null` |
|
|
36
|
+
| 错误率 | 失败请求 / 全部请求 | 失败请求不参与延迟统计,但**一定要进分母** |
|
|
37
|
+
| 稳定性 | `latency_ms.cv`、`degradation_ratio`(后 1/4 ÷ 前 1/4 延迟) | 后者专抓跑久了变慢(显存泄漏、KV 碎片、降频) |
|
|
38
|
+
| 显存增长 | `mem_used_peak_mb − mem_used_baseline_mb` | 没采到样时是 `null`,**不是 0**;`0 MB` 会被读成「测到了、没占显存」 |
|
|
39
|
+
|
|
40
|
+
一个 SSE 块可能含多个 token,所以块间隔只作为传输层观测记录(`chunk_gap_ms`),**不叫 ITL**。详细推导与对齐说明见 [`docs/metrics.md`](docs/metrics.md)。
|
|
41
|
+
|
|
42
|
+
## 门禁
|
|
43
|
+
|
|
44
|
+
策略文件支持两种规则,可同时使用:
|
|
45
|
+
|
|
46
|
+
```json
|
|
47
|
+
{
|
|
48
|
+
"limits": { "ttft_ms.p95": 800, "tpot_ms.p95": 60, "error_rate": 0.0 },
|
|
49
|
+
"regressions": { "ttft_ms.p95": 10, "output_tokens_per_s": 5 }
|
|
50
|
+
}
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
- `limits`:绝对上限/下限,越界即 FAIL。
|
|
54
|
+
- `regressions`:相对基线允许的退化百分比(延迟类越小越好、吞吐类越大越好,方向内置)。
|
|
55
|
+
- 退出码:`0` 通过(可能带 WARN 贴边提示)|`1` 有规则不通过、失败数超过预算(默认 0)、或缺指标|`2` 策略/报告无效,或两次测量不可比。
|
|
56
|
+
|
|
57
|
+
**fail-closed 的地方**(这些都是踩过的坑,不是设计洁癖):
|
|
58
|
+
|
|
59
|
+
- 指标缺失、`NaN`、`inf`、字符串 `"100"`、布尔 `true` → 不通过,绝不当 0 或当通过。
|
|
60
|
+
- 空策略 `{}`、未知指标名(拼错)→ 配置错误(`2`),否则等于什么也没检查。
|
|
61
|
+
- 相对比较要求 `protocol / model / concurrency / max_tokens / requests / prompt_sha256 / profile` 完全一致,否则两次不是同一个实验。
|
|
62
|
+
- 报告里 `requests` 为 0 或 `successful_requests > requests` → 无证据可判。
|
|
63
|
+
- 分布指标的 `count` 少于成功请求数(例如一半请求缺 usage 导致 TPOT 只算了一半)→ 不通过。
|
|
64
|
+
- **基线也要可判定**:baseline 缺 `successful_requests`、`requests=0`、或分布 `count` 不可用 → 配置错误(`2`),不能拿 1 个样本当基准。
|
|
65
|
+
- 失败预算:默认任何失败都拦;若策略写了 `limits.error_rate`,则按该预算判定(预算内的小比例失败放行)。
|
|
66
|
+
|
|
67
|
+
策略写法与取舍见 [`docs/gate.md`](docs/gate.md)。
|
|
68
|
+
|
|
69
|
+
## 显存与 GPU 采样(v0.2)
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
# auto:探测 nvidia-smi → rocm-smi,都没有就不采样(报告里明写「未采样」)
|
|
73
|
+
ttftkit run --endpoint ... --model ... --gpu auto
|
|
74
|
+
# 自定义采样命令(stdout 须是 nvidia-smi 的 CSV 格式)
|
|
75
|
+
ttftkit run --endpoint ... --model ... --gpu command \
|
|
76
|
+
--gpu-command "nvidia-smi --query-gpu=name,driver_version,memory.total,memory.used,utilization.gpu --format=csv,noheader,nounits"
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
报告新增 `hardware` 段(来源、设备、驱动、`mem_used_baseline_mb` / `mem_used_peak_mb` / `mem_used_delta_mb`、峰值利用率、采样次数与错误数),
|
|
80
|
+
summary 里对应三个标量,可直接写进门禁:
|
|
81
|
+
|
|
82
|
+
```json
|
|
83
|
+
{ "limits": { "mem_used_delta_mb": 512 }, "regressions": { "mem_used_peak_mb": 10 } }
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
显存的相对回归要求两侧报告 `config.hardware` 指纹一致(`"设备名/总量MB"`),否则判**不可比**(退出码 `2`)——
|
|
87
|
+
24 GB 卡上的数字不能拿去和 6 GB 卡比。细节、三层验证与边界见 [`docs/hardware.md`](docs/hardware.md)。
|
|
88
|
+
|
|
89
|
+
## 基线来自实测,不是拍的
|
|
90
|
+
|
|
91
|
+
`baselines/` 里是同一台机器(Ryzen 5800H、CPU-only、ollama `qwen2.5-coder:0.5b`)连跑 5 次的结果,
|
|
92
|
+
`baselines/main.json` 取其中 TTFT p50 最接近中位数的那一次。5 次之间的自变异写在 [`baselines/README.md`](baselines/README.md):
|
|
93
|
+
TTFT p95 极差 **143.3%**、p50 37.8%、吞吐 46.6% —— 所以门槛不能按「退化 10% 就拦」拍,`configs/policy.baseline.json` 的阈值就是从这张表推出来的。
|
|
94
|
+
|
|
95
|
+
CI 里先用**影子模式**(`scripts/shadow_gate.py` 不带 `--strict` 时恒返回 0,只打印逐条 PASS/FAIL/WARN),
|
|
96
|
+
观察一段时间误报率再改 `--strict` 上强制门。
|
|
97
|
+
|
|
98
|
+
## 接 CI
|
|
99
|
+
|
|
100
|
+
```yaml
|
|
101
|
+
- name: Benchmark
|
|
102
|
+
run: |
|
|
103
|
+
ttftkit run --endpoint "$ENDPOINT" --model "$MODEL" --requests 32 --concurrency 8 \
|
|
104
|
+
--max-tokens 128 --profile "$PROFILE" --output results/run.json
|
|
105
|
+
- name: Release gate
|
|
106
|
+
run: ttftkit gate results/run.json --policy configs/policy.json --baseline baselines/main.json
|
|
107
|
+
# 或者先用影子门禁:只记录读数,不阻断
|
|
108
|
+
- name: Shadow gate (non-blocking)
|
|
109
|
+
continue-on-error: true
|
|
110
|
+
run: python scripts/shadow_gate.py --report results/run.json --baseline baselines/main.json --policy configs/policy.baseline.json
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
`run` 自身在**有请求失败时返回 1**,所以即使不配 `gate` 也不会把坏结果当成功。`gate` 则按策略里的失败预算判(默认 0,即任何失败都不放行)。
|
|
114
|
+
|
|
115
|
+
## 一次真实运行(本机 CPU,仅演示链路)
|
|
116
|
+
|
|
117
|
+
Ryzen 5800H / 无 GPU / ollama `qwen2.5-coder:0.5b`,3 请求、并发 1、`max_tokens=16`:
|
|
118
|
+
|
|
119
|
+
| 指标 | 值 |
|
|
120
|
+
| --- | --- |
|
|
121
|
+
| TTFT p50 | 84.6 ms |
|
|
122
|
+
| TPOT 均值 | 40.7 ms/token |
|
|
123
|
+
| token 吞吐 | 23.25 tok/s |
|
|
124
|
+
| token 计数覆盖率 | 1.0 |
|
|
125
|
+
| 错误率 | 0 |
|
|
126
|
+
|
|
127
|
+
这是**链路可用性演示**,不是任何模型的性能结论:样本 3 个、CPU 推理、无 GPU 参与。
|
|
128
|
+
|
|
129
|
+
> **复算**:上表每个数字都来自 `results/smoke-ollama.json`(schema 里 `summary` 段),
|
|
130
|
+
> 用 `python3 -c "import json;d=json.load(open('results/smoke-ollama.json'));print(d['summary']['ttft_ms']['p50'], d['summary']['tpot_ms']['mean'], d['summary']['output_tokens_per_s'])"`
|
|
131
|
+
> 即可逐项重取(预期 84.61 / 40.72 / 23.25)。触发该次运行的命令见同目录 `results/README.md`。
|
|
132
|
+
> 注意:这是 CPU + 3 样本的一次演示,**不构成任何模型性能结论**。
|
|
133
|
+
|
|
134
|
+
## 证明什么 / 不证明什么
|
|
135
|
+
|
|
136
|
+
**证明(全部有源码 + 测试 + 可复算证据支撑)**
|
|
137
|
+
|
|
138
|
+
- 这套测量与门禁链路**是闭环、可运行、可判定的**:`run` → 报告 → `gate` → 退出码 `0/1/2`,
|
|
139
|
+
在真实本地 SSE 服务上跑通(`tests/` 含真实本地 SSE 服务用例)。
|
|
140
|
+
- 门禁**fail-closed**:指标缺失/`NaN`/`inf`/字符串/布尔 → 不通过;空策略/未知指标名 → 配置错误(`2`);
|
|
141
|
+
相对比较要求协议/模型/并发/token/请求数/prompt 哈希/指纹完全一致,否则不可比。
|
|
142
|
+
- 已知退化**会被拦住**:`scripts/red_check.py` 逐条回退 32 处修复,对应用例全部变红(32/32),
|
|
143
|
+
证明这些判定不是摆设。
|
|
144
|
+
- 硬件块**自洽且可溯源**:`delta == peak − baseline`、负数/超设备总量 → 配置错误、
|
|
145
|
+
`config.hardware` 必须等于用块重算的指纹(防改块绕过可比性门禁)。
|
|
146
|
+
- 基线与阈值**来自同机 5 次实测**,不是拍的:`baselines/README.md` 给出 5 次全表 + 复现命令,
|
|
147
|
+
`tests/test_baseline_policy.py` 验证 5 次真实跨跑全 PASS、人为退化全 FAIL(双向,非装饰)。
|
|
148
|
+
|
|
149
|
+
**不证明(这些是边界,不要过度解读)**
|
|
150
|
+
|
|
151
|
+
- **真 GPU 读数尚未实测**:本机没有 NVIDIA 用户态库/设备,采样链路只用假采样器(`scripts/fake_gpu.py` 注入已知泄漏)验证过三层。真卡(nvidia-smi / rocm-smi)读数需要在云 GPU 上跑通再写进本 README,不发布未实测数字。
|
|
152
|
+
- 不构成**任何模型的性能结论**:README 里的演示样本(CPU、3 请求、ollama 0.5b)只证明链路可用,不证明模型快慢。
|
|
153
|
+
- 不做**模型横向比较**:单机、单模型、固定配置,换了配置/机器就不是同一个实验(可比性门禁会判 `2`)。
|
|
154
|
+
- 显存是**设备级 + 轮询**:同卡其他进程的变化会混进 `delta`,间隔内的单次尖峰可能被漏掉(见 [`docs/hardware.md`](docs/hardware.md))。
|
|
155
|
+
- 闭环固定并发,**不模拟泊松到达或真实流量分布**;要那个用 `aiperf`。
|
|
156
|
+
- TPOT 是**客户端近似**:同一 SSE 块内的 token 时间无法拆分。
|
|
157
|
+
- **不做分布式多机压测**,不测第三方付费 API 的绝对性能(网络抖动会污染结果)。
|
|
158
|
+
|
|
159
|
+
## 开发
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
python3 -m pytest -q # 全量测试(含真实本地 SSE 服务)
|
|
163
|
+
python3 -m ruff check src tests scripts
|
|
164
|
+
python3 scripts/red_check.py # RED 反证:把修复回退后,对应测试必须变红
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
`red_check.py` 是变异检查的自动化版本:在临时副本上逐条回退修复(TTFT 取值时机、门禁边界、吞吐分母、token 口径、坏 `choices`、总时限、错误串回显、非字符串 `content`、残缺报告、warmup 退出码、`count=0` 带分位数、显存 `null`→`0`、硬件可比性、影子门禁退出码、label 恒 PASS、利用率方向、stop 截断、单样本 delta、零样本块、指纹 `none`、证据链不一致、影子读坏输入、指纹溯源、截断读数、物理不可能值、Collector 参数),跑对应用例并断言它**失败**。当前 32/32 全部变红——修复若只是摆设,这一步会变绿并报错。
|
|
168
|
+
|
|
169
|
+
按 JD 练习的路径见 [`docs/labs.md`](docs/labs.md)(8 个实验,每个都有可复跑的通过判据)。
|
|
170
|
+
|
|
171
|
+
## License
|
|
172
|
+
|
|
173
|
+
Apache-2.0
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ttftkit"
|
|
7
|
+
version = "0.2.1"
|
|
8
|
+
description = "LLM serving benchmark harness and release gate: TTFT / TPOT / throughput / stability, CI-ready"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "Apache-2.0"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "aidless" }]
|
|
14
|
+
keywords = ["llm", "inference", "benchmark", "ttft", "tpot", "vllm", "sglang", "performance", "ci"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Topic :: System :: Benchmark",
|
|
18
|
+
]
|
|
19
|
+
dependencies = [
|
|
20
|
+
"httpx>=0.27",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
prom = ["prometheus-client>=0.20"]
|
|
25
|
+
dev = ["pytest>=8", "ruff>=0.5"]
|
|
26
|
+
|
|
27
|
+
[project.scripts]
|
|
28
|
+
ttftkit = "ttftkit.cli:main"
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://github.com/aidless/ttftkit"
|
|
32
|
+
Issues = "https://github.com/aidless/ttftkit/issues"
|
|
33
|
+
|
|
34
|
+
[tool.setuptools.packages.find]
|
|
35
|
+
where = ["src"]
|
|
36
|
+
|
|
37
|
+
[tool.pytest.ini_options]
|
|
38
|
+
pythonpath = ["src"]
|
|
39
|
+
testpaths = ["tests"]
|
|
40
|
+
addopts = "-q"
|
|
41
|
+
|
|
42
|
+
[tool.ruff]
|
|
43
|
+
line-length = 110
|
|
44
|
+
target-version = "py310"
|
|
45
|
+
|
|
46
|
+
[tool.ruff.lint]
|
|
47
|
+
# 显式固定规则集,避免不同环境的 ruff 默认集不一致导致 CI 与本地结论漂移。
|
|
48
|
+
select = ["E", "F", "W", "I", "UP", "B", "SIM", "C4", "ASYNC"]
|
|
49
|
+
# TRY004:输入校验统一抛 ValueError(CLI 统一映射为退出码 2),不用 TypeError——有意的接口契约。
|
|
50
|
+
# ISC004:报告模板里多行字符串/字节串隐式拼接更易读。
|
|
51
|
+
ignore = ["TRY004", "ISC004"]
|
ttftkit-0.2.1/setup.cfg
ADDED