ttftkit 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ttftkit/gate.py ADDED
@@ -0,0 +1,476 @@
1
+ """回归门禁:把两份报告与一套规则变成退出码。
2
+
3
+ 退出码语义(fail-closed):
4
+ - 0 通过(可能带 WARN)
5
+ - 1 有规则不通过,或有失败请求,或缺指标
6
+ - 2 规则/报告无效,或两次测量不可比
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import math
11
+
12
+ from .hardware import fingerprint as _block_fingerprint
13
+
14
+ # 已知指标路径:规则里写别的路径视为配置错误(防止拼错后静默不检查)
15
+ DIST_FIELDS = ("count", "mean", "min", "max", "p50", "p90", "p95", "p99", "stdev", "cv")
16
+ DIST_METRICS = ("ttft_ms", "tpot_ms", "latency_ms", "decode_ms", "chunk_gap_ms")
17
+ SCALAR_METRICS = (
18
+ "requests",
19
+ "successful_requests",
20
+ "error_rate",
21
+ "requests_per_s",
22
+ "output_tokens_per_s",
23
+ "token_usage_coverage",
24
+ "goodput_per_s",
25
+ "degradation_ratio",
26
+ "mem_used_peak_mb",
27
+ "mem_used_delta_mb",
28
+ "gpu_util_peak_pct",
29
+ )
30
+ # 硬件指标的相对比较只在同一块卡上有意义:显存总量不同 = 不可比
31
+ HARDWARE_METRICS = frozenset({"mem_used_peak_mb", "mem_used_delta_mb", "gpu_util_peak_pct"})
32
+ # summary 标量与 hardware 块字段的一一对应:两者必须一致,否则报告是拼出来的
33
+ _HW_FIELDS = {
34
+ "mem_used_peak_mb": "mem_used_peak_mb",
35
+ "mem_used_delta_mb": "mem_used_delta_mb",
36
+ "gpu_util_peak_pct": "utilization_peak_pct",
37
+ }
38
+ MISSING_FINGERPRINTS = frozenset({"", "none", "None", "null"})
39
+ KNOWN_METRICS = frozenset(
40
+ [f"{m}.{f}" for m in DIST_METRICS for f in DIST_FIELDS] + list(SCALAR_METRICS)
41
+ )
42
+
43
+ # 这些分布的 count 是「参与统计的请求数」,不能超过成功请求数。
44
+ # chunk_gap_ms 的 count 是块间隔个数(可多于请求数),故不在其列。
45
+ PER_REQUEST_DISTS = ("ttft_ms", "tpot_ms", "latency_ms", "decode_ms")
46
+
47
+ HIGHER_IS_BETTER = frozenset(
48
+ {"requests", "successful_requests", "requests_per_s", "output_tokens_per_s",
49
+ "token_usage_coverage", "goodput_per_s", "gpu_util_peak_pct"}
50
+ )
51
+ # 注:gpu_util_peak_pct 方向是「越高越好」——策略里的 limits 是下限(要求真的跑满了卡),
52
+ # 而不是上限。写成上限会把「利用率 99% 的正常压测」判失败、把空转判通过。
53
+
54
+ CONFIG_KEYS = ("protocol", "model", "concurrency", "max_tokens", "requests", "prompt_sha256", "profile")
55
+
56
+
57
+ class _ConfigError(Exception):
58
+ """规则或输入无效 → 退出码 2。"""
59
+
60
+
61
+ def _number(value, what: str) -> float:
62
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
63
+ raise _ConfigError(f"{what} must be a number, got {value!r}")
64
+ if not math.isfinite(value):
65
+ raise _ConfigError(f"{what} must be finite, got {value!r}")
66
+ return float(value)
67
+
68
+
69
+ def _validate_rules(rules, baseline, candidate) -> dict:
70
+ if not isinstance(rules, dict):
71
+ raise _ConfigError("policy must be a JSON object")
72
+ unknown = set(rules) - {"limits", "regressions"}
73
+ if unknown:
74
+ raise _ConfigError(f"unknown policy keys: {sorted(unknown)}")
75
+ raw_limits = rules.get("limits", {})
76
+ raw_regressions = rules.get("regressions", {})
77
+ for key, value in (("limits", raw_limits), ("regressions", raw_regressions)):
78
+ if value is not None and not isinstance(value, dict):
79
+ raise _ConfigError(f"{key!r} must be a JSON object, got {type(value).__name__}")
80
+ limits = raw_limits or {}
81
+ regressions = raw_regressions or {}
82
+ if not limits and not regressions:
83
+ raise _ConfigError("policy is empty: no rule would ever run")
84
+
85
+ for name, value in limits.items():
86
+ if name not in KNOWN_METRICS:
87
+ raise _ConfigError(f"unknown metric in limits: {name!r}")
88
+ if _number(value, f"limit for {name}") < 0:
89
+ raise _ConfigError(f"limit for {name} must be >= 0")
90
+
91
+ for name, value in regressions.items():
92
+ if name not in KNOWN_METRICS:
93
+ raise _ConfigError(f"unknown metric in regressions: {name!r}")
94
+ if _number(value, f"regression tolerance for {name}") < 0:
95
+ raise _ConfigError(f"regression tolerance for {name} must be >= 0")
96
+ if baseline is None:
97
+ raise _ConfigError("relative policy requires a baseline report")
98
+ _check_comparable(candidate, baseline, name)
99
+ return {"limits": limits, "regressions": regressions}
100
+
101
+
102
+ def _check_comparable(candidate: dict, baseline: dict, metric: str) -> None:
103
+ """相对比较要求两次测量的协议指纹完全一致,否则不是同一个实验。"""
104
+ for label, report in (("baseline", baseline), ("candidate", candidate)):
105
+ if not isinstance(report, dict) or not isinstance(report.get("config"), dict):
106
+ raise _ConfigError(f"{label} report has no config block")
107
+ for key in CONFIG_KEYS:
108
+ if key not in report["config"]:
109
+ raise _ConfigError(f"{label} config missing {key!r} (cannot compare for {metric})")
110
+ differing = [k for k in CONFIG_KEYS if candidate["config"][k] != baseline["config"][k]]
111
+ if differing:
112
+ raise _ConfigError(f"measurements are not comparable for {metric}: {differing} differ")
113
+
114
+
115
+ def _lookup(summary: dict, path: str):
116
+ """按点路径取值;缺键或非数值返回 None(由调用方判 FAIL)。"""
117
+ node = summary
118
+ for part in path.split("."):
119
+ if not isinstance(node, dict) or part not in node:
120
+ return None
121
+ node = node[part]
122
+ if isinstance(node, bool) or not isinstance(node, (int, float)):
123
+ return None
124
+ if not math.isfinite(node):
125
+ return None
126
+ return float(node)
127
+
128
+
129
+ def _validate_summary(summary: dict) -> None:
130
+ """报告必须携带可判定的证据量,否则不是「通过」而是「没测」。"""
131
+ requests = summary.get("requests")
132
+ successful = summary.get("successful_requests")
133
+ if isinstance(requests, bool) or not isinstance(requests, int) or requests <= 0:
134
+ raise _ConfigError("report has no positive integer 'requests' — no evidence to judge")
135
+ if isinstance(successful, bool) or not isinstance(successful, int) or not 0 <= successful <= requests:
136
+ raise _ConfigError("'successful_requests' must be an integer within [0, requests]")
137
+ if "error_rate" not in summary:
138
+ raise _ConfigError("report has no 'error_rate' — cannot tell whether any request failed")
139
+ error_rate = summary["error_rate"]
140
+ if (
141
+ isinstance(error_rate, bool)
142
+ or not isinstance(error_rate, (int, float))
143
+ or not math.isfinite(error_rate)
144
+ or not 0.0 <= error_rate <= 1.0
145
+ ):
146
+ raise _ConfigError(f"'error_rate' must be a finite number in [0, 1], got {error_rate!r}")
147
+ expected_error_rate = (requests - successful) / requests
148
+ if abs(float(error_rate) - expected_error_rate) > 1e-3:
149
+ raise _ConfigError(
150
+ f"'error_rate' ({error_rate!r}) is inconsistent with requests/successful_requests "
151
+ f"(expected {expected_error_rate!r})"
152
+ )
153
+ for metric in DIST_METRICS:
154
+ node = summary.get(metric)
155
+ if node is None:
156
+ continue
157
+ if not isinstance(node, dict):
158
+ raise _ConfigError(f"'{metric}' must be an object or null, got {type(node).__name__}")
159
+ count = node.get("count")
160
+ if isinstance(count, bool) or not isinstance(count, int) or count < 1:
161
+ # count=0 却带分位数是自相矛盾(真实 summarize 无样本时整节点为 null),
162
+ # 会经 _lookup 取到数值骗过相对门禁,故一律视为无证据。
163
+ raise _ConfigError(f"'{metric}.count' must be a positive integer, got {count!r}")
164
+ if metric in PER_REQUEST_DISTS and count > successful:
165
+ raise _ConfigError(
166
+ f"'{metric}.count' ({count}) exceeds successful_requests ({successful})"
167
+ )
168
+
169
+
170
+ def _check_hardware_evidence(report: dict, label: str) -> None:
171
+ """summary 里的硬件标量必须能追溯到 hardware 块,且块本身自洽。
172
+
173
+ 没有这一步,手改过的报告或「采样半途卡死」的半程数据都能当完整证据通过:
174
+ gate 对 error_rate ↔ requests/successful 有一致性校验,硬件指标必须有同等待遇。
175
+ """
176
+ summary = report["summary"]
177
+ block = report.get("hardware")
178
+ claimed = {k: v for k, v in ((k, summary.get(k)) for k in _HW_FIELDS) if v is not None}
179
+ if block is None:
180
+ if claimed:
181
+ raise _ConfigError(
182
+ f"{label} summary claims hardware metrics {sorted(claimed)} without a 'hardware' block"
183
+ )
184
+ return
185
+ if not isinstance(block, dict):
186
+ raise _ConfigError(f"{label} 'hardware' must be an object or null")
187
+ samples = block.get("samples")
188
+ if isinstance(samples, bool) or not isinstance(samples, int) or samples < 0:
189
+ raise _ConfigError(f"{label} hardware.samples must be a non-negative integer, got {samples!r}")
190
+ for metric, key in _HW_FIELDS.items():
191
+ expected = block.get(key)
192
+ observed = summary.get(metric)
193
+ if expected is None and observed is None:
194
+ continue
195
+ for value in (expected, observed):
196
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
197
+ raise _ConfigError(
198
+ f"{label} {metric} does not match hardware.{key}: {observed!r} vs {expected!r}"
199
+ )
200
+ if abs(float(observed) - float(expected)) > 1e-6:
201
+ raise _ConfigError(
202
+ f"{label} {metric} ({observed!r}) disagrees with hardware.{key} ({expected!r})"
203
+ )
204
+ # 块自身也要自洽:手拼的报告可以让 summary 与块「一致」却与实际采样形状矛盾
205
+ baseline = block.get("mem_used_baseline_mb")
206
+ peak = block.get("mem_used_peak_mb")
207
+ delta = block.get("mem_used_delta_mb")
208
+ if samples == 0 and any(value is not None for value in (baseline, peak, delta)):
209
+ raise _ConfigError(f"{label} hardware block has 0 samples but carries readings")
210
+ if samples < 2 and delta is not None:
211
+ raise _ConfigError(
212
+ f"{label} hardware.samples={samples} cannot have a mem_used_delta_mb "
213
+ "(needs a baseline and a second sample)"
214
+ )
215
+ if delta is not None:
216
+ for value in (baseline, peak):
217
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
218
+ raise _ConfigError(
219
+ f"{label} hardware.mem_used_delta_mb needs finite baseline/peak, "
220
+ f"got {baseline!r}/{peak!r}"
221
+ )
222
+ if abs(float(delta) - (float(peak) - float(baseline))) > 1e-3:
223
+ raise _ConfigError(
224
+ f"{label} hardware.mem_used_delta_mb ({delta!r}) != peak - baseline "
225
+ f"({float(peak) - float(baseline):.3f})"
226
+ )
227
+ # 物理合理性:显存占用不可能是负数,也不该超过设备的显存总量。
228
+ # 少了这两条,「-5 MB 峰值」或「9999 MB 峰值 / 6144 MB 设备」这种数字能被当成完整证据。
229
+ for key in ("mem_used_baseline_mb", "mem_used_peak_mb"):
230
+ value = block.get(key)
231
+ if value is None:
232
+ continue
233
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
234
+ raise _ConfigError(f"{label} hardware.{key} must be a finite number or null, got {value!r}")
235
+ if float(value) < 0:
236
+ raise _ConfigError(f"{label} hardware.{key} is negative ({value!r}): not a possible reading")
237
+ total = block.get("mem_total_mb")
238
+ if (peak is not None and isinstance(total, (int, float)) and not isinstance(total, bool)
239
+ and float(peak) > float(total) + 1e-6):
240
+ raise _ConfigError(
241
+ f"{label} hardware.mem_used_peak_mb ({peak!r}) exceeds device total ({total!r})"
242
+ )
243
+ # 指纹溯源:config.hardware 必须等于「用块自己重算出来的指纹」。
244
+ # 否则改一份报告的块(换卡/换容量)就能绕过可比性门禁 —— 只信自称等于「写对就拦、写错不拦」。
245
+ declared = (report.get("config") or {}).get("hardware")
246
+ recomputed = _block_fingerprint(block)
247
+ declared_missing = declared is None or (
248
+ isinstance(declared, str) and declared.strip() in MISSING_FINGERPRINTS
249
+ )
250
+ if recomputed is None:
251
+ if not declared_missing:
252
+ raise _ConfigError(
253
+ f"{label} config.hardware claims {declared!r} but the hardware block has no usable device"
254
+ )
255
+ elif declared_missing:
256
+ raise _ConfigError(
257
+ f"{label} config.hardware is missing but the hardware block identifies {recomputed!r}"
258
+ )
259
+ elif declared != recomputed:
260
+ raise _ConfigError(
261
+ f"{label} config.hardware ({declared!r}) disagrees with the hardware block "
262
+ f"({recomputed!r}): the fingerprint was not derived from this block"
263
+ )
264
+
265
+
266
+ def _hardware_fingerprint(report: dict, label: str) -> str:
267
+ """取可比较的硬件指纹;拿不到就 fail-closed(而不是当成同一块卡)。"""
268
+ config = report.get("config")
269
+ if not isinstance(config, dict):
270
+ raise _ConfigError(f"{label} report has no config block")
271
+ value = config.get("hardware")
272
+ if not isinstance(value, str) or value.strip() in MISSING_FINGERPRINTS:
273
+ raise _ConfigError(
274
+ f"{label} config.hardware is missing ({value!r}): "
275
+ "hardware metrics are only comparable on the same device"
276
+ )
277
+ return value
278
+
279
+
280
+ def _insufficient_samples(summary: dict, path: str) -> str | None:
281
+ """分布指标的 count 明显少于成功请求数 → 该指标只覆盖了子集,不能代表整体。"""
282
+ parts = path.split(".")
283
+ if len(parts) != 2 or parts[0] not in PER_REQUEST_DISTS:
284
+ return None
285
+ dist = summary.get(parts[0])
286
+ successful = summary.get("successful_requests")
287
+ if not isinstance(dist, dict):
288
+ return None
289
+ if isinstance(successful, bool) or not isinstance(successful, int):
290
+ # 靠 successful_requests 判「覆盖度」,它不可用就不能当作「样本充足」
291
+ return "'successful_requests' is not an integer, cannot verify sample coverage"
292
+ count = dist.get("count")
293
+ if isinstance(count, bool) or not isinstance(count, int):
294
+ return f"'{parts[0]}.count' is not an integer ({count!r})"
295
+ if count < successful:
296
+ return f"computed from {count} of {successful} successful requests"
297
+ return None
298
+
299
+
300
+ def _hardware_unusable(candidate: dict, path: str) -> str | None:
301
+ """硬件规则 + 截断采样 = 没证据。
302
+
303
+ `stop()` 卡在采样器里意味着窗口末尾的数没采到,峰值可能被低报;
304
+ 拿一个「可能漏掉峰值」的读数去对照上限,等于把噪声当合规。
305
+ 只有本侧(被审的那份报告)如此才判,基线侧截断只会让候选显得更差(保守)。
306
+ """
307
+ if path not in HARDWARE_METRICS:
308
+ return None
309
+ block = candidate.get("hardware")
310
+ if not isinstance(block, dict):
311
+ return None
312
+ if block.get("truncated") is True:
313
+ return "sampling was truncated at stop (peak may be under-reported)"
314
+ return None
315
+
316
+
317
+ def _status(ok: bool, comfortable: bool) -> str:
318
+ """ok=是否合规;comfortable=是否离阈值还有余量。
319
+
320
+ 合规但贴边 → WARN(放行但提示);不合规 → FAIL。
321
+ """
322
+ if not ok:
323
+ return "FAIL"
324
+ return "PASS" if comfortable else "WARN"
325
+
326
+
327
+ def evaluate(candidate: dict, rules: dict, baseline: dict | None = None) -> dict:
328
+ """执行门禁。返回 {exit_code, checks, reasons}。"""
329
+ checks: list[dict] = []
330
+ reasons: list[str] = []
331
+ try:
332
+ if not isinstance(candidate, dict) or not isinstance(candidate.get("summary"), dict):
333
+ raise _ConfigError("candidate report has no summary block")
334
+ policy = _validate_rules(rules, baseline, candidate)
335
+ except _ConfigError as exc:
336
+ return {"exit_code": 2, "checks": [], "reasons": [str(exc)]}
337
+
338
+ summary = candidate["summary"]
339
+ try:
340
+ _validate_summary(summary)
341
+ _check_hardware_evidence(candidate, "candidate")
342
+ except _ConfigError as exc:
343
+ return {"exit_code": 2, "checks": [], "reasons": [str(exc)]}
344
+
345
+ # baseline 也是判据的一部分:基线缺证据(薄样本、0 请求、非法分布 count)
346
+ # 时相对比较就没有意义,必须 fail-closed,而不是拿 1 个样本当基准。
347
+ if baseline is not None:
348
+ base = baseline.get("summary") if isinstance(baseline, dict) else None
349
+ if not isinstance(base, dict):
350
+ return {"exit_code": 2, "checks": [], "reasons": ["baseline report has no summary block"]}
351
+ try:
352
+ _validate_summary(base)
353
+ _check_hardware_evidence(baseline, "baseline")
354
+ except _ConfigError as exc:
355
+ return {"exit_code": 2, "checks": [], "reasons": [f"baseline: {exc}"]}
356
+ # 显存/利用率的比较只对同一块卡有意义(绝对上限同理:512 MB 的增长上限
357
+ # 在 6 GB 卡与 24 GB 卡上不是同一件事)。指纹缺失/非法/不同 → 不可比。
358
+ hw_rules = [n for n in list(policy["regressions"]) + list(policy["limits"])
359
+ if n in HARDWARE_METRICS]
360
+ if hw_rules:
361
+ try:
362
+ cand_hw = _hardware_fingerprint(candidate, "candidate")
363
+ base_hw = _hardware_fingerprint(baseline, "baseline")
364
+ except _ConfigError as exc:
365
+ return {"exit_code": 2, "checks": [], "reasons": [str(exc)]}
366
+ if cand_hw != base_hw:
367
+ return {"exit_code": 2, "checks": [], "reasons": [
368
+ f"hardware differs ({cand_hw!r} vs {base_hw!r}): not comparable"]}
369
+
370
+ failed = summary["requests"] - summary["successful_requests"]
371
+ error_rate = summary["error_rate"]
372
+ # 失败预算:默认 0(任何失败都拦)。若策略显式给了 error_rate 上限,
373
+ # 就按该预算判,避免「文档说可配置、代码却硬编码必挂」的自相矛盾。
374
+ failure_budget = float(policy["limits"].get("error_rate", 0.0))
375
+ if error_rate > failure_budget + 1e-12:
376
+ checks.append({
377
+ "metric": "error_rate", "kind": "implicit", "status": "FAIL",
378
+ "observed": error_rate, "threshold": failure_budget,
379
+ "reason": f"{failed} failed request(s), error_rate {error_rate:g} "
380
+ f"exceeds budget {failure_budget:g}",
381
+ })
382
+ reasons.append(f"{failed} failed request(s) / error_rate {error_rate:g} > {failure_budget:g}")
383
+
384
+ for name, limit in policy["limits"].items():
385
+ observed = _lookup(summary, name)
386
+ if observed is None:
387
+ checks.append({"metric": name, "kind": "limit", "status": "FAIL",
388
+ "observed": None, "threshold": limit,
389
+ "reason": "metric missing or not a finite number"})
390
+ reasons.append(f"{name}: no usable value")
391
+ continue
392
+ short = _insufficient_samples(summary, name)
393
+ if short:
394
+ checks.append({"metric": name, "kind": "limit", "status": "FAIL",
395
+ "observed": observed, "threshold": limit, "reason": short})
396
+ reasons.append(f"{name}: {short}")
397
+ continue
398
+ truncated = _hardware_unusable(candidate, name)
399
+ if truncated:
400
+ checks.append({"metric": name, "kind": "limit", "status": "FAIL",
401
+ "observed": observed, "threshold": limit, "reason": truncated})
402
+ reasons.append(f"{name}: {truncated}")
403
+ continue
404
+ higher = name in HIGHER_IS_BETTER
405
+ if higher:
406
+ ok = observed >= limit
407
+ comfortable = observed >= limit * 1.1 # 高于下限 10% 才算有余量
408
+ else:
409
+ ok = observed <= limit
410
+ comfortable = observed <= limit * 0.9 # 低于上限 10% 才算有余量
411
+ status = _status(ok, comfortable)
412
+ checks.append({"metric": name, "kind": "limit", "status": status,
413
+ "observed": observed, "threshold": limit, "reason": ""})
414
+ if status == "FAIL":
415
+ reasons.append(f"{name}={observed:g} violates limit {limit:g}")
416
+
417
+ base_summary = (baseline or {}).get("summary", {}) if baseline else {}
418
+ for name, tolerance_pct in policy["regressions"].items():
419
+ observed = _lookup(summary, name)
420
+ reference = _lookup(base_summary, name) if baseline else None
421
+ if observed is None or reference is None:
422
+ checks.append({"metric": name, "kind": "regression", "status": "FAIL",
423
+ "observed": observed, "baseline": reference,
424
+ "threshold": tolerance_pct,
425
+ "reason": "metric missing in candidate or baseline"})
426
+ reasons.append(f"{name}: missing value for relative comparison")
427
+ continue
428
+ short = _insufficient_samples(summary, name)
429
+ if short:
430
+ checks.append({"metric": name, "kind": "regression", "status": "FAIL",
431
+ "observed": observed, "baseline": reference,
432
+ "threshold": tolerance_pct, "reason": short})
433
+ reasons.append(f"{name}: {short}")
434
+ continue
435
+ truncated = _hardware_unusable(candidate, name)
436
+ if truncated:
437
+ checks.append({"metric": name, "kind": "regression", "status": "FAIL",
438
+ "observed": observed, "baseline": reference,
439
+ "threshold": tolerance_pct, "reason": truncated})
440
+ reasons.append(f"{name}: {truncated}")
441
+ continue
442
+ short_base = _insufficient_samples(base_summary, name)
443
+ if short_base:
444
+ reason = f"baseline {short_base}"
445
+ checks.append({"metric": name, "kind": "regression", "status": "FAIL",
446
+ "observed": observed, "baseline": reference,
447
+ "threshold": tolerance_pct, "reason": reason})
448
+ reasons.append(f"{name}: {reason}")
449
+ continue
450
+ higher = name in HIGHER_IS_BETTER
451
+ if higher:
452
+ allowed = reference * (1 - tolerance_pct / 100)
453
+ ok = observed >= allowed
454
+ comfortable = observed >= reference * (1 - tolerance_pct / 200) # 退化 < 半个容差
455
+ else:
456
+ if reference == 0:
457
+ ok = observed <= 0 # 基线为 0 时任何正值都是退化
458
+ comfortable = ok
459
+ else:
460
+ allowed = reference * (1 + tolerance_pct / 100)
461
+ ok = observed <= allowed
462
+ comfortable = observed <= reference * (1 + tolerance_pct / 200)
463
+ status = _status(ok, comfortable)
464
+ checks.append({"metric": name, "kind": "regression", "status": status,
465
+ "observed": observed, "baseline": reference,
466
+ "threshold": tolerance_pct, "reason": ""})
467
+ if status == "FAIL":
468
+ reasons.append(
469
+ f"{name}={observed:g} vs baseline {reference:g} "
470
+ f"exceeds {tolerance_pct:g}% tolerance"
471
+ )
472
+
473
+ if not checks:
474
+ return {"exit_code": 2, "checks": [], "reasons": ["no check was executed"]}
475
+ exit_code = 1 if any(c["status"] == "FAIL" for c in checks) else 0
476
+ return {"exit_code": exit_code, "checks": checks, "reasons": reasons}