localdeploy 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api_server.py +15 -0
- benchmark.py +15 -0
- chat_cli.py +15 -0
- compare_models.py +15 -0
- localdeploy/__init__.py +3 -0
- localdeploy/backends/__init__.py +0 -0
- localdeploy/backends/llamacpp.py +143 -0
- localdeploy/backends/ollama.py +374 -0
- localdeploy/backends/openai_compatible.py +179 -0
- localdeploy/benchmark.py +1996 -0
- localdeploy/chat_cli.py +249 -0
- localdeploy/cli.py +59 -0
- localdeploy/compare_models.py +481 -0
- localdeploy/control/__init__.py +75 -0
- localdeploy/control/_config.py +180 -0
- localdeploy/control/_ollama.py +253 -0
- localdeploy/control/bakeoff.py +216 -0
- localdeploy/control/bench.py +516 -0
- localdeploy/control/calibration.py +191 -0
- localdeploy/control/community.py +161 -0
- localdeploy/control/fit.py +651 -0
- localdeploy/control/hardware.py +602 -0
- localdeploy/control/history.py +125 -0
- localdeploy/control/manifest.py +452 -0
- localdeploy/control/models.py +833 -0
- localdeploy/control/monitor.py +378 -0
- localdeploy/control/profiles.py +161 -0
- localdeploy/control/recommend.py +322 -0
- localdeploy/control/registry.py +956 -0
- localdeploy/control/report.py +407 -0
- localdeploy/control/starter.py +520 -0
- localdeploy/control/updates.py +118 -0
- localdeploy/grader_sandbox.py +268 -0
- localdeploy/server.py +1873 -0
- localdeploy/utils.py +155 -0
- localdeploy/web/favicon.png +0 -0
- localdeploy/web/index.html +891 -0
- localdeploy/web/js/app.js +63 -0
- localdeploy/web/js/benchmark-views.js +73 -0
- localdeploy/web/js/benchmark.js +1691 -0
- localdeploy/web/js/chat.js +779 -0
- localdeploy/web/js/models.js +2758 -0
- localdeploy/web/js/shared.js +454 -0
- localdeploy/web/js/system.js +626 -0
- localdeploy/web/logo.svg +22 -0
- localdeploy/web/styles.css +2032 -0
- localdeploy-0.6.0.dist-info/METADATA +202 -0
- localdeploy-0.6.0.dist-info/RECORD +52 -0
- localdeploy-0.6.0.dist-info/WHEEL +5 -0
- localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
- localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
- localdeploy-0.6.0.dist-info/top_level.txt +5 -0
localdeploy/benchmark.py
ADDED
|
@@ -0,0 +1,1996 @@
|
|
|
1
|
+
"""Capability benchmark for local LLM profiles (v2 - harder tests).
|
|
2
|
+
|
|
3
|
+
Tests each enabled profile against five task categories:
|
|
4
|
+
planning software-project decomposition (not domestic tasks)
|
|
5
|
+
classification multi-class label with ambiguity
|
|
6
|
+
code non-trivial Python (DP, intervals, LRU cache, SQL)
|
|
7
|
+
math multi-step arithmetic, algebra, probability, matrices
|
|
8
|
+
structured JSON-schema-shaped output with field-level grading
|
|
9
|
+
|
|
10
|
+
Captures: success, elapsed seconds, response length, approx tokens/sec, accuracy
|
|
11
|
+
per test, and VRAM snapshots (via nvidia-smi) before/peak/after each profile.
|
|
12
|
+
|
|
13
|
+
Writes:
|
|
14
|
+
reports/benchmark_<UTC-timestamp>.json detailed results
|
|
15
|
+
reports/benchmark_<UTC-timestamp>.md human-readable summary
|
|
16
|
+
|
|
17
|
+
Run:
|
|
18
|
+
python benchmark.py # all enabled profiles
|
|
19
|
+
python benchmark.py --profile gemma3_4b_ollama_safe --profile qwen3vl_4b_ollama
|
|
20
|
+
python benchmark.py --skip-categories math
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
import ast
|
|
26
|
+
import json
|
|
27
|
+
import re
|
|
28
|
+
import statistics
|
|
29
|
+
import subprocess
|
|
30
|
+
import sys
|
|
31
|
+
import time
|
|
32
|
+
from dataclasses import asdict, dataclass, field
|
|
33
|
+
from datetime import datetime, timezone
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
|
|
36
|
+
|
|
37
|
+
import requests
|
|
38
|
+
from dotenv import load_dotenv
|
|
39
|
+
|
|
40
|
+
from api_server import load_config
|
|
41
|
+
from localdeploy.grader_sandbox import run_code_fraction
|
|
42
|
+
|
|
43
|
+
from localdeploy.utils import api_auth_headers, api_client_base_url, app_home
|
|
44
|
+
|
|
45
|
+
APP_DIR = Path(__file__).resolve().parents[1] # repo root in a checkout
|
|
46
|
+
APP_HOME = app_home()
|
|
47
|
+
load_dotenv(APP_HOME / ".env")
|
|
48
|
+
|
|
49
|
+
REPORTS_DIR = APP_HOME / "reports"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def api_base_url() -> str:
|
|
53
|
+
return api_client_base_url()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
# Test cases
|
|
58
|
+
# ---------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class TestCase:
|
|
63
|
+
name: str
|
|
64
|
+
category: str
|
|
65
|
+
prompt: str
|
|
66
|
+
grader: Callable[[str], float]
|
|
67
|
+
grader_explainer: str
|
|
68
|
+
max_output_tokens: int = 768
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ----- helpers ----------------------------------------------------------------
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _extract_json(text: str) -> Optional[Any]:
|
|
75
|
+
"""Best-effort JSON extraction: fenced, bracketed, or raw."""
|
|
76
|
+
candidates: List[str] = []
|
|
77
|
+
fence = re.search(r"```(?:json)?\s*(.*?)\s*```", text, flags=re.DOTALL | re.IGNORECASE)
|
|
78
|
+
if fence:
|
|
79
|
+
candidates.append(fence.group(1))
|
|
80
|
+
# Largest balanced object or array
|
|
81
|
+
obj = re.search(r"(\{.*\})", text, flags=re.DOTALL)
|
|
82
|
+
if obj:
|
|
83
|
+
candidates.append(obj.group(1))
|
|
84
|
+
arr = re.search(r"(\[.*\])", text, flags=re.DOTALL)
|
|
85
|
+
if arr:
|
|
86
|
+
candidates.append(arr.group(1))
|
|
87
|
+
candidates.append(text.strip())
|
|
88
|
+
for cand in candidates:
|
|
89
|
+
try:
|
|
90
|
+
return json.loads(cand)
|
|
91
|
+
except Exception:
|
|
92
|
+
continue
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _extract_python_code(text: str) -> str:
|
|
97
|
+
fence = re.search(r"```(?:python|py)?\s*(.*?)\s*```", text, flags=re.DOTALL | re.IGNORECASE)
|
|
98
|
+
return fence.group(1) if fence else text
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _first_word(text: str) -> str:
|
|
102
|
+
line = text.strip().splitlines()[0] if text.strip() else ""
|
|
103
|
+
cleaned = re.sub(r"[^a-z0-9_+-]", " ", line.lower()).split()
|
|
104
|
+
return cleaned[0] if cleaned else ""
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# ----- planning graders -------------------------------------------------------
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _grade_chat_backend_plan(text: str) -> float:
|
|
111
|
+
"""Looks for 6 numbered milestones, each with name + deliverables + dependencies + effort."""
|
|
112
|
+
score = 0.0
|
|
113
|
+
# 6 top-level numbered items
|
|
114
|
+
numbers = re.findall(r"^\s*(\d+)\.\s", text, flags=re.MULTILINE)
|
|
115
|
+
if len(numbers) >= 6:
|
|
116
|
+
score += 0.4
|
|
117
|
+
elif len(numbers) >= 4:
|
|
118
|
+
score += 0.2
|
|
119
|
+
# Field markers
|
|
120
|
+
lowered = text.lower()
|
|
121
|
+
field_hits = sum(1 for kw in ["deliverable", "depend", "effort", "milestone"] if kw in lowered)
|
|
122
|
+
score += 0.15 * (field_hits / 4)
|
|
123
|
+
# T-shirt sizing presence
|
|
124
|
+
if re.search(r"\b(xs|s|m|l|xl)\b", lowered):
|
|
125
|
+
score += 0.15
|
|
126
|
+
# Concrete tech topics (auth, websocket, db, persistence, scaling, deploy)
|
|
127
|
+
topic_hits = sum(1 for kw in ["websocket", "database", "auth", "scal", "deploy", "persist", "message", "presence"] if kw in lowered)
|
|
128
|
+
score += 0.3 * min(1.0, topic_hits / 4)
|
|
129
|
+
return min(1.0, score)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _grade_refactor_json(text: str) -> float:
|
|
133
|
+
data = _extract_json(text)
|
|
134
|
+
if not isinstance(data, dict):
|
|
135
|
+
return 0.0
|
|
136
|
+
score = 0.2 # parsed
|
|
137
|
+
phases = data.get("phases")
|
|
138
|
+
if isinstance(phases, list) and len(phases) == 4:
|
|
139
|
+
score += 0.25
|
|
140
|
+
# Each phase has name/description/risks/success_criteria
|
|
141
|
+
per_phase = []
|
|
142
|
+
for p in phases:
|
|
143
|
+
if not isinstance(p, dict):
|
|
144
|
+
per_phase.append(0)
|
|
145
|
+
continue
|
|
146
|
+
ok = sum(
|
|
147
|
+
1
|
|
148
|
+
for key in ("name", "description", "risks", "success_criteria")
|
|
149
|
+
if key in p
|
|
150
|
+
)
|
|
151
|
+
# risks and success_criteria are lists >= 2
|
|
152
|
+
list_ok = sum(
|
|
153
|
+
1
|
|
154
|
+
for key in ("risks", "success_criteria")
|
|
155
|
+
if isinstance(p.get(key), list) and len(p[key]) >= 2
|
|
156
|
+
)
|
|
157
|
+
per_phase.append((ok / 4) * 0.7 + (list_ok / 2) * 0.3)
|
|
158
|
+
score += 0.3 * (sum(per_phase) / 4)
|
|
159
|
+
if isinstance(data.get("rollback_plan"), str) and len(data["rollback_plan"]) > 30:
|
|
160
|
+
score += 0.1
|
|
161
|
+
if isinstance(data.get("estimated_duration_days"), int):
|
|
162
|
+
score += 0.15
|
|
163
|
+
return min(1.0, score)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _grade_p99_plan(text: str) -> float:
|
|
167
|
+
score = 0.0
|
|
168
|
+
numbers = re.findall(r"^\s*(\d+)\.\s", text, flags=re.MULTILINE)
|
|
169
|
+
if len(numbers) >= 5:
|
|
170
|
+
score += 0.4
|
|
171
|
+
elif len(numbers) >= 3:
|
|
172
|
+
score += 0.2
|
|
173
|
+
lowered = text.lower()
|
|
174
|
+
# Field markers required per step
|
|
175
|
+
field_hits = sum(1 for kw in ["expected", "time", "access", "signal", "check"] if kw in lowered)
|
|
176
|
+
score += 0.2 * (field_hits / 5)
|
|
177
|
+
# Cheap-to-invasive ordering signals (check dashboards / logs / traces before code)
|
|
178
|
+
cheap = any(kw in lowered for kw in ["dashboard", "metric", "log", "grafana", "alert"])
|
|
179
|
+
invasive = any(kw in lowered for kw in ["redeploy", "rollback", "ssh", "production debug", "code change", "git bisect"])
|
|
180
|
+
if cheap and invasive:
|
|
181
|
+
cheap_idx = min((lowered.find(kw) for kw in ["dashboard", "metric", "log"] if kw in lowered), default=10_000)
|
|
182
|
+
invasive_idx = min((lowered.find(kw) for kw in ["redeploy", "rollback", "ssh", "git bisect"] if kw in lowered), default=10_000)
|
|
183
|
+
if cheap_idx < invasive_idx:
|
|
184
|
+
score += 0.2
|
|
185
|
+
# Plausible diagnostic terms
|
|
186
|
+
diag_hits = sum(1 for kw in ["latency", "p99", "trace", "noisy neighbor", "gc", "memory", "cpu", "downstream", "throughput"] if kw in lowered)
|
|
187
|
+
score += 0.2 * min(1.0, diag_hits / 5)
|
|
188
|
+
return min(1.0, score)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
# ----- classification graders -------------------------------------------------
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _grade_first_word(answer: str) -> Callable[[str], float]:
|
|
195
|
+
def grader(text: str) -> float:
|
|
196
|
+
word = _first_word(text)
|
|
197
|
+
if word == answer:
|
|
198
|
+
return 1.0
|
|
199
|
+
if answer in text.lower():
|
|
200
|
+
return 0.5
|
|
201
|
+
return 0.0
|
|
202
|
+
|
|
203
|
+
return grader
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# ----- code graders -----------------------------------------------------------
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _grade_levenshtein(text: str) -> float:
|
|
210
|
+
code = _extract_python_code(text)
|
|
211
|
+
try:
|
|
212
|
+
tree = ast.parse(code)
|
|
213
|
+
except SyntaxError:
|
|
214
|
+
return 0.0
|
|
215
|
+
fns = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)]
|
|
216
|
+
if not any(f.name == "levenshtein" for f in fns):
|
|
217
|
+
return 0.1
|
|
218
|
+
score = 0.4
|
|
219
|
+
# Run the candidate code in an isolated subprocess (see grader_sandbox).
|
|
220
|
+
score += 0.6 * run_code_fraction(code, "levenshtein")
|
|
221
|
+
return min(1.0, score)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _grade_merge_intervals(text: str) -> float:
|
|
225
|
+
code = _extract_python_code(text)
|
|
226
|
+
try:
|
|
227
|
+
tree = ast.parse(code)
|
|
228
|
+
except SyntaxError:
|
|
229
|
+
return 0.0
|
|
230
|
+
if not any(isinstance(n, ast.FunctionDef) and n.name == "merge_intervals" for n in ast.walk(tree)):
|
|
231
|
+
return 0.1
|
|
232
|
+
score = 0.4
|
|
233
|
+
score += 0.6 * run_code_fraction(code, "merge_intervals")
|
|
234
|
+
return min(1.0, score)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _grade_lru_cache(text: str) -> float:
|
|
238
|
+
code = _extract_python_code(text)
|
|
239
|
+
try:
|
|
240
|
+
tree = ast.parse(code)
|
|
241
|
+
except SyntaxError:
|
|
242
|
+
return 0.0
|
|
243
|
+
classes = [n for n in ast.walk(tree) if isinstance(n, ast.ClassDef) and n.name == "LRUCache"]
|
|
244
|
+
if not classes:
|
|
245
|
+
return 0.1
|
|
246
|
+
score = 0.3
|
|
247
|
+
methods = {m.name for m in ast.walk(classes[0]) if isinstance(m, ast.FunctionDef)}
|
|
248
|
+
if {"__init__", "get", "put"}.issubset(methods):
|
|
249
|
+
score += 0.2
|
|
250
|
+
score += 0.5 * run_code_fraction(code, "lru_cache")
|
|
251
|
+
return min(1.0, score)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _grade_sql_second_highest(text: str) -> float:
|
|
255
|
+
cleaned = text.strip().lower()
|
|
256
|
+
score = 0.0
|
|
257
|
+
if "select" in cleaned:
|
|
258
|
+
score += 0.2
|
|
259
|
+
# Acceptable patterns
|
|
260
|
+
if "limit 1" in cleaned and "offset 1" in cleaned:
|
|
261
|
+
score += 0.5
|
|
262
|
+
elif re.search(r"dense_rank|rank\(\)\s*over", cleaned):
|
|
263
|
+
score += 0.5
|
|
264
|
+
elif "distinct" in cleaned and re.search(r"order\s+by\s+salary\s+desc", cleaned):
|
|
265
|
+
score += 0.4
|
|
266
|
+
elif "max" in cleaned and "where" in cleaned and "<" in cleaned:
|
|
267
|
+
score += 0.4
|
|
268
|
+
if "null" in cleaned or "ifnull" in cleaned or "coalesce" in cleaned:
|
|
269
|
+
score += 0.2
|
|
270
|
+
if "employee" in cleaned:
|
|
271
|
+
score += 0.1
|
|
272
|
+
return min(1.0, score)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
# ----- math graders -----------------------------------------------------------
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _grade_number(expected: float, tolerance: float = 0.5) -> Callable[[str], float]:
|
|
279
|
+
def grader(text: str) -> float:
|
|
280
|
+
for cand in re.findall(r"-?\d+(?:\.\d+)?", text):
|
|
281
|
+
try:
|
|
282
|
+
if abs(float(cand) - expected) <= tolerance:
|
|
283
|
+
return 1.0
|
|
284
|
+
except ValueError:
|
|
285
|
+
continue
|
|
286
|
+
return 0.0
|
|
287
|
+
|
|
288
|
+
return grader
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _grade_time_953(text: str) -> float:
|
|
292
|
+
# Accept 9:52, 9:53, 9:54 AM. The correct answer is unambiguously AM (both
|
|
293
|
+
# trains depart in the morning), so a matching "9:53 PM" must be rejected -
|
|
294
|
+
# an explicit PM marker on the match disqualifies it; no marker or an
|
|
295
|
+
# explicit AM marker is accepted.
|
|
296
|
+
for hr, mn, meridiem in re.findall(r"(\d{1,2}):(\d{2})\s*([AaPp]\.?[Mm]\.?)?", text):
|
|
297
|
+
if meridiem and meridiem.strip().lower().startswith("p"):
|
|
298
|
+
continue
|
|
299
|
+
try:
|
|
300
|
+
h = int(hr)
|
|
301
|
+
m = int(mn)
|
|
302
|
+
except ValueError:
|
|
303
|
+
continue
|
|
304
|
+
if h == 9 and abs(m - 53) <= 1:
|
|
305
|
+
return 1.0
|
|
306
|
+
return 0.0
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _grade_fraction(target_num: int, target_den: int) -> Callable[[str], float]:
|
|
310
|
+
def grader(text: str) -> float:
|
|
311
|
+
for num, den in re.findall(r"(\d+)\s*/\s*(\d+)", text):
|
|
312
|
+
try:
|
|
313
|
+
n, d = int(num), int(den)
|
|
314
|
+
if d != 0 and n * target_den == target_num * d:
|
|
315
|
+
return 1.0
|
|
316
|
+
except ValueError:
|
|
317
|
+
continue
|
|
318
|
+
return 0.0
|
|
319
|
+
|
|
320
|
+
return grader
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
# ----- structured output graders ----------------------------------------------
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _grade_request_parse(text: str) -> float:
|
|
327
|
+
data = _extract_json(text)
|
|
328
|
+
if not isinstance(data, dict):
|
|
329
|
+
return 0.0
|
|
330
|
+
score = 0.2
|
|
331
|
+
if data.get("action") == "read":
|
|
332
|
+
score += 0.2
|
|
333
|
+
entity = str(data.get("entity") or "").lower()
|
|
334
|
+
if "invoice" in entity:
|
|
335
|
+
score += 0.15
|
|
336
|
+
filters = data.get("filters")
|
|
337
|
+
if isinstance(filters, dict):
|
|
338
|
+
f_lower = json.dumps(filters).lower()
|
|
339
|
+
if "overdue" in f_lower:
|
|
340
|
+
score += 0.15
|
|
341
|
+
if "acme-42" in f_lower or "acme_42" in f_lower or "acme" in f_lower:
|
|
342
|
+
score += 0.15
|
|
343
|
+
if str(data.get("output_format") or "").lower() == "csv":
|
|
344
|
+
score += 0.15
|
|
345
|
+
return min(1.0, score)
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _grade_flight_extract(text: str) -> float:
|
|
349
|
+
data = _extract_json(text)
|
|
350
|
+
if not isinstance(data, dict):
|
|
351
|
+
return 0.0
|
|
352
|
+
score = 0.2
|
|
353
|
+
if "united" in str(data.get("airline", "")).lower():
|
|
354
|
+
score += 0.15
|
|
355
|
+
flight = data.get("flight", {})
|
|
356
|
+
if isinstance(flight, dict):
|
|
357
|
+
if str(flight.get("origin", "")).upper() == "ORD":
|
|
358
|
+
score += 0.15
|
|
359
|
+
if str(flight.get("destination", "")).upper() == "LAX":
|
|
360
|
+
score += 0.15
|
|
361
|
+
if re.match(r"^\d{1,2}:\d{2}", str(flight.get("departure_time", ""))):
|
|
362
|
+
score += 0.1
|
|
363
|
+
pax = data.get("passenger", {})
|
|
364
|
+
if isinstance(pax, dict):
|
|
365
|
+
if "smith" in str(pax.get("last_name", "")).lower():
|
|
366
|
+
score += 0.1
|
|
367
|
+
if "4429" in str(pax.get("ff_number", "")):
|
|
368
|
+
score += 0.15
|
|
369
|
+
return min(1.0, score)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _grade_function_schema(text: str) -> float:
|
|
373
|
+
data = _extract_json(text)
|
|
374
|
+
if not isinstance(data, dict):
|
|
375
|
+
return 0.0
|
|
376
|
+
score = 0.15
|
|
377
|
+
params = data.get("parameters") or data
|
|
378
|
+
if not isinstance(params, dict):
|
|
379
|
+
return score
|
|
380
|
+
if params.get("type") == "object":
|
|
381
|
+
score += 0.15
|
|
382
|
+
props = params.get("properties") or {}
|
|
383
|
+
if not isinstance(props, dict):
|
|
384
|
+
return score
|
|
385
|
+
needed = {"title", "participants", "duration_minutes", "location"}
|
|
386
|
+
score += 0.3 * (len(needed & set(props.keys())) / len(needed))
|
|
387
|
+
if isinstance(props.get("participants"), dict) and props["participants"].get("type") == "array":
|
|
388
|
+
score += 0.1
|
|
389
|
+
if isinstance(props.get("duration_minutes"), dict) and props["duration_minutes"].get("type") == "integer":
|
|
390
|
+
score += 0.1
|
|
391
|
+
required = params.get("required") or []
|
|
392
|
+
if isinstance(required, list) and "title" in required and "participants" in required:
|
|
393
|
+
score += 0.2
|
|
394
|
+
return min(1.0, score)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
# ----- HARD structured graders (modeled on YBM pydantic schemas) --------------
|
|
398
|
+
# These exercise nested enums, conditional fields, capability-subset constraints,
|
|
399
|
+
# multi-entity extraction, and reasoning-then-output - patterns the easy tests miss.
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
_INTENT_ROUTES = {
|
|
403
|
+
"conversation", "status", "desktop.observe", "computer.use", "browser.open",
|
|
404
|
+
"browser.control", "filesystem.manage", "document.manage", "artifact.deliver",
|
|
405
|
+
"code.interpreter", "coding.agent", "schedule.manage", "configuration", "unknown",
|
|
406
|
+
}
|
|
407
|
+
_TASK_TYPES = {"development", "configuration", "admin_control", "desktop_observation", "question", "status_request", "other"}
|
|
408
|
+
_RISK_LEVELS = {"low", "medium", "high", "critical"}
|
|
409
|
+
_CAPABILITIES = {
|
|
410
|
+
"telegram.send", "telegram.receive", "llm.generate", "stt.transcribe", "tts.synthesize",
|
|
411
|
+
"vscode.read_state", "vscode.write_files", "terminal.run", "filesystem.read", "filesystem.write",
|
|
412
|
+
"desktop.screenshot", "desktop.control", "browser.open", "browser.control", "schedule.manage",
|
|
413
|
+
"github.read", "github.push", "dependencies.install",
|
|
414
|
+
}
|
|
415
|
+
_DELIVERY_KINDS = {"none", "latest", "file", "screenshot"}
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _grade_intent_classify(text: str) -> float:
|
|
419
|
+
"""User said: 'Open chrome go to news.ycombinator.com take a screenshot of the front page and send it to me.'
|
|
420
|
+
|
|
421
|
+
Expected MessageClassification + nested OrchestrationIntent.
|
|
422
|
+
"""
|
|
423
|
+
data = _extract_json(text)
|
|
424
|
+
if not isinstance(data, dict):
|
|
425
|
+
return 0.0
|
|
426
|
+
score = 0.0
|
|
427
|
+
# is_task=true
|
|
428
|
+
if data.get("is_task") is True:
|
|
429
|
+
score += 0.1
|
|
430
|
+
# task_type in valid enum, prefer "other" (browser→other per YBM rule)
|
|
431
|
+
tt = str(data.get("task_type", "")).lower()
|
|
432
|
+
if tt in _TASK_TYPES:
|
|
433
|
+
score += 0.05
|
|
434
|
+
if tt == "other":
|
|
435
|
+
score += 0.05
|
|
436
|
+
# confidence is number in [0,1]
|
|
437
|
+
conf = data.get("confidence")
|
|
438
|
+
if isinstance(conf, (int, float)) and 0.0 <= float(conf) <= 1.0:
|
|
439
|
+
score += 0.05
|
|
440
|
+
if float(conf) >= 0.7:
|
|
441
|
+
score += 0.05
|
|
442
|
+
# normalized_objective mentions key entities
|
|
443
|
+
obj = str(data.get("normalized_objective") or "").lower()
|
|
444
|
+
if "screenshot" in obj and ("ycombinator" in obj or "hacker news" in obj or "front page" in obj):
|
|
445
|
+
score += 0.1
|
|
446
|
+
elif "screenshot" in obj or "ycombinator" in obj:
|
|
447
|
+
score += 0.05
|
|
448
|
+
# reason is a non-empty string
|
|
449
|
+
reason = data.get("reason")
|
|
450
|
+
if isinstance(reason, str) and len(reason.strip()) >= 5:
|
|
451
|
+
score += 0.05
|
|
452
|
+
# nested intent
|
|
453
|
+
intent = data.get("intent")
|
|
454
|
+
if isinstance(intent, dict):
|
|
455
|
+
score += 0.05
|
|
456
|
+
route = str(intent.get("route", "")).lower()
|
|
457
|
+
if route in _INTENT_ROUTES:
|
|
458
|
+
score += 0.05
|
|
459
|
+
if route in {"browser.open", "browser.control"}:
|
|
460
|
+
score += 0.1
|
|
461
|
+
url = str(intent.get("url") or "")
|
|
462
|
+
if "ycombinator" in url.lower():
|
|
463
|
+
score += 0.1
|
|
464
|
+
delivery = str(intent.get("delivery", "")).lower()
|
|
465
|
+
if delivery in _DELIVERY_KINDS:
|
|
466
|
+
score += 0.05
|
|
467
|
+
if delivery in {"screenshot", "file", "latest"}:
|
|
468
|
+
score += 0.05
|
|
469
|
+
if isinstance(intent.get("needs_plan_first"), bool):
|
|
470
|
+
score += 0.05
|
|
471
|
+
return min(1.0, score)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _grade_plan_orchestration(text: str) -> float:
|
|
475
|
+
"""Task: 'Take a screenshot of the current desktop and send it to me.'
|
|
476
|
+
|
|
477
|
+
Expected PlanModel-like JSON with desktop.screenshot + telegram.send capabilities
|
|
478
|
+
and the capability-subset constraint between top-level and per-step.
|
|
479
|
+
"""
|
|
480
|
+
data = _extract_json(text)
|
|
481
|
+
if not isinstance(data, dict):
|
|
482
|
+
return 0.0
|
|
483
|
+
score = 0.05 # parsed
|
|
484
|
+
# objective mentions screenshot
|
|
485
|
+
obj = str(data.get("objective") or "").lower()
|
|
486
|
+
if "screenshot" in obj or "desktop" in obj:
|
|
487
|
+
score += 0.1
|
|
488
|
+
# required_capabilities is array of valid enum values
|
|
489
|
+
top_caps = data.get("required_capabilities")
|
|
490
|
+
if isinstance(top_caps, list):
|
|
491
|
+
score += 0.05
|
|
492
|
+
cap_set = {str(c).lower() for c in top_caps if isinstance(c, str)}
|
|
493
|
+
valid_caps = cap_set & _CAPABILITIES
|
|
494
|
+
if "desktop.screenshot" in valid_caps:
|
|
495
|
+
score += 0.1
|
|
496
|
+
if "telegram.send" in valid_caps or "artifact.deliver" in {str(c).lower() for c in top_caps}:
|
|
497
|
+
score += 0.1
|
|
498
|
+
# steps is array of objects with required keys
|
|
499
|
+
steps = data.get("steps")
|
|
500
|
+
if isinstance(steps, list) and len(steps) >= 2:
|
|
501
|
+
score += 0.1
|
|
502
|
+
# Each step has title, description, required_capabilities, risk_level
|
|
503
|
+
per_step_ok = 0
|
|
504
|
+
capability_subset_ok = True
|
|
505
|
+
top_cap_set = {str(c).lower() for c in (top_caps or []) if isinstance(c, str)}
|
|
506
|
+
for step in steps:
|
|
507
|
+
if not isinstance(step, dict):
|
|
508
|
+
capability_subset_ok = False
|
|
509
|
+
continue
|
|
510
|
+
field_hits = sum(1 for k in ("title", "description", "required_capabilities", "risk_level") if k in step)
|
|
511
|
+
if field_hits >= 3:
|
|
512
|
+
per_step_ok += 1
|
|
513
|
+
# capability subset check
|
|
514
|
+
step_caps = step.get("required_capabilities") or []
|
|
515
|
+
if isinstance(step_caps, list) and top_cap_set:
|
|
516
|
+
if not {str(c).lower() for c in step_caps if isinstance(c, str)}.issubset(top_cap_set):
|
|
517
|
+
capability_subset_ok = False
|
|
518
|
+
# risk_level is valid enum
|
|
519
|
+
if str(step.get("risk_level", "")).lower() in _RISK_LEVELS:
|
|
520
|
+
pass # rolled into field_hits
|
|
521
|
+
score += 0.1 * (per_step_ok / max(1, len(steps)))
|
|
522
|
+
if capability_subset_ok and top_cap_set:
|
|
523
|
+
score += 0.15
|
|
524
|
+
# success_criteria is array
|
|
525
|
+
sc = data.get("success_criteria")
|
|
526
|
+
if isinstance(sc, list) and len(sc) >= 1:
|
|
527
|
+
score += 0.1
|
|
528
|
+
# approval_gates is array (may be empty)
|
|
529
|
+
ag = data.get("approval_gates")
|
|
530
|
+
if isinstance(ag, list):
|
|
531
|
+
score += 0.05
|
|
532
|
+
return min(1.0, score)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _grade_approval_decision(text: str) -> float:
|
|
536
|
+
"""Action: delete user's document. Expected: cautious decision with conditional field.
|
|
537
|
+
|
|
538
|
+
Conditional constraint: confirmation_prompt is required iff needs_user_confirmation=True.
|
|
539
|
+
Cautious decision: 'reject' or 'needs_clarification', NOT auto-approve.
|
|
540
|
+
"""
|
|
541
|
+
data = _extract_json(text)
|
|
542
|
+
if not isinstance(data, dict):
|
|
543
|
+
return 0.0
|
|
544
|
+
score = 0.05 # parsed
|
|
545
|
+
decision = str(data.get("decision", "")).lower()
|
|
546
|
+
if decision in {"reject", "needs_clarification"}:
|
|
547
|
+
score += 0.2
|
|
548
|
+
elif decision == "approve":
|
|
549
|
+
score += 0.0 # wrong call for destructive action
|
|
550
|
+
risk = str(data.get("risk_level", "")).lower()
|
|
551
|
+
if risk in {"high", "critical"}:
|
|
552
|
+
score += 0.15
|
|
553
|
+
elif risk == "medium":
|
|
554
|
+
score += 0.05
|
|
555
|
+
reasons = data.get("reasons")
|
|
556
|
+
if isinstance(reasons, list) and len(reasons) >= 2:
|
|
557
|
+
score += 0.1
|
|
558
|
+
needs_conf = data.get("needs_user_confirmation")
|
|
559
|
+
conf_prompt = data.get("confirmation_prompt")
|
|
560
|
+
# Conditional: confirmation_prompt iff needs_user_confirmation
|
|
561
|
+
if isinstance(needs_conf, bool):
|
|
562
|
+
score += 0.05
|
|
563
|
+
if needs_conf is True and isinstance(conf_prompt, str) and len(conf_prompt.strip()) >= 5:
|
|
564
|
+
score += 0.15
|
|
565
|
+
elif needs_conf is False and (conf_prompt is None or conf_prompt == ""):
|
|
566
|
+
score += 0.1
|
|
567
|
+
elif needs_conf is True and not isinstance(conf_prompt, str):
|
|
568
|
+
pass # constraint violation
|
|
569
|
+
caps = data.get("required_capabilities")
|
|
570
|
+
if isinstance(caps, list):
|
|
571
|
+
cap_set = {str(c).lower() for c in caps if isinstance(c, str)}
|
|
572
|
+
if "filesystem.delete" in cap_set or "filesystem.write" in cap_set:
|
|
573
|
+
score += 0.1
|
|
574
|
+
alt = data.get("alternative_action")
|
|
575
|
+
if isinstance(alt, dict) and "action" in alt and "rationale" in alt:
|
|
576
|
+
score += 0.1
|
|
577
|
+
elif alt is None:
|
|
578
|
+
score += 0.05
|
|
579
|
+
return min(1.0, score)
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _grade_multi_task_extract(text: str) -> float:
|
|
583
|
+
"""User asked for 2 tasks: watch Downloads for PDFs, schedule daily 9am report.
|
|
584
|
+
|
|
585
|
+
Plus a negative constraint about Codex.
|
|
586
|
+
"""
|
|
587
|
+
data = _extract_json(text)
|
|
588
|
+
if not isinstance(data, dict):
|
|
589
|
+
return 0.0
|
|
590
|
+
score = 0.05 # parsed
|
|
591
|
+
tasks = data.get("tasks")
|
|
592
|
+
if not isinstance(tasks, list):
|
|
593
|
+
return score
|
|
594
|
+
if len(tasks) == 2:
|
|
595
|
+
score += 0.2
|
|
596
|
+
elif len(tasks) == 1 or len(tasks) == 3:
|
|
597
|
+
score += 0.1
|
|
598
|
+
# Find task with filesystem/document route and watch trigger
|
|
599
|
+
found_watch = False
|
|
600
|
+
found_schedule = False
|
|
601
|
+
codex_constraint = False
|
|
602
|
+
for t in tasks:
|
|
603
|
+
if not isinstance(t, dict):
|
|
604
|
+
continue
|
|
605
|
+
route = str(t.get("route", "")).lower()
|
|
606
|
+
trigger = t.get("trigger")
|
|
607
|
+
trigger_kind = ""
|
|
608
|
+
if isinstance(trigger, dict):
|
|
609
|
+
trigger_kind = str(trigger.get("kind", "")).lower()
|
|
610
|
+
if route in {"filesystem.manage", "document.manage"} and trigger_kind == "watch":
|
|
611
|
+
found_watch = True
|
|
612
|
+
if route == "schedule.manage" and trigger_kind == "schedule":
|
|
613
|
+
found_schedule = True
|
|
614
|
+
constraints = t.get("constraints")
|
|
615
|
+
if isinstance(constraints, list):
|
|
616
|
+
joined = " ".join(str(c).lower() for c in constraints)
|
|
617
|
+
if "codex" in joined or "copilot" in joined or "no advanced" in joined or "advanced" in joined:
|
|
618
|
+
codex_constraint = True
|
|
619
|
+
if found_watch:
|
|
620
|
+
score += 0.2
|
|
621
|
+
if found_schedule:
|
|
622
|
+
score += 0.2
|
|
623
|
+
# Schedule task should mention 9am or daily
|
|
624
|
+
for t in tasks:
|
|
625
|
+
if not isinstance(t, dict):
|
|
626
|
+
continue
|
|
627
|
+
if str(t.get("route", "")).lower() == "schedule.manage":
|
|
628
|
+
trigger = t.get("trigger") or {}
|
|
629
|
+
details = str(trigger.get("details", "") if isinstance(trigger, dict) else "").lower()
|
|
630
|
+
if "9am" in details or "9:00" in details or "0900" in details or "daily" in details:
|
|
631
|
+
score += 0.1
|
|
632
|
+
break
|
|
633
|
+
if codex_constraint:
|
|
634
|
+
score += 0.15
|
|
635
|
+
# ignored_instructions is array (can be empty)
|
|
636
|
+
if isinstance(data.get("ignored_instructions"), list):
|
|
637
|
+
score += 0.05
|
|
638
|
+
# Sequential ids
|
|
639
|
+
ids = [t.get("id") for t in tasks if isinstance(t, dict)]
|
|
640
|
+
if ids and all(isinstance(i, int) for i in ids) and ids == list(range(1, len(ids) + 1)):
|
|
641
|
+
score += 0.05
|
|
642
|
+
return min(1.0, score)
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def _grade_review_extract(text: str) -> float:
|
|
646
|
+
data = _extract_json(text)
|
|
647
|
+
if not isinstance(data, dict):
|
|
648
|
+
return 0.0
|
|
649
|
+
score = 0.2
|
|
650
|
+
if "eternal drift" in str(data.get("title", "")).lower():
|
|
651
|
+
score += 0.2
|
|
652
|
+
try:
|
|
653
|
+
if abs(float(data.get("score", -1)) - 7) <= 0.01:
|
|
654
|
+
score += 0.2
|
|
655
|
+
except (TypeError, ValueError):
|
|
656
|
+
pass
|
|
657
|
+
pros = data.get("pros")
|
|
658
|
+
cons = data.get("cons")
|
|
659
|
+
if isinstance(pros, list) and len(pros) >= 1:
|
|
660
|
+
score += 0.2
|
|
661
|
+
if isinstance(cons, list) and len(cons) >= 1:
|
|
662
|
+
score += 0.2
|
|
663
|
+
return min(1.0, score)
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
# ---------------------------------------------------------------------------
|
|
667
|
+
# Test catalog
|
|
668
|
+
# ---------------------------------------------------------------------------
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
TEST_CASES: List[TestCase] = [
|
|
672
|
+
# --- Planning (software/project oriented) ---
|
|
673
|
+
TestCase(
|
|
674
|
+
name="plan_chat_backend",
|
|
675
|
+
category="planning",
|
|
676
|
+
prompt=(
|
|
677
|
+
"Plan the implementation of a real-time chat application backend (1-on-1 + group rooms, presence, "
|
|
678
|
+
"message history, scalable to 50k concurrent users). Output 6 ordered milestones as a numbered "
|
|
679
|
+
"Markdown list. For each milestone include: name, 2-3 concrete deliverables, dependencies on earlier "
|
|
680
|
+
"milestones, and effort estimate as XS/S/M/L/XL. Use indented sub-items for the deliverables."
|
|
681
|
+
),
|
|
682
|
+
grader=_grade_chat_backend_plan,
|
|
683
|
+
grader_explainer="6 numbered milestones with deliverables/dependencies/effort fields + concrete tech topics",
|
|
684
|
+
max_output_tokens=1024,
|
|
685
|
+
),
|
|
686
|
+
TestCase(
|
|
687
|
+
name="plan_refactor_json",
|
|
688
|
+
category="planning",
|
|
689
|
+
prompt=(
|
|
690
|
+
"I need to refactor a 5000-line monolithic Python file into modules. Output ONLY a JSON object "
|
|
691
|
+
"(no markdown fence) with keys: 'phases' (array of exactly 4 objects, each having 'name', "
|
|
692
|
+
"'description', 'risks' as array of at least 2 strings, 'success_criteria' as array of at least 2 "
|
|
693
|
+
"strings), 'rollback_plan' (string of 2-3 sentences), 'estimated_duration_days' (integer)."
|
|
694
|
+
),
|
|
695
|
+
grader=_grade_refactor_json,
|
|
696
|
+
grader_explainer="Valid JSON with 4 phases (each w/ name+description+risks[≥2]+criteria[≥2]) + rollback + days",
|
|
697
|
+
max_output_tokens=1024,
|
|
698
|
+
),
|
|
699
|
+
TestCase(
|
|
700
|
+
name="plan_p99_investigation",
|
|
701
|
+
category="planning",
|
|
702
|
+
prompt=(
|
|
703
|
+
"Production microservice p99 latency just doubled from 200ms to 400ms with no recent deploys. "
|
|
704
|
+
"Outline a 5-step investigation plan, ordered from cheapest/fastest to most invasive. "
|
|
705
|
+
"For each step include: what to check, expected signal of a hit, time estimate, required access. "
|
|
706
|
+
"Numbered Markdown list."
|
|
707
|
+
),
|
|
708
|
+
grader=_grade_p99_plan,
|
|
709
|
+
grader_explainer="5 ordered steps, cheap-before-invasive, all 4 fields per step + diagnostic vocabulary",
|
|
710
|
+
max_output_tokens=900,
|
|
711
|
+
),
|
|
712
|
+
# --- Classification (multi-class, some ambiguity) ---
|
|
713
|
+
TestCase(
|
|
714
|
+
name="cls_tone",
|
|
715
|
+
category="classification",
|
|
716
|
+
prompt=(
|
|
717
|
+
"Classify the tone of this sentence as exactly one of: sarcastic, sincere, anxious, defiant. "
|
|
718
|
+
"'Oh sure, because what could possibly go wrong with another all-hands meeting.' "
|
|
719
|
+
"Reply with one word, lowercase, no punctuation."
|
|
720
|
+
),
|
|
721
|
+
grader=_grade_first_word("sarcastic"),
|
|
722
|
+
grader_explainer="First word equals 'sarcastic'",
|
|
723
|
+
max_output_tokens=16,
|
|
724
|
+
),
|
|
725
|
+
TestCase(
|
|
726
|
+
name="cls_intensity",
|
|
727
|
+
category="classification",
|
|
728
|
+
prompt=(
|
|
729
|
+
"Pick one: strong_positive, mild_positive, neutral, mild_negative, strong_negative. "
|
|
730
|
+
"'The interface is fine I guess, gets the job done eventually.' "
|
|
731
|
+
"Reply with the label only."
|
|
732
|
+
),
|
|
733
|
+
grader=_grade_first_word("mild_negative"),
|
|
734
|
+
grader_explainer="First word equals 'mild_negative'",
|
|
735
|
+
max_output_tokens=16,
|
|
736
|
+
),
|
|
737
|
+
TestCase(
|
|
738
|
+
name="cls_bug_severity",
|
|
739
|
+
category="classification",
|
|
740
|
+
prompt=(
|
|
741
|
+
"Bug severity (one of: critical, major, minor, trivial): 'Form labels are missing for 3 fields "
|
|
742
|
+
"in our internal admin panel; screen readers cannot announce which field is focused.' "
|
|
743
|
+
"Reply with the label only."
|
|
744
|
+
),
|
|
745
|
+
grader=_grade_first_word("major"),
|
|
746
|
+
grader_explainer="First word equals 'major'",
|
|
747
|
+
max_output_tokens=16,
|
|
748
|
+
),
|
|
749
|
+
TestCase(
|
|
750
|
+
name="cls_code_review",
|
|
751
|
+
category="classification",
|
|
752
|
+
prompt=(
|
|
753
|
+
"Code review comment type (one of: nitpick, suggestion, blocker, question): "
|
|
754
|
+
"'This violates our auth policy - every endpoint needs the @require_auth decorator before merge.' "
|
|
755
|
+
"Reply with the label only."
|
|
756
|
+
),
|
|
757
|
+
grader=_grade_first_word("blocker"),
|
|
758
|
+
grader_explainer="First word equals 'blocker'",
|
|
759
|
+
max_output_tokens=16,
|
|
760
|
+
),
|
|
761
|
+
TestCase(
|
|
762
|
+
name="cls_emotion",
|
|
763
|
+
category="classification",
|
|
764
|
+
prompt=(
|
|
765
|
+
"Dominant emotion (one of: anger, fear, joy, sadness, surprise, disgust): "
|
|
766
|
+
"'Wait. They scheduled the deploy for Friday afternoon?' "
|
|
767
|
+
"Reply with the label only."
|
|
768
|
+
),
|
|
769
|
+
grader=_grade_first_word("surprise"),
|
|
770
|
+
grader_explainer="First word equals 'surprise'",
|
|
771
|
+
max_output_tokens=16,
|
|
772
|
+
),
|
|
773
|
+
# --- Code (harder) ---
|
|
774
|
+
TestCase(
|
|
775
|
+
name="code_levenshtein",
|
|
776
|
+
category="code",
|
|
777
|
+
prompt=(
|
|
778
|
+
"Write a Python function `levenshtein(a: str, b: str) -> int` returning the Levenshtein edit "
|
|
779
|
+
"distance using dynamic programming. Handle empty strings. Reply with only the function in a "
|
|
780
|
+
"single ```python``` block."
|
|
781
|
+
),
|
|
782
|
+
grader=_grade_levenshtein,
|
|
783
|
+
grader_explainer="Parses; defines levenshtein; passes 5 unit cases incl. empty + kitten/sitting=3",
|
|
784
|
+
max_output_tokens=900,
|
|
785
|
+
),
|
|
786
|
+
TestCase(
|
|
787
|
+
name="code_merge_intervals",
|
|
788
|
+
category="code",
|
|
789
|
+
prompt=(
|
|
790
|
+
"Write a Python function `merge_intervals(intervals: list[tuple[int,int]]) -> list[tuple[int,int]]` "
|
|
791
|
+
"that merges overlapping intervals. Example: [(1,3),(2,6),(8,10)] -> [(1,6),(8,10)]. "
|
|
792
|
+
"Empty input returns []. Reply with only the function in a ```python``` block."
|
|
793
|
+
),
|
|
794
|
+
grader=_grade_merge_intervals,
|
|
795
|
+
grader_explainer="Parses; merges 4 unit cases correctly including empty + chained overlaps",
|
|
796
|
+
max_output_tokens=700,
|
|
797
|
+
),
|
|
798
|
+
TestCase(
|
|
799
|
+
name="code_lru_cache",
|
|
800
|
+
category="code",
|
|
801
|
+
prompt=(
|
|
802
|
+
"Write a Python class `LRUCache` with __init__(capacity), get(key) -> value or -1, put(key, value). "
|
|
803
|
+
"Both O(1). Use dict + doubly-linked list (or OrderedDict). Reply with only the class in a "
|
|
804
|
+
"```python``` block."
|
|
805
|
+
),
|
|
806
|
+
grader=_grade_lru_cache,
|
|
807
|
+
grader_explainer="Parses; standard LeetCode LRU sequence passes (5 assertions)",
|
|
808
|
+
max_output_tokens=900,
|
|
809
|
+
),
|
|
810
|
+
TestCase(
|
|
811
|
+
name="code_sql_second_highest",
|
|
812
|
+
category="code",
|
|
813
|
+
prompt=(
|
|
814
|
+
"Write SQL to find the second-highest salary from an Employee table with columns id, name, salary. "
|
|
815
|
+
"Return NULL when there is no second-highest. Use standard SQL. Reply with only the SQL statement, "
|
|
816
|
+
"no explanation."
|
|
817
|
+
),
|
|
818
|
+
grader=_grade_sql_second_highest,
|
|
819
|
+
grader_explainer="SELECT + (LIMIT 1 OFFSET 1 | DENSE_RANK | MAX <) + NULL handling + Employee table",
|
|
820
|
+
max_output_tokens=300,
|
|
821
|
+
),
|
|
822
|
+
# --- Math (harder) ---
|
|
823
|
+
TestCase(
|
|
824
|
+
name="math_power",
|
|
825
|
+
category="math",
|
|
826
|
+
prompt="What is 23 to the 4th power (23^4)? Reply with only the integer answer.",
|
|
827
|
+
grader=_grade_number(23 ** 4),
|
|
828
|
+
grader_explainer=f"Answer is {23 ** 4}",
|
|
829
|
+
max_output_tokens=32,
|
|
830
|
+
),
|
|
831
|
+
TestCase(
|
|
832
|
+
name="math_trains",
|
|
833
|
+
category="math",
|
|
834
|
+
prompt=(
|
|
835
|
+
"Train A leaves station A at 8:00 AM going 60 mph east. Train B leaves station B (180 miles east "
|
|
836
|
+
"of A) at 9:00 AM going 75 mph west, toward A. At what time do they meet? "
|
|
837
|
+
"Reply with only the time in HH:MM AM/PM format."
|
|
838
|
+
),
|
|
839
|
+
grader=_grade_time_953,
|
|
840
|
+
grader_explainer="9:53 AM ± 1 minute",
|
|
841
|
+
max_output_tokens=64,
|
|
842
|
+
),
|
|
843
|
+
TestCase(
|
|
844
|
+
name="math_algebra",
|
|
845
|
+
category="math",
|
|
846
|
+
prompt="Solve for x: 2x + 5 = 3(x - 4) + 7. Reply with only the integer value of x.",
|
|
847
|
+
grader=_grade_number(10),
|
|
848
|
+
grader_explainer="x = 10",
|
|
849
|
+
max_output_tokens=32,
|
|
850
|
+
),
|
|
851
|
+
TestCase(
|
|
852
|
+
name="math_determinant",
|
|
853
|
+
category="math",
|
|
854
|
+
prompt=(
|
|
855
|
+
"Compute the determinant of the 3x3 matrix [[1,2,3],[4,5,6],[7,8,10]]. "
|
|
856
|
+
"Reply with only the integer."
|
|
857
|
+
),
|
|
858
|
+
grader=_grade_number(-3),
|
|
859
|
+
grader_explainer="det = -3",
|
|
860
|
+
max_output_tokens=64,
|
|
861
|
+
),
|
|
862
|
+
TestCase(
|
|
863
|
+
name="math_probability",
|
|
864
|
+
category="math",
|
|
865
|
+
prompt=(
|
|
866
|
+
"A fair coin is flipped 5 times. What's the probability of getting exactly 3 heads? "
|
|
867
|
+
"Reply with only a reduced fraction like 5/16."
|
|
868
|
+
),
|
|
869
|
+
grader=_grade_fraction(5, 16),
|
|
870
|
+
grader_explainer="5/16",
|
|
871
|
+
max_output_tokens=32,
|
|
872
|
+
),
|
|
873
|
+
# --- Structured output (JSON) ---
|
|
874
|
+
TestCase(
|
|
875
|
+
name="json_request_parse",
|
|
876
|
+
category="structured",
|
|
877
|
+
prompt=(
|
|
878
|
+
"Parse this user request into a JSON object. Schema: "
|
|
879
|
+
"{action: one of [create, read, update, delete], entity: string, filters: object of key-value pairs, "
|
|
880
|
+
"output_format: one of [json, csv, markdown]}. "
|
|
881
|
+
"User said: 'Show me all overdue invoices for customer ACME-42 as a CSV'. "
|
|
882
|
+
"Reply with valid JSON only, no markdown fence, no prose."
|
|
883
|
+
),
|
|
884
|
+
grader=_grade_request_parse,
|
|
885
|
+
grader_explainer="JSON with action=read, entity=invoices, filters contain overdue + ACME-42, output=csv",
|
|
886
|
+
max_output_tokens=300,
|
|
887
|
+
),
|
|
888
|
+
TestCase(
|
|
889
|
+
name="json_flight_extract",
|
|
890
|
+
category="structured",
|
|
891
|
+
prompt=(
|
|
892
|
+
"Extract structured data from this booking request: "
|
|
893
|
+
"'Book the 8:30am ORD to LAX on United for Mr. James Smith, frequent flyer 4429-XK.' "
|
|
894
|
+
"Output a JSON object matching: {airline: string, flight: {origin: string, destination: string, "
|
|
895
|
+
"departure_time: string in HH:MM format}, passenger: {first_name: string, last_name: string, "
|
|
896
|
+
"title: string, ff_number: string}}. Reply with JSON only, no markdown fence."
|
|
897
|
+
),
|
|
898
|
+
grader=_grade_flight_extract,
|
|
899
|
+
grader_explainer="JSON with airline=United, ORD->LAX, 8:30, Smith, 4429-XK",
|
|
900
|
+
max_output_tokens=400,
|
|
901
|
+
),
|
|
902
|
+
TestCase(
|
|
903
|
+
name="json_function_schema",
|
|
904
|
+
category="structured",
|
|
905
|
+
prompt=(
|
|
906
|
+
"Generate the JSON Schema for an OpenAI function-calling parameters object for a function named "
|
|
907
|
+
"schedule_meeting. Parameters: title (string, required), participants (array of strings, required), "
|
|
908
|
+
"duration_minutes (integer, min 15), location (string, optional). Output a JSON object with a top-"
|
|
909
|
+
"level 'parameters' key whose value is the JSON Schema (type=object, properties, required). "
|
|
910
|
+
"Reply with JSON only, no markdown fence, no prose."
|
|
911
|
+
),
|
|
912
|
+
grader=_grade_function_schema,
|
|
913
|
+
grader_explainer="JSON has parameters.type=object, properties has 4 fields, required contains title+participants",
|
|
914
|
+
max_output_tokens=500,
|
|
915
|
+
),
|
|
916
|
+
TestCase(
|
|
917
|
+
name="json_review_extract",
|
|
918
|
+
category="structured",
|
|
919
|
+
prompt=(
|
|
920
|
+
"From this review extract structured data: 'The Eternal Drift movie was visually stunning with great "
|
|
921
|
+
"cinematography but the pacing dragged in act two, ultimately 7 out of 10.' "
|
|
922
|
+
"Output JSON: {title: string, score: number 0-10, pros: array of strings, cons: array of strings}. "
|
|
923
|
+
"JSON only, no markdown fence."
|
|
924
|
+
),
|
|
925
|
+
grader=_grade_review_extract,
|
|
926
|
+
grader_explainer="JSON: title=The Eternal Drift, score=7, ≥1 pro, ≥1 con",
|
|
927
|
+
max_output_tokens=400,
|
|
928
|
+
),
|
|
929
|
+
# --- Hard structured output (modeled on YBM pydantic schemas) ---
|
|
930
|
+
TestCase(
|
|
931
|
+
name="json_intent_classify",
|
|
932
|
+
category="structured_hard",
|
|
933
|
+
prompt=(
|
|
934
|
+
"You are a router for a local agent-control bot. Classify this user message into JSON matching:\n"
|
|
935
|
+
"{\n"
|
|
936
|
+
" \"is_task\": boolean,\n"
|
|
937
|
+
" \"task_type\": one of [development, configuration, admin_control, desktop_observation, question, status_request, other],\n"
|
|
938
|
+
" \"normalized_objective\": string or null,\n"
|
|
939
|
+
" \"confidence\": number 0.0-1.0,\n"
|
|
940
|
+
" \"reason\": string (one sentence),\n"
|
|
941
|
+
" \"intent\": {\n"
|
|
942
|
+
" \"route\": one of [conversation, status, desktop.observe, computer.use, browser.open, browser.control, filesystem.manage, document.manage, artifact.deliver, code.interpreter, coding.agent, schedule.manage, configuration, unknown],\n"
|
|
943
|
+
" \"objective\": string or null,\n"
|
|
944
|
+
" \"url\": string or null,\n"
|
|
945
|
+
" \"folder_path\": string or null,\n"
|
|
946
|
+
" \"file_path\": string or null,\n"
|
|
947
|
+
" \"page_limit\": integer or null,\n"
|
|
948
|
+
" \"delivery\": one of [none, latest, file, screenshot],\n"
|
|
949
|
+
" \"needs_plan_first\": boolean\n"
|
|
950
|
+
" } or null\n"
|
|
951
|
+
"}\n\n"
|
|
952
|
+
"Rule: anything browser/file/automation/code → task_type=other (the specific route lives in intent.route).\n\n"
|
|
953
|
+
"User message: \"Open chrome go to news.ycombinator.com take a screenshot of the front page and send it to me.\"\n\n"
|
|
954
|
+
"Reply with JSON only, no markdown fence, no prose."
|
|
955
|
+
),
|
|
956
|
+
grader=_grade_intent_classify,
|
|
957
|
+
grader_explainer="is_task=true, task_type=other, route=browser.open, url has ycombinator, delivery=screenshot, nested intent shape",
|
|
958
|
+
max_output_tokens=600,
|
|
959
|
+
),
|
|
960
|
+
TestCase(
|
|
961
|
+
name="json_plan_orchestration",
|
|
962
|
+
category="structured_hard",
|
|
963
|
+
prompt=(
|
|
964
|
+
"Generate a JSON execution plan for: \"Take a screenshot of the current desktop and send it to me.\"\n\n"
|
|
965
|
+
"Schema:\n"
|
|
966
|
+
"{\n"
|
|
967
|
+
" \"objective\": string,\n"
|
|
968
|
+
" \"assumptions\": array of strings,\n"
|
|
969
|
+
" \"required_capabilities\": array of capability strings (from: telegram.send, llm.generate, vscode.read_state, terminal.run, filesystem.read, filesystem.write, desktop.screenshot, desktop.control, browser.open, browser.control, schedule.manage, github.read, github.push, dependencies.install),\n"
|
|
970
|
+
" \"steps\": array of {\n"
|
|
971
|
+
" \"title\": string,\n"
|
|
972
|
+
" \"description\": string,\n"
|
|
973
|
+
" \"required_capabilities\": array of capability strings (subset of the plan's top-level required_capabilities),\n"
|
|
974
|
+
" \"risk_level\": one of [low, medium, high, critical],\n"
|
|
975
|
+
" \"requires_approval\": boolean,\n"
|
|
976
|
+
" \"tool_name\": string,\n"
|
|
977
|
+
" \"tool_input\": object,\n"
|
|
978
|
+
" \"expected_output\": string\n"
|
|
979
|
+
" },\n"
|
|
980
|
+
" \"success_criteria\": array of strings,\n"
|
|
981
|
+
" \"approval_gates\": array of {capability: string, risk_level: enum, summary: string}\n"
|
|
982
|
+
"}\n\n"
|
|
983
|
+
"Hard constraints:\n"
|
|
984
|
+
"- The plan MUST include at least one step using desktop.screenshot AND at least one using telegram.send.\n"
|
|
985
|
+
"- Each step's required_capabilities MUST be a subset of the plan's top-level required_capabilities.\n"
|
|
986
|
+
"- desktop.screenshot and telegram.send are both risk_level=low and do NOT require approval.\n\n"
|
|
987
|
+
"Reply with JSON only, no markdown fence, no prose."
|
|
988
|
+
),
|
|
989
|
+
grader=_grade_plan_orchestration,
|
|
990
|
+
grader_explainer="Valid PlanModel with screenshot+send caps, ≥2 steps, capability-subset constraint holds",
|
|
991
|
+
max_output_tokens=1200,
|
|
992
|
+
),
|
|
993
|
+
TestCase(
|
|
994
|
+
name="json_approval_decision",
|
|
995
|
+
category="structured_hard",
|
|
996
|
+
prompt=(
|
|
997
|
+
"A user wants to delete the file C:\\Users\\me\\Documents\\proposal_draft.docx. "
|
|
998
|
+
"This is a destructive action on a user document. Generate a cautious approval decision as JSON:\n\n"
|
|
999
|
+
"{\n"
|
|
1000
|
+
" \"decision\": one of [approve, reject, needs_clarification],\n"
|
|
1001
|
+
" \"risk_level\": one of [low, medium, high, critical],\n"
|
|
1002
|
+
" \"reasons\": array of at least 2 strings,\n"
|
|
1003
|
+
" \"required_capabilities\": array of strings (from: filesystem.read, filesystem.write, filesystem.delete, telegram.send),\n"
|
|
1004
|
+
" \"mitigations\": array of strings (can be empty),\n"
|
|
1005
|
+
" \"alternative_action\": object with {action: string, rationale: string} OR null,\n"
|
|
1006
|
+
" \"needs_user_confirmation\": boolean,\n"
|
|
1007
|
+
" \"confirmation_prompt\": string OR null\n"
|
|
1008
|
+
"}\n\n"
|
|
1009
|
+
"Hard constraint: confirmation_prompt MUST be a non-empty string when needs_user_confirmation=true, "
|
|
1010
|
+
"and MUST be null when needs_user_confirmation=false.\n\n"
|
|
1011
|
+
"Reply with JSON only, no markdown fence, no prose."
|
|
1012
|
+
),
|
|
1013
|
+
grader=_grade_approval_decision,
|
|
1014
|
+
grader_explainer="Cautious decision (reject/needs_clarification), high risk, ≥2 reasons, conditional confirmation_prompt holds",
|
|
1015
|
+
max_output_tokens=600,
|
|
1016
|
+
),
|
|
1017
|
+
TestCase(
|
|
1018
|
+
name="json_multi_task_extract",
|
|
1019
|
+
category="structured_hard",
|
|
1020
|
+
prompt=(
|
|
1021
|
+
"Extract a task list from this user message:\n"
|
|
1022
|
+
"\"Hey can you do these: Watch my Downloads folder for new PDFs and send a summary of each one to me. "
|
|
1023
|
+
"Also schedule a daily 9am report on yesterday's commits in my github.com/iodriller/LocalDeploy repo. "
|
|
1024
|
+
"Don't bother me about Codex or anything advanced.\"\n\n"
|
|
1025
|
+
"Schema:\n"
|
|
1026
|
+
"{\n"
|
|
1027
|
+
" \"tasks\": array of {\n"
|
|
1028
|
+
" \"id\": integer (sequential starting at 1),\n"
|
|
1029
|
+
" \"objective\": string (concise),\n"
|
|
1030
|
+
" \"route\": one of [filesystem.manage, document.manage, schedule.manage, browser.open, coding.agent],\n"
|
|
1031
|
+
" \"trigger\": {kind: one of [immediate, watch, schedule], details: string},\n"
|
|
1032
|
+
" \"delivery\": one of [none, latest, file, screenshot],\n"
|
|
1033
|
+
" \"constraints\": array of strings (user's negative constraints)\n"
|
|
1034
|
+
" },\n"
|
|
1035
|
+
" \"ignored_instructions\": array of strings (instructions you cannot fulfill, can be empty)\n"
|
|
1036
|
+
"}\n\n"
|
|
1037
|
+
"Reply with JSON only, no markdown fence, no prose."
|
|
1038
|
+
),
|
|
1039
|
+
grader=_grade_multi_task_extract,
|
|
1040
|
+
grader_explainer="2 tasks: one fs/doc watch + one schedule.manage with 9am/daily, Codex constraint preserved",
|
|
1041
|
+
max_output_tokens=900,
|
|
1042
|
+
),
|
|
1043
|
+
]
|
|
1044
|
+
|
|
1045
|
+
|
|
1046
|
+
# ---------------------------------------------------------------------------
|
|
1047
|
+
# Runtime
|
|
1048
|
+
# ---------------------------------------------------------------------------
|
|
1049
|
+
|
|
1050
|
+
|
|
1051
|
+
@dataclass
|
|
1052
|
+
class TestResult:
|
|
1053
|
+
name: str
|
|
1054
|
+
category: str
|
|
1055
|
+
success: bool
|
|
1056
|
+
elapsed_seconds: float
|
|
1057
|
+
response_length: int
|
|
1058
|
+
response_preview: str
|
|
1059
|
+
accuracy: float
|
|
1060
|
+
error: Optional[str] = None
|
|
1061
|
+
warning: Optional[str] = None
|
|
1062
|
+
approx_tokens_per_second: Optional[float] = None
|
|
1063
|
+
repetition: int = 1
|
|
1064
|
+
metrics: Dict[str, Any] = field(default_factory=dict)
|
|
1065
|
+
tokens_per_second_source: str = "estimated"
|
|
1066
|
+
warm_state: Optional[str] = None
|
|
1067
|
+
context_limit_used: Optional[int] = None
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
@dataclass
|
|
1071
|
+
class ProfileResult:
|
|
1072
|
+
profile: str
|
|
1073
|
+
model_id: str
|
|
1074
|
+
backend: str
|
|
1075
|
+
enabled: bool
|
|
1076
|
+
recommended_for_8gb_vram: Any
|
|
1077
|
+
vram_before_mb: Optional[int]
|
|
1078
|
+
vram_after_mb: Optional[int]
|
|
1079
|
+
vram_peak_mb: Optional[int]
|
|
1080
|
+
tests: List[TestResult] = field(default_factory=list)
|
|
1081
|
+
fits_in_vram: bool = True
|
|
1082
|
+
notes: List[str] = field(default_factory=list)
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
def _percentile(values: List[float], pct: float) -> Optional[float]:
|
|
1086
|
+
"""Linear-interpolation percentile (the common default, e.g. numpy's).
|
|
1087
|
+
None for an empty list; the single value for a singleton list."""
|
|
1088
|
+
if not values:
|
|
1089
|
+
return None
|
|
1090
|
+
ordered = sorted(values)
|
|
1091
|
+
if len(ordered) == 1:
|
|
1092
|
+
return ordered[0]
|
|
1093
|
+
k = (len(ordered) - 1) * (pct / 100.0)
|
|
1094
|
+
f = int(k)
|
|
1095
|
+
c = min(f + 1, len(ordered) - 1)
|
|
1096
|
+
if f == c:
|
|
1097
|
+
return ordered[f]
|
|
1098
|
+
return ordered[f] * (c - k) + ordered[c] * (k - f)
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
def _stats_block(values: List[float], digits: int = 3) -> Dict[str, Optional[float]]:
|
|
1102
|
+
"""Mean/median/stdev/min/max/P90/P95 in one call - the repeated-run
|
|
1103
|
+
statistics the benchmark expansion plan asks for, computed once and
|
|
1104
|
+
reused for both latency and tokens/sec rather than duplicated per metric."""
|
|
1105
|
+
if not values:
|
|
1106
|
+
return {
|
|
1107
|
+
"mean": None, "median": None, "stdev": 0.0, "min": None, "max": None,
|
|
1108
|
+
"p90": None, "p95": None,
|
|
1109
|
+
}
|
|
1110
|
+
return {
|
|
1111
|
+
"mean": round(statistics.mean(values), digits),
|
|
1112
|
+
"median": round(statistics.median(values), digits),
|
|
1113
|
+
"stdev": round(statistics.stdev(values), digits) if len(values) > 1 else 0.0,
|
|
1114
|
+
"min": round(min(values), digits),
|
|
1115
|
+
"max": round(max(values), digits),
|
|
1116
|
+
"p90": round(_percentile(values, 90), digits),
|
|
1117
|
+
"p95": round(_percentile(values, 95), digits),
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def nvidia_smi_used_mb() -> Optional[int]:
|
|
1122
|
+
try:
|
|
1123
|
+
output = subprocess.check_output(
|
|
1124
|
+
["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],
|
|
1125
|
+
stderr=subprocess.STDOUT,
|
|
1126
|
+
timeout=5,
|
|
1127
|
+
)
|
|
1128
|
+
except (FileNotFoundError, subprocess.SubprocessError):
|
|
1129
|
+
return None
|
|
1130
|
+
try:
|
|
1131
|
+
return int(output.decode("ascii", errors="ignore").strip().splitlines()[0].strip())
|
|
1132
|
+
except (ValueError, IndexError):
|
|
1133
|
+
return None
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
_THINK_BLOCK = re.compile(r"<think>.*?</think>", flags=re.DOTALL | re.IGNORECASE)
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
def strip_thinking_tags(text: str) -> str:
|
|
1140
|
+
"""Remove <think>...</think> blocks emitted by reasoning models (Qwen3, DeepSeek-R1).
|
|
1141
|
+
|
|
1142
|
+
Leaves the final answer that comes after the last </think>. If the response is just
|
|
1143
|
+
thinking with no </think> close (truncated), returns the original so the grader sees
|
|
1144
|
+
something rather than empty.
|
|
1145
|
+
"""
|
|
1146
|
+
if "</think>" in text.lower():
|
|
1147
|
+
cleaned = _THINK_BLOCK.sub("", text).strip()
|
|
1148
|
+
return cleaned if cleaned else text
|
|
1149
|
+
return text
|
|
1150
|
+
|
|
1151
|
+
|
|
1152
|
+
def call_chat(
|
|
1153
|
+
base_url: str,
|
|
1154
|
+
profile_name: str,
|
|
1155
|
+
profile_cfg: Dict[str, Any],
|
|
1156
|
+
test: TestCase,
|
|
1157
|
+
timeout: int,
|
|
1158
|
+
num_gpu: Optional[int] = None,
|
|
1159
|
+
context_override: Optional[int] = None,
|
|
1160
|
+
) -> Dict[str, Any]:
|
|
1161
|
+
suffix = str(profile_cfg.get("prompt_suffix") or "")
|
|
1162
|
+
prompt = test.prompt + (("\n\n" + suffix) if suffix else "")
|
|
1163
|
+
payload = {
|
|
1164
|
+
"profile": profile_name,
|
|
1165
|
+
"prompt": prompt,
|
|
1166
|
+
"safe_mode": True,
|
|
1167
|
+
"max_output_tokens": test.max_output_tokens,
|
|
1168
|
+
}
|
|
1169
|
+
# Pin the device on the inference call itself so a CPU/GPU benchmark stays on
|
|
1170
|
+
# the device it warmed on instead of letting Ollama re-place the model.
|
|
1171
|
+
if num_gpu is not None:
|
|
1172
|
+
payload["num_gpu"] = num_gpu
|
|
1173
|
+
# Context-scaling sweep: /chat already accepts a per-request context_limit
|
|
1174
|
+
# override (prepare_request), so no server-side plumbing is needed to test
|
|
1175
|
+
# the same model at several context lengths without editing its profile.
|
|
1176
|
+
if context_override is not None:
|
|
1177
|
+
payload["context_limit"] = context_override
|
|
1178
|
+
payload["allow_clamp"] = True
|
|
1179
|
+
try:
|
|
1180
|
+
response = requests.post(
|
|
1181
|
+
f"{base_url}/chat",
|
|
1182
|
+
json=payload,
|
|
1183
|
+
headers=api_auth_headers(),
|
|
1184
|
+
timeout=timeout + 10,
|
|
1185
|
+
)
|
|
1186
|
+
except requests.Timeout:
|
|
1187
|
+
return {"success": False, "error": f"client-side timeout after {timeout}s", "elapsed_seconds": float(timeout)}
|
|
1188
|
+
except requests.ConnectionError:
|
|
1189
|
+
return {"success": False, "error": f"could not reach {base_url}", "elapsed_seconds": 0.0}
|
|
1190
|
+
try:
|
|
1191
|
+
data = response.json()
|
|
1192
|
+
except ValueError:
|
|
1193
|
+
return {"success": False, "error": f"HTTP {response.status_code}: {response.text[:200]}", "elapsed_seconds": 0.0}
|
|
1194
|
+
|
|
1195
|
+
# Strip <think> blocks if the profile is a reasoning model - the answer lives
|
|
1196
|
+
# after </think> for these. Leave the raw response intact when not requested.
|
|
1197
|
+
if profile_cfg.get("strip_thinking_tags") and isinstance(data.get("response"), str):
|
|
1198
|
+
data["response"] = strip_thinking_tags(data["response"])
|
|
1199
|
+
return data
|
|
1200
|
+
|
|
1201
|
+
|
|
1202
|
+
def _is_oom(error: Optional[str]) -> bool:
|
|
1203
|
+
err = str(error or "").lower()
|
|
1204
|
+
return "memory" in err or "cuda" in err or "out of memory" in err
|
|
1205
|
+
|
|
1206
|
+
|
|
1207
|
+
def _is_not_pulled(error: Optional[str]) -> bool:
|
|
1208
|
+
err = str(error or "").lower()
|
|
1209
|
+
return "not available" in err or "not found" in err or "pull" in err
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
def execute_test(
|
|
1213
|
+
base_url: str,
|
|
1214
|
+
profile_name: str,
|
|
1215
|
+
profile: Dict[str, Any],
|
|
1216
|
+
test: TestCase,
|
|
1217
|
+
timeout: int,
|
|
1218
|
+
num_gpu: Optional[int] = None,
|
|
1219
|
+
repetition: int = 1,
|
|
1220
|
+
warm_state: Optional[str] = None,
|
|
1221
|
+
context_override: Optional[int] = None,
|
|
1222
|
+
) -> TestResult:
|
|
1223
|
+
"""Run one test against one profile and grade it.
|
|
1224
|
+
|
|
1225
|
+
This is the shared per-test unit used by both the CLI run loop (`run_profile`)
|
|
1226
|
+
and the streaming `/benchmark/run` endpoint, so call + grading never diverge.
|
|
1227
|
+
It does not print PASS/FAIL or sample VRAM - those stay with the caller.
|
|
1228
|
+
``num_gpu`` pins the device on the inference call (0 = CPU) for forced runs.
|
|
1229
|
+
``context_override`` pins a per-call context length for context-scaling sweeps.
|
|
1230
|
+
"""
|
|
1231
|
+
started = time.perf_counter()
|
|
1232
|
+
data = call_chat(base_url, profile_name, profile, test, timeout, num_gpu=num_gpu, context_override=context_override)
|
|
1233
|
+
elapsed = float(data.get("elapsed_seconds") or (time.perf_counter() - started))
|
|
1234
|
+
raw_resp = data.get("response")
|
|
1235
|
+
response_text = raw_resp if isinstance(raw_resp, str) else json.dumps(raw_resp or "", ensure_ascii=False)
|
|
1236
|
+
response_text = str(response_text or "")
|
|
1237
|
+
success = bool(data.get("success"))
|
|
1238
|
+
try:
|
|
1239
|
+
accuracy = test.grader(response_text) if success else 0.0
|
|
1240
|
+
except Exception as exc: # grader bug should not crash the run
|
|
1241
|
+
accuracy = 0.0
|
|
1242
|
+
print(f" WARN grader for {test.name} raised: {exc}", flush=True)
|
|
1243
|
+
metrics = data.get("metrics") if isinstance(data.get("metrics"), dict) else {}
|
|
1244
|
+
native_tps = metrics.get("tokens_per_second")
|
|
1245
|
+
try:
|
|
1246
|
+
native_tps = float(native_tps) if native_tps is not None else None
|
|
1247
|
+
except (TypeError, ValueError):
|
|
1248
|
+
native_tps = None
|
|
1249
|
+
approx_tps = native_tps or (((len(response_text) / 4) / elapsed) if success and elapsed > 0 else None)
|
|
1250
|
+
|
|
1251
|
+
return TestResult(
|
|
1252
|
+
name=test.name,
|
|
1253
|
+
category=test.category,
|
|
1254
|
+
success=success,
|
|
1255
|
+
elapsed_seconds=round(elapsed, 3),
|
|
1256
|
+
response_length=len(response_text),
|
|
1257
|
+
response_preview=response_text[:240].replace("\n", " "),
|
|
1258
|
+
accuracy=round(accuracy, 3),
|
|
1259
|
+
error=data.get("error"),
|
|
1260
|
+
warning=data.get("warning"),
|
|
1261
|
+
approx_tokens_per_second=round(approx_tps, 2) if approx_tps is not None else None,
|
|
1262
|
+
repetition=repetition,
|
|
1263
|
+
metrics=metrics,
|
|
1264
|
+
tokens_per_second_source="backend" if native_tps is not None else "estimated",
|
|
1265
|
+
warm_state=warm_state,
|
|
1266
|
+
context_limit_used=data.get("context_limit_used"),
|
|
1267
|
+
)
|
|
1268
|
+
|
|
1269
|
+
|
|
1270
|
+
def run_profile(base_url: str, profile_name: str, profile: Dict[str, Any], tests: List[TestCase], timeout: int) -> ProfileResult:
|
|
1271
|
+
print(f"\n=== {profile_name} ({profile.get('model_id')}) ===", flush=True)
|
|
1272
|
+
vram_before = nvidia_smi_used_mb()
|
|
1273
|
+
print(f" VRAM before: {vram_before} MB" if vram_before is not None else " VRAM before: unavailable", flush=True)
|
|
1274
|
+
|
|
1275
|
+
result = ProfileResult(
|
|
1276
|
+
profile=profile_name,
|
|
1277
|
+
model_id=str(profile.get("model_id") or ""),
|
|
1278
|
+
backend=str(profile.get("backend") or ""),
|
|
1279
|
+
enabled=bool(profile.get("enabled", False)),
|
|
1280
|
+
recommended_for_8gb_vram=profile.get("recommended_for_8gb_vram"),
|
|
1281
|
+
vram_before_mb=vram_before,
|
|
1282
|
+
vram_after_mb=None,
|
|
1283
|
+
vram_peak_mb=vram_before,
|
|
1284
|
+
)
|
|
1285
|
+
|
|
1286
|
+
consecutive_fail = 0
|
|
1287
|
+
for test in tests:
|
|
1288
|
+
item = execute_test(base_url, profile_name, profile, test, timeout)
|
|
1289
|
+
|
|
1290
|
+
vram_now = nvidia_smi_used_mb()
|
|
1291
|
+
if vram_now is not None and (result.vram_peak_mb is None or vram_now > result.vram_peak_mb):
|
|
1292
|
+
result.vram_peak_mb = vram_now
|
|
1293
|
+
|
|
1294
|
+
result.tests.append(item)
|
|
1295
|
+
|
|
1296
|
+
status = "PASS" if item.success else "FAIL"
|
|
1297
|
+
acc_label = f"acc={item.accuracy:.2f}" if item.success else "acc=n/a"
|
|
1298
|
+
tps_label = f"~{item.approx_tokens_per_second:.1f} tok/s" if item.approx_tokens_per_second else ""
|
|
1299
|
+
err_label = f" ({item.error})" if not item.success else ""
|
|
1300
|
+
print(
|
|
1301
|
+
f" [{status}] {test.category:14} {test.name:26} {item.elapsed_seconds:6.2f}s {acc_label} {tps_label}{err_label}",
|
|
1302
|
+
flush=True,
|
|
1303
|
+
)
|
|
1304
|
+
|
|
1305
|
+
if not item.success:
|
|
1306
|
+
consecutive_fail += 1
|
|
1307
|
+
if _is_oom(item.error):
|
|
1308
|
+
result.fits_in_vram = False
|
|
1309
|
+
result.notes.append(f"OOM-like failure on {test.name}")
|
|
1310
|
+
if _is_not_pulled(item.error):
|
|
1311
|
+
result.notes.append(f"Model not pulled: {item.error}")
|
|
1312
|
+
break
|
|
1313
|
+
if consecutive_fail >= 4:
|
|
1314
|
+
result.notes.append(f"Aborted after 4 consecutive failures starting at {test.name}.")
|
|
1315
|
+
break
|
|
1316
|
+
else:
|
|
1317
|
+
consecutive_fail = 0
|
|
1318
|
+
|
|
1319
|
+
result.vram_after_mb = nvidia_smi_used_mb()
|
|
1320
|
+
print(f" VRAM after: {result.vram_after_mb} MB peak: {result.vram_peak_mb} MB", flush=True)
|
|
1321
|
+
return result
|
|
1322
|
+
|
|
1323
|
+
|
|
1324
|
+
def iter_run(
|
|
1325
|
+
base_url: str,
|
|
1326
|
+
profiles_map: Dict[str, Any],
|
|
1327
|
+
selected: List[str],
|
|
1328
|
+
tests: List[TestCase],
|
|
1329
|
+
timeout: int,
|
|
1330
|
+
num_gpu: Optional[int] = None,
|
|
1331
|
+
repetitions: int = 1,
|
|
1332
|
+
provenance: Optional[Dict[str, Any]] = None,
|
|
1333
|
+
) -> Iterator[Dict[str, Any]]:
|
|
1334
|
+
"""Stream a benchmark run as a sequence of event dicts.
|
|
1335
|
+
|
|
1336
|
+
Shares `execute_test` with the CLI so grading is identical. Emits:
|
|
1337
|
+
run_start, profile_start, test_start/test_result(*), profile_aborted?,
|
|
1338
|
+
profile_end, run_end. Used by the streaming /benchmark/run endpoint. No printing, no
|
|
1339
|
+
VRAM tracking - the caller decides how to present events. ``num_gpu`` pins
|
|
1340
|
+
the device on each inference call (0 = CPU) for forced CPU/GPU runs.
|
|
1341
|
+
"""
|
|
1342
|
+
yield {
|
|
1343
|
+
"event": "run_start",
|
|
1344
|
+
"profiles": list(selected),
|
|
1345
|
+
"test_count": len(tests) * repetitions,
|
|
1346
|
+
"unique_test_count": len(tests),
|
|
1347
|
+
"repetitions": repetitions,
|
|
1348
|
+
"tests": [t.name for t in tests],
|
|
1349
|
+
"categories": sorted({t.category for t in tests}),
|
|
1350
|
+
"provenance": provenance or {},
|
|
1351
|
+
}
|
|
1352
|
+
started = time.perf_counter()
|
|
1353
|
+
overall: List[Dict[str, Any]] = []
|
|
1354
|
+
for name in selected:
|
|
1355
|
+
profile = profiles_map[name]
|
|
1356
|
+
profile_provenance = ((provenance or {}).get("profiles") or {}).get(name, {})
|
|
1357
|
+
warm_state = profile_provenance.get("warm_state")
|
|
1358
|
+
yield {
|
|
1359
|
+
"event": "profile_start",
|
|
1360
|
+
"profile": name,
|
|
1361
|
+
"model_id": profile.get("model_id"),
|
|
1362
|
+
"provenance": profile_provenance,
|
|
1363
|
+
}
|
|
1364
|
+
consecutive_fail = 0
|
|
1365
|
+
prof: List[TestResult] = []
|
|
1366
|
+
abort = False
|
|
1367
|
+
for test in tests:
|
|
1368
|
+
repeated: List[TestResult] = []
|
|
1369
|
+
for repetition in range(1, repetitions + 1):
|
|
1370
|
+
yield {
|
|
1371
|
+
"event": "test_start",
|
|
1372
|
+
"profile": name,
|
|
1373
|
+
"name": test.name,
|
|
1374
|
+
"category": test.category,
|
|
1375
|
+
"repetition": repetition,
|
|
1376
|
+
"repetitions": repetitions,
|
|
1377
|
+
}
|
|
1378
|
+
item = execute_test(
|
|
1379
|
+
base_url,
|
|
1380
|
+
name,
|
|
1381
|
+
profile,
|
|
1382
|
+
test,
|
|
1383
|
+
timeout,
|
|
1384
|
+
num_gpu=num_gpu,
|
|
1385
|
+
)
|
|
1386
|
+
item.repetition = repetition
|
|
1387
|
+
item.warm_state = warm_state if repetition == 1 else "warm"
|
|
1388
|
+
prof.append(item)
|
|
1389
|
+
repeated.append(item)
|
|
1390
|
+
event = asdict(item)
|
|
1391
|
+
event["event"] = "test_result"
|
|
1392
|
+
event["profile"] = name
|
|
1393
|
+
event["repetitions"] = repetitions
|
|
1394
|
+
yield event
|
|
1395
|
+
|
|
1396
|
+
if not item.success:
|
|
1397
|
+
consecutive_fail += 1
|
|
1398
|
+
if _is_not_pulled(item.error):
|
|
1399
|
+
yield {
|
|
1400
|
+
"event": "profile_aborted",
|
|
1401
|
+
"profile": name,
|
|
1402
|
+
"reason": f"model not pulled: {item.error}",
|
|
1403
|
+
}
|
|
1404
|
+
abort = True
|
|
1405
|
+
break
|
|
1406
|
+
if consecutive_fail >= 4:
|
|
1407
|
+
yield {"event": "profile_aborted", "profile": name, "reason": "4 consecutive failures"}
|
|
1408
|
+
abort = True
|
|
1409
|
+
break
|
|
1410
|
+
else:
|
|
1411
|
+
consecutive_fail = 0
|
|
1412
|
+
successful = [item for item in repeated if item.success]
|
|
1413
|
+
rates = [item.approx_tokens_per_second for item in successful if item.approx_tokens_per_second is not None]
|
|
1414
|
+
latency_stats = _stats_block([item.elapsed_seconds for item in successful], digits=3)
|
|
1415
|
+
tps_stats = _stats_block(rates, digits=2)
|
|
1416
|
+
yield {
|
|
1417
|
+
"event": "test_aggregate",
|
|
1418
|
+
"profile": name,
|
|
1419
|
+
"name": test.name,
|
|
1420
|
+
"runs": len(repeated),
|
|
1421
|
+
"successful_runs": len(successful),
|
|
1422
|
+
"mean_latency_seconds": latency_stats["mean"],
|
|
1423
|
+
"latency_stdev_seconds": latency_stats["stdev"],
|
|
1424
|
+
"latency_p90_seconds": latency_stats["p90"],
|
|
1425
|
+
"latency_p95_seconds": latency_stats["p95"],
|
|
1426
|
+
"mean_tokens_per_second": tps_stats["mean"],
|
|
1427
|
+
"tokens_per_second_stdev": tps_stats["stdev"],
|
|
1428
|
+
"tokens_per_second_p90": tps_stats["p90"],
|
|
1429
|
+
"accuracy_stdev": round(statistics.stdev([item.accuracy for item in repeated]), 3)
|
|
1430
|
+
if len(repeated) > 1
|
|
1431
|
+
else 0.0,
|
|
1432
|
+
}
|
|
1433
|
+
if abort:
|
|
1434
|
+
break
|
|
1435
|
+
|
|
1436
|
+
passed = sum(1 for t in prof if t.success)
|
|
1437
|
+
avg_acc = round(statistics.mean([t.accuracy for t in prof]), 3) if prof else 0.0
|
|
1438
|
+
successful_rates = [t.approx_tokens_per_second for t in prof if t.success and t.approx_tokens_per_second is not None]
|
|
1439
|
+
successful_latencies = [t.elapsed_seconds for t in prof if t.success]
|
|
1440
|
+
summary = {"tests": len(prof), "passed": passed, "avg_accuracy": avg_acc}
|
|
1441
|
+
if repetitions > 1 or provenance is not None:
|
|
1442
|
+
latency_stats = _stats_block(successful_latencies, digits=3)
|
|
1443
|
+
tps_stats = _stats_block(successful_rates, digits=2)
|
|
1444
|
+
summary.update(
|
|
1445
|
+
{
|
|
1446
|
+
"unique_tests": len({t.name for t in prof}),
|
|
1447
|
+
"repetitions": repetitions,
|
|
1448
|
+
"latency_stdev_seconds": latency_stats["stdev"],
|
|
1449
|
+
"latency_p90_seconds": latency_stats["p90"],
|
|
1450
|
+
"latency_p95_seconds": latency_stats["p95"],
|
|
1451
|
+
"tokens_per_second_stdev": tps_stats["stdev"],
|
|
1452
|
+
"tokens_per_second_p90": tps_stats["p90"],
|
|
1453
|
+
}
|
|
1454
|
+
)
|
|
1455
|
+
overall.append({"profile": name, **summary})
|
|
1456
|
+
yield {"event": "profile_end", "profile": name, "summary": summary}
|
|
1457
|
+
|
|
1458
|
+
yield {
|
|
1459
|
+
"event": "run_end",
|
|
1460
|
+
"elapsed_seconds": round(time.perf_counter() - started, 2),
|
|
1461
|
+
"profiles": overall,
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
|
|
1465
|
+
# ---------------------------------------------------------------------------
|
|
1466
|
+
# Reporting
|
|
1467
|
+
# ---------------------------------------------------------------------------
|
|
1468
|
+
|
|
1469
|
+
|
|
1470
|
+
def category_summary(tests: List[TestResult]) -> Dict[str, Dict[str, float]]:
|
|
1471
|
+
out: Dict[str, Dict[str, float]] = {}
|
|
1472
|
+
for cat in {t.category for t in tests}:
|
|
1473
|
+
subset = [t for t in tests if t.category == cat]
|
|
1474
|
+
successes = [t for t in subset if t.success]
|
|
1475
|
+
elapsed = [t.elapsed_seconds for t in successes]
|
|
1476
|
+
accs = [t.accuracy for t in subset]
|
|
1477
|
+
out[cat] = {
|
|
1478
|
+
"n": len(subset),
|
|
1479
|
+
"passed": len(successes),
|
|
1480
|
+
"avg_elapsed_seconds": round(statistics.mean(elapsed), 3) if elapsed else 0.0,
|
|
1481
|
+
"avg_accuracy": round(statistics.mean(accs), 3) if accs else 0.0,
|
|
1482
|
+
}
|
|
1483
|
+
return out
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def write_reports(results: List[ProfileResult], run_meta: Dict[str, Any]) -> Tuple[Path, Path]:
|
|
1487
|
+
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
|
1488
|
+
stamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H%M%SZ")
|
|
1489
|
+
json_path = REPORTS_DIR / f"benchmark_{stamp}.json"
|
|
1490
|
+
md_path = REPORTS_DIR / f"benchmark_{stamp}.md"
|
|
1491
|
+
|
|
1492
|
+
payload = {
|
|
1493
|
+
"meta": run_meta,
|
|
1494
|
+
"results": [
|
|
1495
|
+
{**asdict(p), "tests": [asdict(t) for t in p.tests], "category_summary": category_summary(p.tests)}
|
|
1496
|
+
for p in results
|
|
1497
|
+
],
|
|
1498
|
+
}
|
|
1499
|
+
json_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
1500
|
+
|
|
1501
|
+
lines: List[str] = [
|
|
1502
|
+
f"# LocalDeploy benchmark v2 - {stamp}",
|
|
1503
|
+
"",
|
|
1504
|
+
f"- GPU: `{run_meta.get('gpu_name')}` ({run_meta.get('gpu_total_mb')} MB total)",
|
|
1505
|
+
f"- API: `{run_meta.get('base_url')}`",
|
|
1506
|
+
f"- Test cases per profile: {run_meta.get('test_count')}",
|
|
1507
|
+
f"- Total elapsed: {run_meta.get('elapsed_seconds')}s",
|
|
1508
|
+
"",
|
|
1509
|
+
"## Categories",
|
|
1510
|
+
"",
|
|
1511
|
+
"- **planning** - software-project decomposition (3 tests)",
|
|
1512
|
+
"- **classification** - multi-class label with ambiguity (5 tests)",
|
|
1513
|
+
"- **code** - non-trivial Python + SQL, graded by AST + unit tests (4 tests)",
|
|
1514
|
+
"- **math** - multi-step arithmetic, algebra, probability, matrices (5 tests)",
|
|
1515
|
+
"- **structured** - JSON-schema-shaped output, field-level grading (4 tests)",
|
|
1516
|
+
"- **structured_hard** - pydantic-schema output: classification, plan orchestration, "
|
|
1517
|
+
"approval gate, multi-task extraction (4 tests)",
|
|
1518
|
+
"",
|
|
1519
|
+
"## Overall scoreboard",
|
|
1520
|
+
"",
|
|
1521
|
+
"| Profile | Model | Pass | Avg accuracy | Avg latency | Peak VRAM | Verdict |",
|
|
1522
|
+
"|---|---|---:|---:|---:|---:|---|",
|
|
1523
|
+
]
|
|
1524
|
+
|
|
1525
|
+
def overall_acc(p: ProfileResult) -> float:
|
|
1526
|
+
accs = [t.accuracy for t in p.tests if t.success]
|
|
1527
|
+
return statistics.mean(accs) if accs else 0.0
|
|
1528
|
+
|
|
1529
|
+
for p in sorted(results, key=lambda x: (-overall_acc(x), -sum(1 for t in x.tests if t.success))):
|
|
1530
|
+
passed = sum(1 for t in p.tests if t.success)
|
|
1531
|
+
total = len(p.tests)
|
|
1532
|
+
accs = [t.accuracy for t in p.tests if t.success]
|
|
1533
|
+
avg_acc = statistics.mean(accs) if accs else 0.0
|
|
1534
|
+
lat = [t.elapsed_seconds for t in p.tests if t.success]
|
|
1535
|
+
avg_lat = statistics.mean(lat) if lat else 0.0
|
|
1536
|
+
peak = p.vram_peak_mb or 0
|
|
1537
|
+
if not p.fits_in_vram:
|
|
1538
|
+
verdict = "OOM"
|
|
1539
|
+
elif passed == 0:
|
|
1540
|
+
verdict = "not pulled / unreachable"
|
|
1541
|
+
elif passed == total:
|
|
1542
|
+
verdict = "fits"
|
|
1543
|
+
else:
|
|
1544
|
+
verdict = f"partial ({passed}/{total})"
|
|
1545
|
+
lines.append(
|
|
1546
|
+
f"| `{p.profile}` | `{p.model_id}` | {passed}/{total} | "
|
|
1547
|
+
f"{avg_acc:.2f} | {avg_lat:.2f}s | {peak} MB | {verdict} |"
|
|
1548
|
+
)
|
|
1549
|
+
|
|
1550
|
+
# Category-by-model breakdown. Categories are derived from TEST_CASES (not
|
|
1551
|
+
# hardcoded) so a newly added category always shows up here instead of
|
|
1552
|
+
# silently being dropped from the scoreboard while still scored elsewhere.
|
|
1553
|
+
category_order = list(dict.fromkeys(t.category for t in TEST_CASES))
|
|
1554
|
+
lines.extend(
|
|
1555
|
+
[
|
|
1556
|
+
"",
|
|
1557
|
+
"## Per-category accuracy (avg, per profile)",
|
|
1558
|
+
"",
|
|
1559
|
+
"| Profile | " + " | ".join(category_order) + " |",
|
|
1560
|
+
"|---|" + "---:|" * len(category_order),
|
|
1561
|
+
]
|
|
1562
|
+
)
|
|
1563
|
+
for p in results:
|
|
1564
|
+
cs = category_summary(p.tests)
|
|
1565
|
+
cells = [f"`{p.profile}`"]
|
|
1566
|
+
for cat in category_order:
|
|
1567
|
+
s = cs.get(cat, {"avg_accuracy": 0.0, "n": 0})
|
|
1568
|
+
cells.append(f"{s['avg_accuracy']:.2f}" if s.get("n", 0) else "-")
|
|
1569
|
+
lines.append("| " + " | ".join(cells) + " |")
|
|
1570
|
+
|
|
1571
|
+
lines.append("")
|
|
1572
|
+
for p in results:
|
|
1573
|
+
lines.append(f"## `{p.profile}`")
|
|
1574
|
+
lines.append("")
|
|
1575
|
+
lines.append(f"- Model: `{p.model_id}` ({p.backend}) - recommended_for_8gb_vram: `{p.recommended_for_8gb_vram}`")
|
|
1576
|
+
lines.append(f"- VRAM: before {p.vram_before_mb} MB, peak {p.vram_peak_mb} MB, after {p.vram_after_mb} MB")
|
|
1577
|
+
if p.notes:
|
|
1578
|
+
for note in p.notes:
|
|
1579
|
+
lines.append(f"- Note: {note}")
|
|
1580
|
+
lines.append("")
|
|
1581
|
+
cs = category_summary(p.tests)
|
|
1582
|
+
lines.append("| Category | n | passed | avg latency | avg accuracy |")
|
|
1583
|
+
lines.append("|---|---:|---:|---:|---:|")
|
|
1584
|
+
for cat, s in sorted(cs.items()):
|
|
1585
|
+
lines.append(f"| {cat} | {int(s['n'])} | {int(s['passed'])} | {s['avg_elapsed_seconds']}s | {s['avg_accuracy']} |")
|
|
1586
|
+
lines.append("")
|
|
1587
|
+
lines.append("| Test | Result | Latency | tok/s | Accuracy | Response preview |")
|
|
1588
|
+
lines.append("|---|---|---:|---:|---:|---|")
|
|
1589
|
+
for t in p.tests:
|
|
1590
|
+
status = "PASS" if t.success else "FAIL"
|
|
1591
|
+
tps = f"{t.approx_tokens_per_second}" if t.approx_tokens_per_second else "-"
|
|
1592
|
+
preview = (t.response_preview or t.error or "").replace("|", "\\|")[:160]
|
|
1593
|
+
lines.append(f"| {t.name} | {status} | {t.elapsed_seconds}s | {tps} | {t.accuracy} | {preview} |")
|
|
1594
|
+
lines.append("")
|
|
1595
|
+
|
|
1596
|
+
md_path.write_text("\n".join(lines), encoding="utf-8")
|
|
1597
|
+
return json_path, md_path
|
|
1598
|
+
|
|
1599
|
+
|
|
1600
|
+
def gpu_info() -> Dict[str, Any]:
|
|
1601
|
+
try:
|
|
1602
|
+
output = subprocess.check_output(
|
|
1603
|
+
["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"],
|
|
1604
|
+
stderr=subprocess.STDOUT,
|
|
1605
|
+
timeout=5,
|
|
1606
|
+
).decode("ascii", errors="ignore").strip().splitlines()[0]
|
|
1607
|
+
name, total = [x.strip() for x in output.split(",", 1)]
|
|
1608
|
+
return {"gpu_name": name, "gpu_total_mb": int(total)}
|
|
1609
|
+
except Exception:
|
|
1610
|
+
return {"gpu_name": None, "gpu_total_mb": None}
|
|
1611
|
+
|
|
1612
|
+
|
|
1613
|
+
# ---------------------------------------------------------------------------
|
|
1614
|
+
# Question-set grader registry (safe, JSON-driven graders for uploaded sets)
|
|
1615
|
+
# ---------------------------------------------------------------------------
|
|
1616
|
+
|
|
1617
|
+
GRADER_TYPES = [
|
|
1618
|
+
"contains_all",
|
|
1619
|
+
"json_array_min_len",
|
|
1620
|
+
"number_within",
|
|
1621
|
+
"exact_match",
|
|
1622
|
+
"classification_set",
|
|
1623
|
+
"json_keys_present",
|
|
1624
|
+
]
|
|
1625
|
+
|
|
1626
|
+
|
|
1627
|
+
def build_grader(spec: Any) -> Callable[[str], float]:
|
|
1628
|
+
"""Build a grader callable from a JSON spec. Raises ValueError on a bad spec.
|
|
1629
|
+
|
|
1630
|
+
Keeps uploaded question sets safe JSON (no arbitrary code): each grader is
|
|
1631
|
+
selected by `type` from a fixed registry and reuses the helpers already in
|
|
1632
|
+
this module (`_extract_json`, `_grade_number`).
|
|
1633
|
+
"""
|
|
1634
|
+
if not isinstance(spec, dict):
|
|
1635
|
+
raise ValueError("grader must be an object with a 'type' field")
|
|
1636
|
+
gtype = spec.get("type")
|
|
1637
|
+
|
|
1638
|
+
if gtype == "contains_all":
|
|
1639
|
+
keywords = spec.get("keywords")
|
|
1640
|
+
if not isinstance(keywords, list) or not keywords:
|
|
1641
|
+
raise ValueError("contains_all requires a non-empty 'keywords' list")
|
|
1642
|
+
case_sensitive = bool(spec.get("case_sensitive", False))
|
|
1643
|
+
needles = [str(k) if case_sensitive else str(k).lower() for k in keywords]
|
|
1644
|
+
|
|
1645
|
+
def grade_contains(text: str) -> float:
|
|
1646
|
+
hay = text if case_sensitive else text.lower()
|
|
1647
|
+
hits = sum(1 for needle in needles if needle in hay)
|
|
1648
|
+
return hits / len(needles)
|
|
1649
|
+
|
|
1650
|
+
return grade_contains
|
|
1651
|
+
|
|
1652
|
+
if gtype == "json_array_min_len":
|
|
1653
|
+
minimum = spec.get("min", 1)
|
|
1654
|
+
if not isinstance(minimum, int) or isinstance(minimum, bool) or minimum < 0:
|
|
1655
|
+
raise ValueError("json_array_min_len requires a non-negative integer 'min'")
|
|
1656
|
+
|
|
1657
|
+
def grade_array(text: str) -> float:
|
|
1658
|
+
data = _extract_json(text)
|
|
1659
|
+
return 1.0 if isinstance(data, list) and len(data) >= minimum else 0.0
|
|
1660
|
+
|
|
1661
|
+
return grade_array
|
|
1662
|
+
|
|
1663
|
+
if gtype == "number_within":
|
|
1664
|
+
if "expected" not in spec:
|
|
1665
|
+
raise ValueError("number_within requires 'expected'")
|
|
1666
|
+
try:
|
|
1667
|
+
expected = float(spec["expected"])
|
|
1668
|
+
tolerance = float(spec.get("tolerance", 0.5))
|
|
1669
|
+
except (TypeError, ValueError):
|
|
1670
|
+
raise ValueError("number_within 'expected'/'tolerance' must be numbers")
|
|
1671
|
+
return _grade_number(expected, tolerance)
|
|
1672
|
+
|
|
1673
|
+
if gtype == "exact_match":
|
|
1674
|
+
if "expected" not in spec:
|
|
1675
|
+
raise ValueError("exact_match requires 'expected'")
|
|
1676
|
+
expected_str = str(spec["expected"])
|
|
1677
|
+
case_sensitive = bool(spec.get("case_sensitive", False))
|
|
1678
|
+
|
|
1679
|
+
def grade_exact(text: str) -> float:
|
|
1680
|
+
got, want = text.strip(), expected_str.strip()
|
|
1681
|
+
if not case_sensitive:
|
|
1682
|
+
got, want = got.lower(), want.lower()
|
|
1683
|
+
return 1.0 if got == want else 0.0
|
|
1684
|
+
|
|
1685
|
+
return grade_exact
|
|
1686
|
+
|
|
1687
|
+
if gtype == "classification_set":
|
|
1688
|
+
expected = spec.get("expected")
|
|
1689
|
+
if not isinstance(expected, list):
|
|
1690
|
+
raise ValueError("classification_set requires an 'expected' list")
|
|
1691
|
+
expected_set = {str(x).strip().lower() for x in expected}
|
|
1692
|
+
|
|
1693
|
+
def grade_set(text: str) -> float:
|
|
1694
|
+
data = _extract_json(text)
|
|
1695
|
+
if isinstance(data, list):
|
|
1696
|
+
got = {str(x).strip().lower() for x in data}
|
|
1697
|
+
else:
|
|
1698
|
+
got = {p.strip().lower() for p in re.split(r"[,\n]", text) if p.strip()}
|
|
1699
|
+
return 1.0 if got == expected_set else 0.0
|
|
1700
|
+
|
|
1701
|
+
return grade_set
|
|
1702
|
+
|
|
1703
|
+
if gtype == "json_keys_present":
|
|
1704
|
+
required = spec.get("required")
|
|
1705
|
+
if not isinstance(required, list) or not required:
|
|
1706
|
+
raise ValueError("json_keys_present requires a non-empty 'required' list")
|
|
1707
|
+
if len(required) > 100:
|
|
1708
|
+
raise ValueError("json_keys_present accepts at most 100 required keys")
|
|
1709
|
+
if any(not isinstance(key, str) or not key.strip() or len(key) > 256 for key in required):
|
|
1710
|
+
raise ValueError("json_keys_present keys must be non-empty strings up to 256 characters")
|
|
1711
|
+
if len(set(required)) != len(required):
|
|
1712
|
+
raise ValueError("json_keys_present 'required' keys must be unique")
|
|
1713
|
+
allow_extra = spec.get("allow_extra", True)
|
|
1714
|
+
if not isinstance(allow_extra, bool):
|
|
1715
|
+
raise ValueError("json_keys_present 'allow_extra' must be a boolean")
|
|
1716
|
+
required_keys = set(required)
|
|
1717
|
+
|
|
1718
|
+
def grade_keys(text: str) -> float:
|
|
1719
|
+
data = _extract_json(text)
|
|
1720
|
+
if not isinstance(data, dict):
|
|
1721
|
+
return 0.0
|
|
1722
|
+
actual_keys = set(data)
|
|
1723
|
+
if not required_keys.issubset(actual_keys):
|
|
1724
|
+
return 0.0
|
|
1725
|
+
if not allow_extra and actual_keys != required_keys:
|
|
1726
|
+
return 0.0
|
|
1727
|
+
return 1.0
|
|
1728
|
+
|
|
1729
|
+
return grade_keys
|
|
1730
|
+
|
|
1731
|
+
if gtype == "builtin":
|
|
1732
|
+
name = spec.get("name")
|
|
1733
|
+
if not isinstance(name, str) or name not in BUILTIN_GRADER_BY_NAME:
|
|
1734
|
+
raise ValueError("builtin grader requires a valid built-in test name")
|
|
1735
|
+
return BUILTIN_GRADER_BY_NAME[name]
|
|
1736
|
+
|
|
1737
|
+
raise ValueError(f"unknown grader type '{gtype}' (allowed: {', '.join(GRADER_TYPES)})")
|
|
1738
|
+
|
|
1739
|
+
|
|
1740
|
+
def validate_question_set(payload: Any) -> Dict[str, Any]:
|
|
1741
|
+
"""Validate an uploaded question set against the schema and grader registry.
|
|
1742
|
+
|
|
1743
|
+
Returns a structured report with per-row errors; never raises.
|
|
1744
|
+
"""
|
|
1745
|
+
errors: List[Dict[str, Any]] = []
|
|
1746
|
+
if not isinstance(payload, dict):
|
|
1747
|
+
return {
|
|
1748
|
+
"success": True,
|
|
1749
|
+
"valid": False,
|
|
1750
|
+
"question_count": 0,
|
|
1751
|
+
"errors": [{"index": -1, "error": "top-level must be a JSON object"}],
|
|
1752
|
+
"grader_types": GRADER_TYPES,
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
questions = payload.get("questions")
|
|
1756
|
+
if not isinstance(questions, list) or not questions:
|
|
1757
|
+
errors.append({"index": -1, "error": "'questions' must be a non-empty list"})
|
|
1758
|
+
questions = []
|
|
1759
|
+
|
|
1760
|
+
seen_names: set = set()
|
|
1761
|
+
for index, question in enumerate(questions):
|
|
1762
|
+
if not isinstance(question, dict):
|
|
1763
|
+
errors.append({"index": index, "error": "question must be an object"})
|
|
1764
|
+
continue
|
|
1765
|
+
name = question.get("name")
|
|
1766
|
+
if not name or not isinstance(name, str):
|
|
1767
|
+
errors.append({"index": index, "error": "missing or invalid 'name'"})
|
|
1768
|
+
elif name in seen_names:
|
|
1769
|
+
errors.append({"index": index, "name": name, "error": "duplicate 'name'"})
|
|
1770
|
+
else:
|
|
1771
|
+
seen_names.add(name)
|
|
1772
|
+
if not question.get("category") or not isinstance(question.get("category"), str):
|
|
1773
|
+
errors.append({"index": index, "name": question.get("name"), "error": "missing or invalid 'category'"})
|
|
1774
|
+
if not question.get("prompt") or not isinstance(question.get("prompt"), str):
|
|
1775
|
+
errors.append({"index": index, "name": question.get("name"), "error": "missing or invalid 'prompt'"})
|
|
1776
|
+
mot = question.get("max_output_tokens", 256)
|
|
1777
|
+
if not isinstance(mot, int) or isinstance(mot, bool) or mot <= 0:
|
|
1778
|
+
errors.append({"index": index, "name": question.get("name"), "error": "'max_output_tokens' must be a positive integer"})
|
|
1779
|
+
try:
|
|
1780
|
+
build_grader(question.get("grader"))
|
|
1781
|
+
except ValueError as exc:
|
|
1782
|
+
errors.append({"index": index, "name": question.get("name"), "error": f"grader: {exc}"})
|
|
1783
|
+
|
|
1784
|
+
return {
|
|
1785
|
+
"success": True,
|
|
1786
|
+
"valid": len(errors) == 0,
|
|
1787
|
+
"question_count": len(questions),
|
|
1788
|
+
"errors": errors,
|
|
1789
|
+
"grader_types": GRADER_TYPES,
|
|
1790
|
+
}
|
|
1791
|
+
|
|
1792
|
+
|
|
1793
|
+
def build_test_cases(payload: Dict[str, Any]) -> List[TestCase]:
|
|
1794
|
+
"""Turn a validated question set into TestCase objects for a run."""
|
|
1795
|
+
cases: List[TestCase] = []
|
|
1796
|
+
for question in payload.get("questions", []):
|
|
1797
|
+
cases.append(
|
|
1798
|
+
TestCase(
|
|
1799
|
+
name=str(question["name"]),
|
|
1800
|
+
category=str(question.get("category", "custom")),
|
|
1801
|
+
prompt=str(question["prompt"]),
|
|
1802
|
+
grader=build_grader(question.get("grader")),
|
|
1803
|
+
grader_explainer=str(question.get("grader_explainer", "")),
|
|
1804
|
+
max_output_tokens=int(question.get("max_output_tokens", 256)),
|
|
1805
|
+
)
|
|
1806
|
+
)
|
|
1807
|
+
return cases
|
|
1808
|
+
|
|
1809
|
+
|
|
1810
|
+
GRADER_TYPES = tuple(dict.fromkeys([*GRADER_TYPES, "builtin"]))
|
|
1811
|
+
|
|
1812
|
+
|
|
1813
|
+
BUILTIN_DATA_QUESTIONS: List[Dict[str, Any]] = [
|
|
1814
|
+
{
|
|
1815
|
+
"name": "plan_incident_triage_array",
|
|
1816
|
+
"category": "planning",
|
|
1817
|
+
"prompt": (
|
|
1818
|
+
"A checkout API has a sudden error-rate spike. Return a JSON array containing at least four "
|
|
1819
|
+
"ordered triage steps, starting with the cheapest observation. JSON only."
|
|
1820
|
+
),
|
|
1821
|
+
"max_output_tokens": 300,
|
|
1822
|
+
"grader": {"type": "json_array_min_len", "min": 4},
|
|
1823
|
+
"grader_explainer": "Valid JSON array with at least four ordered triage steps.",
|
|
1824
|
+
},
|
|
1825
|
+
{
|
|
1826
|
+
"name": "cls_incident_signals",
|
|
1827
|
+
"category": "classification",
|
|
1828
|
+
"prompt": (
|
|
1829
|
+
"A service returns HTTP 503 for 30% of requests and its p95 response time doubled. "
|
|
1830
|
+
"Choose every matching label from [availability, latency, correctness, security]. "
|
|
1831
|
+
"Return a JSON array of labels only."
|
|
1832
|
+
),
|
|
1833
|
+
"max_output_tokens": 40,
|
|
1834
|
+
"grader": {"type": "classification_set", "expected": ["availability", "latency"]},
|
|
1835
|
+
"grader_explainer": "Label set is exactly availability and latency.",
|
|
1836
|
+
},
|
|
1837
|
+
{
|
|
1838
|
+
"name": "code_merge_sort_complexity",
|
|
1839
|
+
"category": "code",
|
|
1840
|
+
"prompt": (
|
|
1841
|
+
"State the worst-case time complexity and auxiliary space complexity of merge sort on an array. "
|
|
1842
|
+
"Use Big O notation and include no implementation."
|
|
1843
|
+
),
|
|
1844
|
+
"max_output_tokens": 80,
|
|
1845
|
+
"grader": {"type": "contains_all", "keywords": ["O(n log n)", "O(n)"]},
|
|
1846
|
+
"grader_explainer": "Contains O(n log n) time and O(n) auxiliary space.",
|
|
1847
|
+
},
|
|
1848
|
+
{
|
|
1849
|
+
"name": "math_percentage_increase",
|
|
1850
|
+
"category": "math",
|
|
1851
|
+
"prompt": "A latency value rises from 80 ms to 100 ms. What is the percentage increase? Reply with the number only.",
|
|
1852
|
+
"max_output_tokens": 32,
|
|
1853
|
+
"grader": {"type": "number_within", "expected": 25, "tolerance": 0.1},
|
|
1854
|
+
"grader_explainer": "Answer is within 0.1 of 25.",
|
|
1855
|
+
},
|
|
1856
|
+
{
|
|
1857
|
+
"name": "json_release_metadata_keys",
|
|
1858
|
+
"category": "structured",
|
|
1859
|
+
"prompt": (
|
|
1860
|
+
"Return a JSON object describing a software release with keys version, date, changes, and breaking. "
|
|
1861
|
+
"Use an array for changes and a boolean for breaking. JSON only."
|
|
1862
|
+
),
|
|
1863
|
+
"max_output_tokens": 180,
|
|
1864
|
+
"grader": {
|
|
1865
|
+
"type": "json_keys_present",
|
|
1866
|
+
"required": ["version", "date", "changes", "breaking"],
|
|
1867
|
+
},
|
|
1868
|
+
"grader_explainer": "JSON object contains version, date, changes, and breaking keys.",
|
|
1869
|
+
},
|
|
1870
|
+
]
|
|
1871
|
+
|
|
1872
|
+
|
|
1873
|
+
BUILTIN_GRADER_BY_NAME = {test.name: test.grader for test in TEST_CASES}
|
|
1874
|
+
BUILTIN_GRADER_BY_NAME.update(
|
|
1875
|
+
{question["name"]: build_grader(question["grader"]) for question in BUILTIN_DATA_QUESTIONS}
|
|
1876
|
+
)
|
|
1877
|
+
|
|
1878
|
+
|
|
1879
|
+
BUILTIN_QUESTION_SET: Dict[str, Any] = {
|
|
1880
|
+
"version": 1,
|
|
1881
|
+
"questions": [
|
|
1882
|
+
{
|
|
1883
|
+
"name": test.name,
|
|
1884
|
+
"category": test.category,
|
|
1885
|
+
"prompt": test.prompt,
|
|
1886
|
+
"max_output_tokens": test.max_output_tokens,
|
|
1887
|
+
"grader": {"type": "builtin", "name": test.name},
|
|
1888
|
+
"grader_explainer": test.grader_explainer,
|
|
1889
|
+
}
|
|
1890
|
+
for test in TEST_CASES
|
|
1891
|
+
] + [dict(question) for question in BUILTIN_DATA_QUESTIONS],
|
|
1892
|
+
}
|
|
1893
|
+
|
|
1894
|
+
|
|
1895
|
+
def builtin_test_cases() -> List[TestCase]:
|
|
1896
|
+
"""Return a fresh set of every built-in benchmark case.
|
|
1897
|
+
|
|
1898
|
+
Building new objects for each caller keeps request-specific token limits
|
|
1899
|
+
and filtering from mutating the shared grader definitions.
|
|
1900
|
+
"""
|
|
1901
|
+
return build_test_cases(BUILTIN_QUESTION_SET)
|
|
1902
|
+
|
|
1903
|
+
|
|
1904
|
+
EXAMPLE_QUESTION_SET: Dict[str, Any] = {
|
|
1905
|
+
"version": 1,
|
|
1906
|
+
"questions": [
|
|
1907
|
+
{
|
|
1908
|
+
"name": "planning_triage_basic",
|
|
1909
|
+
"category": "planning",
|
|
1910
|
+
"prompt": "List 3 first steps to triage a service outage. Return a JSON array of strings.",
|
|
1911
|
+
"max_output_tokens": 512,
|
|
1912
|
+
"grader": {"type": "json_array_min_len", "min": 3},
|
|
1913
|
+
"grader_explainer": "Passes if the model returns a JSON array with at least 3 steps.",
|
|
1914
|
+
},
|
|
1915
|
+
{
|
|
1916
|
+
"name": "math_tolerance",
|
|
1917
|
+
"category": "reasoning",
|
|
1918
|
+
"prompt": "What is 12.5% of 240? Answer with the number only.",
|
|
1919
|
+
"max_output_tokens": 32,
|
|
1920
|
+
"grader": {"type": "number_within", "expected": 30, "tolerance": 0.5},
|
|
1921
|
+
"grader_explainer": "Passes if the parsed number is within 0.5 of 30.",
|
|
1922
|
+
},
|
|
1923
|
+
],
|
|
1924
|
+
}
|
|
1925
|
+
|
|
1926
|
+
|
|
1927
|
+
def main() -> int:
|
|
1928
|
+
parser = argparse.ArgumentParser(description="LocalDeploy capability benchmark (v2).")
|
|
1929
|
+
parser.add_argument("--profile", action="append", help="Limit to one or more named profiles (repeatable).")
|
|
1930
|
+
parser.add_argument("--max-output-tokens", type=int, help="Bump all non-classification tests to this cap (if higher).")
|
|
1931
|
+
parser.add_argument("--timeout", type=int, default=240, help="Per-request timeout seconds.")
|
|
1932
|
+
parser.add_argument("--skip-categories", help="Comma-separated categories to skip.")
|
|
1933
|
+
parser.add_argument("--include-categories", help="Comma-separated categories to ONLY include (whitelist).")
|
|
1934
|
+
args = parser.parse_args()
|
|
1935
|
+
|
|
1936
|
+
config = load_config()
|
|
1937
|
+
profiles = config.get("profiles", {})
|
|
1938
|
+
selected = args.profile or [name for name, p in profiles.items() if p.get("enabled", False)]
|
|
1939
|
+
selected = [name for name in selected if name in profiles]
|
|
1940
|
+
if not selected:
|
|
1941
|
+
print("No profiles selected. Enable some in config.json or pass --profile NAME.")
|
|
1942
|
+
return 2
|
|
1943
|
+
|
|
1944
|
+
skip = {c.strip() for c in (args.skip_categories or "").split(",") if c.strip()}
|
|
1945
|
+
include = {c.strip() for c in (args.include_categories or "").split(",") if c.strip()}
|
|
1946
|
+
tests = [
|
|
1947
|
+
t
|
|
1948
|
+
for t in builtin_test_cases()
|
|
1949
|
+
if t.category not in skip and (not include or t.category in include)
|
|
1950
|
+
]
|
|
1951
|
+
if args.max_output_tokens:
|
|
1952
|
+
for t in tests:
|
|
1953
|
+
if t.category != "classification":
|
|
1954
|
+
t.max_output_tokens = max(t.max_output_tokens, args.max_output_tokens)
|
|
1955
|
+
|
|
1956
|
+
base_url = api_base_url()
|
|
1957
|
+
try:
|
|
1958
|
+
health = requests.get(f"{base_url}/health", headers=api_auth_headers(), timeout=10).json()
|
|
1959
|
+
except Exception as exc:
|
|
1960
|
+
print(f"API server is not reachable at {base_url}: {exc}")
|
|
1961
|
+
print("Start it with: .\\scripts\\start.ps1 (or set $env:API_PORT='8011' first)")
|
|
1962
|
+
return 2
|
|
1963
|
+
if not health.get("server"):
|
|
1964
|
+
print(f"API server at {base_url} returned unexpected health: {health}")
|
|
1965
|
+
return 2
|
|
1966
|
+
|
|
1967
|
+
print(f"Profiles selected ({len(selected)}): {', '.join(selected)}")
|
|
1968
|
+
print(f"Tests per profile: {len(tests)}")
|
|
1969
|
+
print(f"Timeout: {args.timeout}s per request")
|
|
1970
|
+
|
|
1971
|
+
info = gpu_info()
|
|
1972
|
+
run_meta = {
|
|
1973
|
+
"base_url": base_url,
|
|
1974
|
+
"default_profile": health.get("default_profile"),
|
|
1975
|
+
"test_count": len(tests),
|
|
1976
|
+
"tests": [t.name for t in tests],
|
|
1977
|
+
"categories": sorted({t.category for t in tests}),
|
|
1978
|
+
**info,
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
started = time.perf_counter()
|
|
1982
|
+
results: List[ProfileResult] = []
|
|
1983
|
+
for name in selected:
|
|
1984
|
+
result = run_profile(base_url, name, profiles[name], tests, args.timeout)
|
|
1985
|
+
results.append(result)
|
|
1986
|
+
|
|
1987
|
+
elapsed = time.perf_counter() - started
|
|
1988
|
+
run_meta["elapsed_seconds"] = round(elapsed, 2)
|
|
1989
|
+
|
|
1990
|
+
json_path, md_path = write_reports(results, run_meta)
|
|
1991
|
+
print(f"\nReports written:\n {json_path}\n {md_path}")
|
|
1992
|
+
return 0
|
|
1993
|
+
|
|
1994
|
+
|
|
1995
|
+
if __name__ == "__main__":
|
|
1996
|
+
sys.exit(main())
|