u1s1-cli 0.16.6 → 0.17.0

package/dist/bench.js

@@ -6,6 +6,24 @@ import { CUSTOM_ENDPOINTS, loadConfig, MODELS, PROVIDER_ID, } from "./config.js"
 import { loadCustomEndpoints } from "./api.js";
 // ─── 工具 ───
 const BENCH_DIR = join(homedir(), ".u1s1", "bench");
+/** 题目未声明 maxScore 时的默认满分 */
+const DEFAULT_MAX_SCORE = 10;
+/** 报告里单条回答最多保留的字符数 */
+const MAX_ANSWER_CHARS = 2000;
+/** compare 子命令里单条回答的截断长度 */
+const MAX_COMPARE_CHARS = 200;
+/** 去掉回答里的 markdown 代码块围栏,便于 JSON 解析 */
+function stripCodeFences(text) {
+    return text.replace(/```json\s*\n?/gi, "").replace(/\n?```/g, "").trim();
+}
+/** 自定义端点的模型 id 常带 org/ 前缀,取斜杠后的短 id */
+function shortModelId(id) {
+    return id.includes("/") ? id.split("/")[1] : id;
+}
+/** 结果里模型的显示名:内置模型不带前缀,自定义端点带 provider 前缀 */
+function resultModelLabel(r) {
+    return r.providerName === PROVIDER_ID ? r.modelId : `${r.providerName}:${r.modelId}`;
+}
 /** 项目 bench 数据目录 */
 function benchDataDir() {
     const distDir = dirname(fileURLToPath(import.meta.url));
@@ -118,24 +136,19 @@ function scoreResponse(response, checks, maxScore) {
 function runCheck(response, check) {
     const val = check.value;
     switch (check.type) {
-        case "contains": {
-            const needle = String(val);
-            const text = check.ignoreCase ? response.toLowerCase() : response;
-            const search = check.ignoreCase ? needle.toLowerCase() : needle;
-            return text.includes(search);
-        }
+        case "contains":
         case "not_contains": {
             const needle = String(val);
             const text = check.ignoreCase ? response.toLowerCase() : response;
             const search = check.ignoreCase ? needle.toLowerCase() : needle;
-            return !text.includes(search);
+            const found = text.includes(search);
+            return check.type === "contains" ? found : !found;
         }
         case "contains_code_block":
             return /```[\s\S]*?```/.test(response) || /`[^`]+`/.test(response);
         case "parse_json": {
             try {
-                const stripped = response.replace(/```json\s*\n?/gi, "").replace(/\n?```/g, "").trim();
-                JSON.parse(stripped);
+                JSON.parse(stripCodeFences(response));
                 return true;
             }
             catch {
@@ -168,8 +181,7 @@ function runCheck(response, check) {
         }
         case "has_json_key": {
             try {
-                const stripped = response.replace(/```json\s*\n?/gi, "").replace(/\n?```/g, "").trim();
-                const obj = JSON.parse(stripped);
+                const obj = JSON.parse(stripCodeFences(response));
                 return obj[String(val)] !== undefined;
             }
             catch {
@@ -303,6 +315,44 @@ function listRuns() {
     }
     return runs;
 }
+/** 把一批结果按模型聚合成分数/延迟/花费/错误数 */
+function summarizeByModel(results, keyOf) {
+    const map = new Map();
+    for (const r of results) {
+        const key = keyOf(r);
+        const s = map.get(key) ?? { totalScore: 0, totalMax: 0, totalLatency: 0, totalCost: 0, errors: 0, total: 0, okTokensOut: 0, okLatency: 0 };
+        s.totalScore += r.score ?? 0;
+        s.totalMax += r.maxScore ?? DEFAULT_MAX_SCORE;
+        s.totalLatency += r.latencyMs;
+        s.totalCost += r.costUsd;
+        if (r.error) {
+            s.errors++;
+        }
+        else {
+            s.okTokensOut += r.tokensOut;
+            s.okLatency += r.latencyMs;
+        }
+        s.total++;
+        map.set(key, s);
+    }
+    return map;
+}
+/** 按总分从高到低排序 */
+function sortByScore(entries) {
+    return entries.sort((a, b) => b[1].totalScore - a[1].totalScore);
+}
+/** 得分率百分数字符串(一位小数) */
+function scoreRate(s) {
+    return s.totalMax > 0 ? ((s.totalScore / s.totalMax) * 100).toFixed(1) : "0.0";
+}
+/** 错误率百分数字符串(整数) */
+function errorRate(s) {
+    return ((s.errors / s.total) * 100).toFixed(0);
+}
+/** 推理速度:成功题的输出 token / 耗时(tok/s),没有成功题返回 — */
+function tokPerSec(s) {
+    return s.okLatency > 0 ? (s.okTokensOut / s.okLatency * 1000).toFixed(1) : "—";
+}
 // ─── 报告生成 ───
 function generateReport(run) {
     const lines = [];
@@ -328,7 +378,7 @@ function generateReport(run) {
         }
         for (const [qId, qResults] of byQuestion) {
             const prompt = qResults[0].prompt;
-            const maxScore = qResults[0].maxScore ?? 10;
+            const maxScore = qResults[0].maxScore ?? DEFAULT_MAX_SCORE;
             lines.push(`### ${qId}`);
             lines.push(`> ${prompt}`);
             lines.push("");
@@ -336,10 +386,9 @@ function generateReport(run) {
             const header = ["模型", `得分/${maxScore}`, "延迟", "输出tok", "状态"];
             const rows = [];
             for (const r of qResults) {
-                const modelLabel = r.providerName === PROVIDER_ID ? r.modelId : `${r.providerName}:${r.modelId}`;
                 const scoreStr = r.score !== undefined ? `${r.score}/${maxScore}` : "—";
                 rows.push([
-                    modelLabel,
+                    resultModelLabel(r),
                     scoreStr,
                     formatDuration(r.latencyMs),
                     String(r.tokensOut),
@@ -356,8 +405,7 @@ function generateReport(run) {
                 lines.push("<details><summary>评分明细</summary>");
                 lines.push("");
                 for (const r of scored) {
-                    const modelLabel = r.providerName === PROVIDER_ID ? r.modelId : `${r.providerName}:${r.modelId}`;
-                    lines.push(`**${modelLabel}:** ${r.score}/${maxScore}`);
+                    lines.push(`**${resultModelLabel(r)}:** ${r.score}/${maxScore}`);
                     for (const d of r.scoreDetails) {
                         const icon = d.passed ? "✅" : "❌";
                         lines.push(`- ${icon} ${d.check} (+${d.earned}/${d.points})`);
@@ -369,8 +417,12 @@ function generateReport(run) {
             }
             // 各模型回答
             for (const r of qResults) {
-                const modelLabel = r.providerName === PROVIDER_ID ? r.modelId : `${r.providerName}:${r.modelId}`;
-                lines.push(`<details><summary>${modelLabel} 的回答</summary>`);
+                lines.push(`<details><summary>${resultModelLabel(r)} 的回答</summary>`);
+                lines.push("");
+                lines.push("```");
+                lines.push(r.response.slice(0, MAX_ANSWER_CHARS));
+                if (r.response.length > MAX_ANSWER_CHARS)
+                    lines.push("...(截断)");
                 lines.push("");
                 lines.push("```");
                 lines.push(r.response.slice(0, 2000));
@@ -385,33 +437,17 @@ function generateReport(run) {
     // 汇总 — 带分数
     lines.push("## 汇总");
     lines.push("");
-    const modelSummary = new Map();
-    for (const r of run.results) {
-        const key = r.providerName === PROVIDER_ID ? r.modelId : `${r.providerName}:${r.modelId}`;
-        if (!modelSummary.has(key))
-            modelSummary.set(key, { totalScore: 0, totalMax: 0, totalLatency: 0, totalCost: 0, errors: 0, total: 0 });
-        const s = modelSummary.get(key);
-        s.totalScore += r.score ?? 0;
-        s.totalMax += r.maxScore ?? 10;
-        s.totalLatency += r.latencyMs;
-        s.totalCost += r.costUsd;
-        if (r.error)
-            s.errors++;
-        s.total++;
-    }
-    const header = ["模型", "总分", "得分率", "平均延迟", "总花费", "错误率"];
+    const header = ["模型", "总分", "得分率", "平均延迟", "速度(tok/s)", "总花费", "错误率"];
     const rows = [];
-    // 按总分排序
-    const sorted = [...modelSummary.entries()].sort((a, b) => b[1].totalScore - a[1].totalScore);
-    for (const [model, s] of sorted) {
-        const pct = s.totalMax > 0 ? ((s.totalScore / s.totalMax) * 100).toFixed(1) : "0.0";
+    for (const [model, s] of sortByScore([...summarizeByModel(run.results, resultModelLabel)])) {
         rows.push([
             model,
             `${s.totalScore}/${s.totalMax}`,
-            `${pct}%`,
+            `${scoreRate(s)}%`,
             formatDuration(s.totalLatency / s.total),
+            tokPerSec(s),
             costStr(s.totalCost),
-            s.errors > 0 ? `${((s.errors / s.total) * 100).toFixed(0)}%` : "0%",
+            s.errors > 0 ? `${errorRate(s)}%` : "0%",
         ]);
     }
     lines.push("```");
@@ -420,243 +456,256 @@ function generateReport(run) {
     return lines.join("\n");
 }
 // ─── 主命令 ───
-export async function benchCommand(args) {
-    const sub = args[0];
-    if (!sub || sub === "help" || sub === "--help") {
-        console.log("");
-        console.log("  u1s1 bench — 模型质量基准测试（带自动评分）");
-        console.log("");
-        console.log("  用法:");
-        console.log("    u1s1 bench list                   列出可用的测试套件");
-        console.log("    u1s1 bench run <suite> [models..]  跑基准测试");
-        console.log("    u1s1 bench report <run-id>         查看报告");
-        console.log("    u1s1 bench compare <run-id1> <run-id2>  对比两次运行");
-        console.log("");
-        console.log("  示例:");
-        console.log("    u1s1 bench run quick               用当前默认模型跑快速测试");
-        console.log("    u1s1 bench run full deepseek pro   用多个模型跑全面测试");
-        console.log("    u1s1 bench report latest           看最近一次报告");
-        console.log("");
-        return;
+/** 打印错误信息并以退出码 1 结束。永不返回。 */
+function failWithError(e) {
+    console.error(`  ${e instanceof Error ? e.message : e}`);
+    process.exit(1);
+}
+function printHelp() {
+    console.log("");
+    console.log("  u1s1 bench — 模型质量基准测试（带自动评分）");
+    console.log("");
+    console.log("  用法:");
+    console.log("    u1s1 bench list                   列出可用的测试套件");
+    console.log("    u1s1 bench run <suite> [models..]  跑基准测试(--delay 秒数:题目间隔,限流渠道用)");
+    console.log("    u1s1 bench report <run-id>         查看报告");
+    console.log("    u1s1 bench compare <run-id1> <run-id2>  对比两次运行");
+    console.log("");
+    console.log("  示例:");
+    console.log("    u1s1 bench run quick               用当前默认模型跑快速测试");
+    console.log("    u1s1 bench run full deepseek pro   用多个模型跑全面测试");
+    console.log("    u1s1 bench report latest           看最近一次报告");
+    console.log("");
+}
+function listSuitesCommand() {
+    const suites = listSuites();
+    console.log("");
+    console.log("  Bench 套件:");
+    for (const s of suites) {
+        const tag = s.builtin ? "(内置)" : "(自定义)";
+        console.log(`    ${s.name.padEnd(12)} ${s.description} ${tag}`);
     }
-    if (sub === "list") {
-        const suites = listSuites();
-        console.log("");
-        console.log("  Bench 套件:");
-        for (const s of suites) {
-            const tag = s.builtin ? "(内置)" : "(自定义)";
-            console.log(`    ${s.name.padEnd(12)} ${s.description} ${tag}`);
+    console.log("");
+    console.log("  自定义套件放 ~/.u1s1/bench/<name>.json,格式同内置 suite.json");
+    console.log("");
+}
+/** 把模型名解析成走 u1s1 默认端点的调用目标 */
+function u1s1Target(id, cfg) {
+    return { id, label: id, baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, source: "u1s1" };
+}
+/** 把用户输入的模型名解析成一个可调用的目标 */
+function resolveModelTarget(query, cfg) {
+    const lower = query.toLowerCase();
+    const m = MODELS.find((x) => x.id.toLowerCase() === lower || x.aliases.includes(lower));
+    if (m)
+        return u1s1Target(m.id, cfg);
+    for (const ep of CUSTOM_ENDPOINTS) {
+        const em = ep.models.find((x) => x.id.toLowerCase() === lower || x.aliases.includes(lower));
+        if (em) {
+            return { id: shortModelId(em.id), label: `${ep.name}:${em.id}`, baseUrl: ep.baseUrl, apiKey: ep.apiKey ?? "", source: "custom" };
         }
-        console.log("");
-        console.log("  自定义套件放 ~/.u1s1/bench/<name>.json,格式同内置 suite.json");
-        console.log("");
-        return;
     }
-    if (sub === "run") {
-        const suiteName = args[1] ?? "quick";
-        const modelQueries = args.slice(2);
-        const cfg = loadConfig();
-        if (!cfg.apiKey) {
-            console.error("  还没有登录,先 u1s1 login");
-            process.exit(1);
-        }
-        await loadCustomEndpoints(cfg);
-        let suite;
-        try {
-            suite = loadSuite(suiteName);
-        }
-        catch (e) {
-            console.error(`  ${e instanceof Error ? e.message : e}`);
-            process.exit(1);
-        }
-        let targets;
-        function resolveModelTarget(query) {
-            const lower = query.toLowerCase();
-            const m = MODELS.find((x) => x.id.toLowerCase() === lower || x.aliases.includes(lower));
-            if (m)
-                return { id: m.id, label: m.id, baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, source: "u1s1" };
-            for (const ep of CUSTOM_ENDPOINTS) {
-                const em = ep.models.find((x) => x.id.toLowerCase() === lower || x.aliases.includes(lower));
-                if (em) {
-                    const shortId = em.id.includes("/") ? em.id.split("/")[1] : em.id;
-                    return { id: shortId, label: `${ep.name}:${em.id}`, baseUrl: ep.baseUrl, apiKey: ep.apiKey ?? "", source: "custom" };
-                }
-            }
-            return { id: query, label: query, baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, source: "u1s1" };
-        }
-        if (modelQueries.length > 0) {
-            targets = modelQueries.map((q) => resolveModelTarget(q)).filter((t) => t !== null);
-        }
-        else {
-            const { resolvePreferredModel } = await import("./config.js");
-            const pref = resolvePreferredModel(cfg);
-            if (pref.provider === PROVIDER_ID) {
-                targets = [{ id: pref.id, label: pref.id, baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, source: "u1s1" }];
-            }
-            else {
-                const ep = CUSTOM_ENDPOINTS.find((e) => e.id === pref.provider);
-                if (ep) {
-                    const shortId = pref.id.includes("/") ? pref.id.split("/")[1] : pref.id;
-                    targets = [{ id: shortId, label: `${ep.name}:${pref.id}`, baseUrl: ep.baseUrl, apiKey: ep.apiKey ?? "", source: "custom" }];
-                }
-                else {
-                    targets = [{ id: pref.id, label: pref.id, baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, source: "u1s1" }];
-                }
-            }
-        }
-        console.log("");
-        console.log(`  📊 Bench: ${suite.name}`);
-        console.log(`  ${suite.description}`);
-        console.log(`  题目数: ${suite.questions.length}`);
-        console.log(`  模型: ${targets.map((t) => t.label).join(", ")}`);
-        console.log("");
-        const results = [];
-        const total = suite.questions.length * targets.length;
-        let done = 0;
-        for (const target of targets) {
-            for (const q of suite.questions) {
-                const label = `${target.label} / ${q.id}`;
-                process.stdout.write(`  [${++done}/${total}] ${label.padEnd(40)} `);
-                const resp = await callModel(target.baseUrl, target.apiKey, target.id, q.prompt);
-                const costUsd = estimateCost(target.id, resp.tokensIn, resp.tokensOut);
-                // 评分
-                let score;
-                let maxScore;
-                let scoreDetails;
-                if (!resp.error && q.checks && q.checks.length > 0) {
-                    const ms = q.maxScore ?? 10;
-                    const result = scoreResponse(resp.response, q.checks, ms);
-                    score = result.score;
-                    maxScore = ms;
-                    scoreDetails = result.details;
-                }
-                results.push({
-                    questionId: q.id,
-                    category: q.category,
-                    prompt: q.prompt,
-                    modelId: target.label,
-                    providerName: target.source === "u1s1" ? PROVIDER_ID : target.label,
-                    response: resp.response,
-                    latencyMs: resp.latencyMs,
-                    tokensIn: resp.tokensIn,
-                    tokensOut: resp.tokensOut,
-                    costUsd,
-                    error: resp.error,
-                    score,
-                    maxScore,
-                    scoreDetails,
-                });
-                if (resp.error) {
-                    console.log(`❌ ${resp.error.slice(0, 60)}`);
-                }
-                else if (score !== undefined) {
-                    const bar = scoreBar(score, maxScore);
-                    console.log(`✅ ${bar} ${score}/${maxScore} · ${formatDuration(resp.latencyMs)}`);
-                }
-                else {
-                    console.log(`✅ ${formatDuration(resp.latencyMs)} · ${resp.tokensIn}→${resp.tokensOut} tok`);
-                }
-            }
-        }
-        const runId = `bench-${suite.name}-${Date.now()}`;
-        const run = {
-            id: runId,
-            timestamp: new Date().toISOString(),
-            suiteName: suite.name,
-            models: [...new Set(results.map((r) => r.modelId))],
-            results,
-        };
-        saveRun(run);
-        // 简易汇总（带分数）
-        console.log("");
-        console.log("  📋 评分汇总:");
-        const byModel = new Map();
-        for (const r of results) {
-            const key = r.modelId;
-            if (!byModel.has(key))
-                byModel.set(key, { totalScore: 0, totalMax: 0, totalLatency: 0, totalCost: 0, ok: 0, fail: 0 });
-            const s = byModel.get(key);
-            s.totalScore += r.score ?? 0;
-            s.totalMax += r.maxScore ?? 10;
-            s.totalLatency += r.latencyMs;
-            s.totalCost += r.costUsd;
-            if (r.error)
-                s.fail++;
-            else
-                s.ok++;
-        }
-        const sumHeader = ["模型", "总分", "得分率", "平均延迟", "花费", "状态"];
-        const sumRows = [];
-        const sorted = [...byModel.entries()].sort((a, b) => b[1].totalScore - a[1].totalScore);
-        for (const [model, s] of sorted) {
-            const pct = s.totalMax > 0 ? ((s.totalScore / s.totalMax) * 100).toFixed(1) : "0.0";
-            sumRows.push([
-                model,
-                `${s.totalScore}/${s.totalMax}`,
-                `${pct}%`,
-                formatDuration(s.totalLatency / (s.ok + s.fail)),
-                costStr(s.totalCost),
-                s.fail > 0 ? `${((s.fail / (s.ok + s.fail)) * 100).toFixed(0)}%失败` : "✅全部通过",
-            ]);
-        }
-        console.log("  " + renderTable(sumHeader, sumRows));
-        console.log("");
-        console.log(`  查看完整报告: u1s1 bench report ${runId}`);
-        console.log("");
-        return;
+    // 兜底:当作原始模型 id 直接走默认端点
+    return u1s1Target(query, cfg);
+}
+/** 没指定模型时,用当前偏好的默认模型 */
+async function resolveDefaultTargets(cfg) {
+    const { resolvePreferredModel } = await import("./config.js");
+    const pref = resolvePreferredModel(cfg);
+    if (pref.provider === PROVIDER_ID)
+        return [u1s1Target(pref.id, cfg)];
+    const ep = CUSTOM_ENDPOINTS.find((e) => e.id === pref.provider);
+    if (!ep)
+        return [u1s1Target(pref.id, cfg)];
+    return [{
+            id: shortModelId(pref.id),
+            label: `${ep.name}:${pref.id}`,
+            baseUrl: ep.baseUrl,
+            apiKey: ep.apiKey ?? "",
+            source: "custom",
+        }];
+}
+/** 打印单题进度行(✅/❌ + 得分条或 token 统计) */
+function printQuestionOutcome(resp, score, maxScore) {
+    if (resp.error) {
+        console.log(`❌ ${resp.error.slice(0, 60)}`);
     }
-    if (sub === "report") {
-        const runId = args[1] ?? "latest";
-        let run;
-        try {
-            const id = runId === "latest" ? findLatestRun() : runId;
-            if (!id)
-                throw new Error("还没有运行记录");
-            run = loadRun(id);
-        }
-        catch (e) {
-            console.error(`  ${e instanceof Error ? e.message : e}`);
-            process.exit(1);
-        }
-        const report = generateReport(run);
-        const reportFile = join(BENCH_DIR, `${run.id}.md`);
-        writeFileSync(reportFile, report);
-        console.log(report);
-        console.log(`  报告已保存: ${reportFile}`);
-        return;
+    else if (score !== undefined && maxScore !== undefined) {
+        console.log(`✅ ${scoreBar(score, maxScore)} ${score}/${maxScore} · ${formatDuration(resp.latencyMs)}`);
     }
-    if (sub === "compare" && args[1] && args[2]) {
-        try {
-            const run1 = loadRun(args[1]);
-            const run2 = loadRun(args[2]);
-            console.log("");
-            console.log(`  对比 ${run1.suiteName}(${run1.timestamp}) vs ${run2.suiteName}(${run2.timestamp})`);
-            console.log("");
-            const qIds1 = new Set(run1.results.map((r) => r.questionId));
-            const qIds2 = new Set(run2.results.map((r) => r.questionId));
-            const common = [...qIds1].filter((q) => qIds2.has(q));
-            for (const qId of common) {
-                console.log(`  ── ${qId} ──`);
-                const r1 = run1.results.find((r) => r.questionId === qId);
-                const r2 = run2.results.find((r) => r.questionId === qId);
-                if (r1 && r2) {
-                    const s1 = r1.score !== undefined ? `[${r1.score}/${r1.maxScore}]` : "";
-                    const s2 = r2.score !== undefined ? `[${r2.score}/${r2.maxScore}]` : "";
-                    console.log(`  ${s1} ${r1.modelId}: ${r1.response.slice(0, 200)}`);
-                    console.log(`  ${s2} ${r2.modelId}: ${r2.response.slice(0, 200)}`);
-                    console.log("");
-                }
-            }
-        }
-        catch (e) {
-            console.error(`  ${e instanceof Error ? e.message : e}`);
-            process.exit(1);
+    else {
+        console.log(`✅ ${formatDuration(resp.latencyMs)} · ${resp.tokensIn}→${resp.tokensOut} tok`);
+    }
+}
+async function runBench(args) {
+    const suiteName = args[0] ?? "quick";
+    const modelQueries = [];
+    let delayMs = 0;
+    for (let i = 1; i < args.length; i++) {
+        const a = args[i];
+        if (a === "--delay")
+            delayMs = Number(args[++i]) || 0;
+        else if (a.startsWith("--delay="))
+            delayMs = Number(a.slice("--delay=".length)) || 0;
+        else
+            modelQueries.push(a);
+    }
+    if (delayMs > 0)
+        console.log(`  ⏳ 题目间隔:${(delayMs / 1000).toFixed(0)}s(限流渠道用)`);
+    const cfg = loadConfig();
+    if (!cfg.apiKey) {
+        console.error("  还没有登录,先 u1s1 login");
+        process.exit(1);
+    }
+    await loadCustomEndpoints(cfg);
+    let suite;
+    try {
+        suite = loadSuite(suiteName);
+    }
+    catch (e) {
+        failWithError(e);
+    }
+    const targets = modelQueries.length > 0
+        ? modelQueries.map((q) => resolveModelTarget(q, cfg))
+        : await resolveDefaultTargets(cfg);
+    console.log("");
+    console.log(`  📊 Bench: ${suite.name}`);
+    console.log(`  ${suite.description}`);
+    console.log(`  题目数: ${suite.questions.length}`);
+    console.log(`  模型: ${targets.map((t) => t.label).join(", ")}`);
+    console.log("");
+    const results = [];
+    const total = suite.questions.length * targets.length;
+    let done = 0;
+    for (const target of targets) {
+        for (const q of suite.questions) {
+            const label = `${target.label} / ${q.id}`;
+            process.stdout.write(`  [${++done}/${total}] ${label.padEnd(40)} `);
+            const resp = await callModel(target.baseUrl, target.apiKey, target.id, q.prompt);
+            const costUsd = estimateCost(target.id, resp.tokensIn, resp.tokensOut);
+            // 有评分规则且调用没出错时才打分
+            const maxScore = q.maxScore ?? DEFAULT_MAX_SCORE;
+            const scoring = !resp.error && q.checks && q.checks.length > 0
+                ? scoreResponse(resp.response, q.checks, maxScore)
+                : null;
+            results.push({
+                questionId: q.id,
+                category: q.category,
+                prompt: q.prompt,
+                modelId: target.label,
+                providerName: target.source === "u1s1" ? PROVIDER_ID : target.label,
+                response: resp.response,
+                latencyMs: resp.latencyMs,
+                tokensIn: resp.tokensIn,
+                tokensOut: resp.tokensOut,
+                costUsd,
+                error: resp.error,
+                score: scoring?.score,
+                maxScore: scoring ? maxScore : undefined,
+                scoreDetails: scoring?.details,
+            });
+            printQuestionOutcome(resp, scoring?.score, scoring ? maxScore : undefined);
+            if (delayMs > 0)
+                await new Promise((r) => setTimeout(r, delayMs));
         }
-        return;
     }
+    const runId = `bench-${suite.name}-${Date.now()}`;
+    saveRun({
+        id: runId,
+        timestamp: new Date().toISOString(),
+        suiteName: suite.name,
+        models: [...new Set(results.map((r) => r.modelId))],
+        results,
+    });
+    printConsoleSummary(results);
+    console.log(`  查看完整报告: u1s1 bench report ${runId}`);
+    console.log("");
+}
+/** 控制台里的简易评分汇总表 */
+function printConsoleSummary(results) {
+    console.log("");
+    console.log("  📋 评分汇总:");
+    const header = ["模型", "总分", "得分率", "平均延迟", "速度", "花费", "状态"];
+    const rows = sortByScore([...summarizeByModel(results, (r) => r.modelId)]).map(([model, s]) => [
+        model,
+        `${s.totalScore}/${s.totalMax}`,
+        `${scoreRate(s)}%`,
+        formatDuration(s.totalLatency / s.total),
+        tokPerSec(s),
+        costStr(s.totalCost),
+        s.errors > 0 ? `${errorRate(s)}%失败` : "✅全部通过",
+    ]);
+    console.log("  " + renderTable(header, rows));
+    console.log("");
+}
+// ─── 报告 & 对比 ───
+function showReport(args) {
+    const runIdArg = args[0] ?? "latest";
+    let run;
+    try {
+        const id = runIdArg === "latest" ? findLatestRun() : runIdArg;
+        if (!id)
+            throw new Error("还没有运行记录");
+        run = loadRun(id);
+    }
+    catch (e) {
+        failWithError(e);
+    }
+    const report = generateReport(run);
+    const reportFile = join(BENCH_DIR, `${run.id}.md`);
+    writeFileSync(reportFile, report);
+    console.log(report);
+    console.log(`  报告已保存: ${reportFile}`);
+}
+/** 两次运行共同拥有的题目 id */
+function commonQuestionIds(a, b) {
+    const idsInB = new Set(b.results.map((r) => r.questionId));
+    return [...new Set(a.results.map((r) => r.questionId))].filter((id) => idsInB.has(id));
+}
+/** 单条回答带分数打印(截断到一行能看下的长度) */
+function printAnswerWithScore(r) {
+    const s = r.score !== undefined ? `[${r.score}/${r.maxScore}]` : "";
+    console.log(`  ${s} ${r.modelId}: ${r.response.slice(0, MAX_COMPARE_CHARS)}`);
+}
+function compareRuns(runId1, runId2) {
+    let run1;
+    let run2;
+    try {
+        run1 = loadRun(runId1);
+        run2 = loadRun(runId2);
+    }
+    catch (e) {
+        failWithError(e);
+    }
+    console.log("");
+    console.log(`  对比 ${run1.suiteName}(${run1.timestamp}) vs ${run2.suiteName}(${run2.timestamp})`);
+    console.log("");
+    for (const qId of commonQuestionIds(run1, run2)) {
+        console.log(`  ── ${qId} ──`);
+        const r1 = run1.results.find((r) => r.questionId === qId);
+        const r2 = run2.results.find((r) => r.questionId === qId);
+        printAnswerWithScore(r1);
+        printAnswerWithScore(r2);
+        console.log("");
+    }
+}
+// ─── 入口 ───
+export async function benchCommand(args) {
+    const sub = args[0];
+    if (!sub || sub === "help" || sub === "--help")
+        return printHelp();
+    if (sub === "list")
+        return listSuitesCommand();
+    if (sub === "run")
+        return runBench(args.slice(1));
+    if (sub === "report")
+        return showReport(args.slice(1));
     if (sub === "compare") {
-        console.log("  用法: u1s1 bench compare <run-id1> <run-id2>");
-        return;
+        if (!args[1] || !args[2]) {
+            console.log("  用法: u1s1 bench compare <run-id1> <run-id2>");
+            return;
+        }
+        return compareRuns(args[1], args[2]);
     }
     console.error(`  未知子命令: ${sub}`);
     console.log("  u1s1 bench help 查看用法");