u1s1-cli 1.2.3 → 1.2.4

package/dist/bench.js

@@ -5,26 +5,17 @@ import { homedir } from "node:os";
 import { CUSTOM_ENDPOINTS, loadConfig, MODELS, PROVIDER_ID, } from "./config.js";
 import { loadCustomEndpoints } from "./api.js";
 import { authorizedFetch, hasDeviceCredential } from "./device-auth.js";
+import { costStr, errorRate, formatDuration, generateReport, renderTable, scoreBar, scoreRate, sortByScore, summarizeByModel, tokPerSec, } from "./bench-report.js";
+import { scoreResponse } from "./bench-scoring.js";
+import { DEFAULT_MAX_SCORE, } from "./bench-types.js";
 // ─── 工具 ───
 const BENCH_DIR = join(homedir(), ".u1s1", "bench");
-/** 题目未声明 maxScore 时的默认满分 */
-const DEFAULT_MAX_SCORE = 10;
-/** 报告里单条回答最多保留的字符数 */
-const MAX_ANSWER_CHARS = 2000;
 /** compare 子命令里单条回答的截断长度 */
 const MAX_COMPARE_CHARS = 200;
-/** 去掉回答里的 markdown 代码块围栏,便于 JSON 解析 */
-function stripCodeFences(text) {
-    return text.replace(/```json\s*\n?/gi, "").replace(/\n?```/g, "").trim();
-}
 /** 自定义端点的模型 id 常带 org/ 前缀,取斜杠后的短 id */
 function shortModelId(id) {
     return id.includes("/") ? id.split("/")[1] : id;
 }
-/** 结果里模型的显示名:内置模型不带前缀,自定义端点带 provider 前缀 */
-function resultModelLabel(r) {
-    return r.providerName === PROVIDER_ID ? r.modelId : `${r.providerName}:${r.modelId}`;
-}
 /** 项目 bench 数据目录 */
 function benchDataDir() {
     const distDir = dirname(fileURLToPath(import.meta.url));
@@ -55,184 +46,44 @@ function loadSuite(name) {
     throw new Error(`找不到 bench suite「${name}」`);
 }
 /** 列出可用的 suite */
-function listSuites() {
-    const suites = [];
-    const builtinPath = join(benchDataDir(), "suite.json");
-    if (existsSync(builtinPath)) {
-        const raw = JSON.parse(readFileSync(builtinPath, "utf8"));
-        for (const s of raw) {
-            suites.push({ name: s.name, description: s.description, builtin: true });
-        }
+function suiteSummaries(filePath, builtin) {
+    try {
+        return readSuitesFile(filePath).map((suite) => ({
+            name: suite.name,
+            description: suite.description,
+            builtin,
+        }));
     }
-    if (existsSync(BENCH_DIR)) {
-        try {
-            const files = readdirSync(BENCH_DIR).filter((f) => f.endsWith(".json"));
-            for (const f of files) {
-                try {
-                    const raw = JSON.parse(readFileSync(join(BENCH_DIR, f), "utf8"));
-                    for (const s of raw) {
-                        if (!suites.find((x) => x.name === s.name)) {
-                            suites.push({ name: s.name, description: s.description, builtin: false });
-                        }
-                    }
-                }
-                catch { /* 格式不对跳过 */ }
-            }
-        }
-        catch { /* 忽略 */ }
+    catch {
+        return [];
     }
-    return suites;
 }
-function formatDuration(ms) {
-    if (ms < 1000)
-        return `${ms.toFixed(0)}ms`;
-    return `${(ms / 1000).toFixed(1)}s`;
-}
-function costStr(usd) {
-    if (usd === 0)
-        return "—";
-    if (usd < 0.001)
-        return "<$0.001";
-    return `$${usd.toFixed(4)}`;
-}
-/** 排列表格 */
-function renderTable(header, rows) {
-    const colW = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i]?.length ?? 0)));
-    const sep = "─".repeat(colW.reduce((a, b) => a + b + 3, 1));
-    const line = (cells) => " " + cells.map((c, i) => c.padEnd(colW[i])).join(" │ ") + " ";
-    return [line(header), sep, ...rows.map((r) => line(r))].join("\n");
-}
-/** 热度条:分数按比例转成视觉条 */
-function scoreBar(score, maxScore, width = 8) {
-    const ratio = maxScore > 0 ? Math.min(score / maxScore, 1) : 0;
-    const filled = Math.round(ratio * width);
-    const empty = width - filled;
-    const bar = "█".repeat(filled) + "░".repeat(empty);
-    // 颜色标记
-    if (ratio >= 0.8)
-        return `\x1b[32m${bar}\x1b[0m`; // 绿
-    if (ratio >= 0.5)
-        return `\x1b[33m${bar}\x1b[0m`; // 黄
-    return `\x1b[31m${bar}\x1b[0m`; // 红
-}
-// ─── 评分引擎 ───
-/** 对一条回答按 checks 逐项打分 */
-function scoreResponse(response, checks, maxScore) {
-    const details = [];
-    let total = 0;
-    for (const check of checks) {
-        const result = runCheck(response, check);
-        details.push({
-            check: check.description,
-            passed: result,
-            points: check.points,
-            earned: result ? check.points : 0,
-        });
-        if (result)
-            total += check.points;
+function userSuitePaths() {
+    if (!existsSync(BENCH_DIR))
+        return [];
+    try {
+        return readdirSync(BENCH_DIR)
+            .filter((file) => file.endsWith(".json"))
+            .map((file) => join(BENCH_DIR, file));
     }
-    return { score: Math.min(total, maxScore), details };
-}
-/** 执行单项检查 */
-function runCheck(response, check) {
-    const val = check.value;
-    switch (check.type) {
-        case "contains":
-        case "not_contains": {
-            const needle = String(val);
-            const text = check.ignoreCase ? response.toLowerCase() : response;
-            const search = check.ignoreCase ? needle.toLowerCase() : needle;
-            const found = text.includes(search);
-            return check.type === "contains" ? found : !found;
-        }
-        case "contains_code_block":
-            return /```[\s\S]*?```/.test(response) || /`[^`]+`/.test(response);
-        case "parse_json": {
-            try {
-                JSON.parse(stripCodeFences(response));
-                return true;
-            }
-            catch {
-                return false;
-            }
-        }
-        case "valid_regex": {
-            try {
-                new RegExp(String(val));
-                return true;
-            }
-            catch {
-                // 从回答里提取 regex 来验证
-                const match = response.match(/\/(.+)\/[gimsuy]*/);
-                if (match) {
-                    new RegExp(match[1]);
-                    return true;
-                }
-                // 也可能是文本形式
-                const cleaned = response.replace(/```[\s\S]*?```/g, "").trim();
-                if (cleaned.startsWith("/")) {
-                    const parts = cleaned.match(/^\/(.+)\/([gimsuy]*)$/);
-                    if (parts) {
-                        new RegExp(parts[1]);
-                        return true;
-                    }
-                }
-                return false;
-            }
-        }
-        case "has_json_key": {
-            try {
-                const obj = JSON.parse(stripCodeFences(response));
-                return obj[String(val)] !== undefined;
-            }
-            catch {
-                return false;
-            }
-        }
-        case "max_length": {
-            const limit = Number(val);
-            return response.length <= limit;
-        }
-        case "min_length": {
-            const limit = Number(val);
-            return response.length >= limit;
-        }
-        case "allowed_words_only": {
-            const allowed = val;
-            // 提取回答中的所有中文/英文字词
-            const tokens = response.split(/[\s，。！？、；：""''（）\[【】\]「」]+/).filter(Boolean);
-            for (const token of tokens) {
-                if (token.length === 0)
-                    continue;
-                // 标点/空格不检查
-                if (/^[，。！？、；：""''（）《》\s.。，,、!?;:…\-—]+$/.test(token))
-                    continue;
-                if (!allowed.includes(token))
-                    return false;
-            }
-            return true;
-        }
-        case "has_chinese": {
-            return /[\u4e00-\u9fff\u3400-\u4dbf]/.test(response);
-        }
-        case "line_count_5chars": {
-            // 检查有几行是恰好 5 个汉字（去除标点空格）
-            const expected = Number(val);
-            const lines = response.split("\n");
-            let count5 = 0;
-            for (const line of lines) {
-                const chars = line.replace(/[，。！？、；：""''（）《》\s，\.\-,;:!?\dA-Za-z]/g, "").trim();
-                if (/^[\u4e00-\u9fff]{5}$/.test(chars))
-                    count5++;
-            }
-            return count5 >= expected;
+    catch {
+        return [];
+    }
+}
+function listSuites() {
+    const builtinPath = join(benchDataDir(), "suite.json");
+    const suites = existsSync(builtinPath) ? suiteSummaries(builtinPath, true) : [];
+    for (const path of userSuitePaths()) {
+        for (const suite of suiteSummaries(path, false)) {
+            if (!suites.some((current) => current.name === suite.name))
+                suites.push(suite);
         }
-        default:
-            return false;
     }
+    return suites;
 }
 // ─── 模型调用 ───
-async function callModel(baseUrl, apiKey, modelId, prompt, officialCfg) {
+async function callModel(input) {
+    const { baseUrl, apiKey, modelId, prompt, officialCfg } = input;
     const start = performance.now();
     const isReasoning = /reasoner|grok/i.test(modelId);
     const body = {
@@ -319,146 +170,6 @@ function listRuns() {
     }
     return runs;
 }
-/** 把一批结果按模型聚合成分数/延迟/花费/错误数 */
-function summarizeByModel(results, keyOf) {
-    const map = new Map();
-    for (const r of results) {
-        const key = keyOf(r);
-        const s = map.get(key) ?? { totalScore: 0, totalMax: 0, totalLatency: 0, totalCost: 0, errors: 0, total: 0, okTokensOut: 0, okLatency: 0 };
-        s.totalScore += r.score ?? 0;
-        s.totalMax += r.maxScore ?? DEFAULT_MAX_SCORE;
-        s.totalLatency += r.latencyMs;
-        s.totalCost += r.costUsd;
-        if (r.error) {
-            s.errors++;
-        }
-        else {
-            s.okTokensOut += r.tokensOut;
-            s.okLatency += r.latencyMs;
-        }
-        s.total++;
-        map.set(key, s);
-    }
-    return map;
-}
-/** 按总分从高到低排序 */
-function sortByScore(entries) {
-    return entries.sort((a, b) => b[1].totalScore - a[1].totalScore);
-}
-/** 得分率百分数字符串(一位小数) */
-function scoreRate(s) {
-    return s.totalMax > 0 ? ((s.totalScore / s.totalMax) * 100).toFixed(1) : "0.0";
-}
-/** 错误率百分数字符串(整数) */
-function errorRate(s) {
-    return ((s.errors / s.total) * 100).toFixed(0);
-}
-/** 推理速度:成功题的输出 token / 耗时(tok/s),没有成功题返回 — */
-function tokPerSec(s) {
-    return s.okLatency > 0 ? (s.okTokensOut / s.okLatency * 1000).toFixed(1) : "—";
-}
-// ─── 报告生成 ───
-function generateReport(run) {
-    const lines = [];
-    lines.push(`# Bench 报告: ${run.suiteName}`);
-    lines.push(`运行时间: ${run.timestamp}`);
-    lines.push(`模型: ${run.models.join(", ")}`);
-    lines.push("");
-    const byCategory = new Map();
-    for (const r of run.results) {
-        const cat = r.category || "未分类";
-        if (!byCategory.has(cat))
-            byCategory.set(cat, []);
-        byCategory.get(cat).push(r);
-    }
-    for (const [cat, results] of byCategory) {
-        lines.push(`## ${cat}`);
-        lines.push("");
-        const byQuestion = new Map();
-        for (const r of results) {
-            if (!byQuestion.has(r.questionId))
-                byQuestion.set(r.questionId, []);
-            byQuestion.get(r.questionId).push(r);
-        }
-        for (const [qId, qResults] of byQuestion) {
-            const prompt = qResults[0].prompt;
-            const maxScore = qResults[0].maxScore ?? DEFAULT_MAX_SCORE;
-            lines.push(`### ${qId}`);
-            lines.push(`> ${prompt}`);
-            lines.push("");
-            // 表格: 模型 | 分数 | 延迟 | 输出token | 状态
-            const header = ["模型", `得分/${maxScore}`, "延迟", "输出tok", "状态"];
-            const rows = [];
-            for (const r of qResults) {
-                const scoreStr = r.score !== undefined ? `${r.score}/${maxScore}` : "—";
-                rows.push([
-                    resultModelLabel(r),
-                    scoreStr,
-                    formatDuration(r.latencyMs),
-                    String(r.tokensOut),
-                    r.error ? `❌ ${r.error.slice(0, 40)}` : "✅",
-                ]);
-            }
-            lines.push("```");
-            lines.push(renderTable(header, rows));
-            lines.push("```");
-            // 评分明细
-            const scored = qResults.filter((r) => r.scoreDetails && r.scoreDetails.length > 0);
-            if (scored.length > 0) {
-                lines.push("");
-                lines.push("<details><summary>评分明细</summary>");
-                lines.push("");
-                for (const r of scored) {
-                    lines.push(`**${resultModelLabel(r)}:** ${r.score}/${maxScore}`);
-                    for (const d of r.scoreDetails) {
-                        const icon = d.passed ? "✅" : "❌";
-                        lines.push(`- ${icon} ${d.check} (+${d.earned}/${d.points})`);
-                    }
-                    lines.push("");
-                }
-                lines.push("</details>");
-                lines.push("");
-            }
-            // 各模型回答
-            for (const r of qResults) {
-                lines.push(`<details><summary>${resultModelLabel(r)} 的回答</summary>`);
-                lines.push("");
-                lines.push("```");
-                lines.push(r.response.slice(0, MAX_ANSWER_CHARS));
-                if (r.response.length > MAX_ANSWER_CHARS)
-                    lines.push("...(截断)");
-                lines.push("");
-                lines.push("```");
-                lines.push(r.response.slice(0, 2000));
-                if (r.response.length > 2000)
-                    lines.push("...(截断)");
-                lines.push("```");
-                lines.push("</details>");
-                lines.push("");
-            }
-        }
-    }
-    // 汇总 — 带分数
-    lines.push("## 汇总");
-    lines.push("");
-    const header = ["模型", "总分", "得分率", "平均延迟", "速度(tok/s)", "总花费", "错误率"];
-    const rows = [];
-    for (const [model, s] of sortByScore([...summarizeByModel(run.results, resultModelLabel)])) {
-        rows.push([
-            model,
-            `${s.totalScore}/${s.totalMax}`,
-            `${scoreRate(s)}%`,
-            formatDuration(s.totalLatency / s.total),
-            tokPerSec(s),
-            costStr(s.totalCost),
-            s.errors > 0 ? `${errorRate(s)}%` : "0%",
-        ]);
-    }
-    lines.push("```");
-    lines.push(renderTable(header, rows));
-    lines.push("```");
-    return lines.join("\n");
-}
 // ─── 主命令 ───
 /** 打印错误信息并以退出码 1 结束。永不返回。 */
 function failWithError(e) {
@@ -541,19 +252,70 @@ function printQuestionOutcome(resp, score, maxScore) {
         console.log(`✅ ${formatDuration(resp.latencyMs)} · ${resp.tokensIn}→${resp.tokensOut} tok`);
     }
 }
-async function runBench(args) {
+function parseBenchRunArgs(args) {
     const suiteName = args[0] ?? "quick";
     const modelQueries = [];
     let delayMs = 0;
-    for (let i = 1; i < args.length; i++) {
-        const a = args[i];
-        if (a === "--delay")
-            delayMs = Number(args[++i]) || 0;
-        else if (a.startsWith("--delay="))
-            delayMs = Number(a.slice("--delay=".length)) || 0;
-        else
-            modelQueries.push(a);
+    for (let index = 1; index < args.length; index++) {
+        const argument = args[index];
+        if (argument === "--delay") {
+            delayMs = Number(args[++index]) || 0;
+        }
+        else if (argument.startsWith("--delay=")) {
+            delayMs = Number(argument.slice("--delay=".length)) || 0;
+        }
+        else {
+            modelQueries.push(argument);
+        }
     }
+    return { suiteName, modelQueries, delayMs };
+}
+function loadSuiteOrExit(name) {
+    try {
+        return loadSuite(name);
+    }
+    catch (error) {
+        return failWithError(error);
+    }
+}
+async function runBenchQuestion(input) {
+    const { target, question, progress, delayMs } = input;
+    const label = `${target.label} / ${question.id}`;
+    process.stdout.write(`  [${progress}] ${label.padEnd(40)} `);
+    const response = await callModel({
+        baseUrl: target.baseUrl,
+        apiKey: target.apiKey,
+        modelId: target.id,
+        prompt: question.prompt,
+        officialCfg: target.officialCfg,
+    });
+    const maxScore = question.maxScore ?? DEFAULT_MAX_SCORE;
+    const scoring = !response.error && question.checks && question.checks.length > 0
+        ? scoreResponse(response.response, question.checks, maxScore)
+        : null;
+    const result = {
+        questionId: question.id,
+        category: question.category,
+        prompt: question.prompt,
+        modelId: target.label,
+        providerName: target.source === "u1s1" ? PROVIDER_ID : target.label,
+        response: response.response,
+        latencyMs: response.latencyMs,
+        tokensIn: response.tokensIn,
+        tokensOut: response.tokensOut,
+        costUsd: estimateCost(target.id, response.tokensIn, response.tokensOut),
+        error: response.error,
+        score: scoring?.score,
+        maxScore: scoring ? maxScore : undefined,
+        scoreDetails: scoring?.details,
+    };
+    printQuestionOutcome(response, scoring?.score, scoring ? maxScore : undefined);
+    if (delayMs > 0)
+        await new Promise((resolveDelay) => setTimeout(resolveDelay, delayMs));
+    return result;
+}
+async function runBench(args) {
+    const { suiteName, modelQueries, delayMs } = parseBenchRunArgs(args);
     if (delayMs > 0)
         console.log(`  ⏳ 题目间隔:${(delayMs / 1000).toFixed(0)}s(限流渠道用)`);
     const cfg = loadConfig();
@@ -562,13 +324,7 @@ async function runBench(args) {
         process.exit(1);
     }
     await loadCustomEndpoints(cfg);
-    let suite;
-    try {
-        suite = loadSuite(suiteName);
-    }
-    catch (e) {
-        failWithError(e);
-    }
+    const suite = loadSuiteOrExit(suiteName);
     const targets = modelQueries.length > 0
         ? modelQueries.map((q) => resolveModelTarget(q, cfg))
         : await resolveDefaultTargets(cfg);
@@ -582,35 +338,13 @@ async function runBench(args) {
     const total = suite.questions.length * targets.length;
     let done = 0;
     for (const target of targets) {
-        for (const q of suite.questions) {
-            const label = `${target.label} / ${q.id}`;
-            process.stdout.write(`  [${++done}/${total}] ${label.padEnd(40)} `);
-            const resp = await callModel(target.baseUrl, target.apiKey, target.id, q.prompt, target.officialCfg);
-            const costUsd = estimateCost(target.id, resp.tokensIn, resp.tokensOut);
-            // 有评分规则且调用没出错时才打分
-            const maxScore = q.maxScore ?? DEFAULT_MAX_SCORE;
-            const scoring = !resp.error && q.checks && q.checks.length > 0
-                ? scoreResponse(resp.response, q.checks, maxScore)
-                : null;
-            results.push({
-                questionId: q.id,
-                category: q.category,
-                prompt: q.prompt,
-                modelId: target.label,
-                providerName: target.source === "u1s1" ? PROVIDER_ID : target.label,
-                response: resp.response,
-                latencyMs: resp.latencyMs,
-                tokensIn: resp.tokensIn,
-                tokensOut: resp.tokensOut,
-                costUsd,
-                error: resp.error,
-                score: scoring?.score,
-                maxScore: scoring ? maxScore : undefined,
-                scoreDetails: scoring?.details,
-            });
-            printQuestionOutcome(resp, scoring?.score, scoring ? maxScore : undefined);
-            if (delayMs > 0)
-                await new Promise((r) => setTimeout(r, delayMs));
+        for (const question of suite.questions) {
+            results.push(await runBenchQuestion({
+                target,
+                question,
+                progress: `${++done}/${total}`,
+                delayMs,
+            }));
         }
     }
     const runId = `bench-${suite.name}-${Date.now()}`;

package/dist/brand.d.ts

@@ -9,8 +9,14 @@ export declare const HERO_ART: string[];
  * Startup hero, responsive to terminal width:
  * wide → wordmark with info column beside it; medium → stacked; narrow → one-liner.
  */
-export declare function renderBrandHeader(theme: Theme, version: string, cwd: string, width: number, notice?: string, announcement?: {
-    text: string;
-    url?: string;
+export declare function renderBrandHeader(theme: Theme, input: {
+    version: string;
+    cwd: string;
+    width: number;
+    notice?: string;
+    announcement?: {
+        text: string;
+        url?: string;
+    };
 }): string[];
 export declare function printConsoleBanner(version: string): void;

package/dist/brand.js

@@ -51,24 +51,24 @@ function announcementLine(theme, a) {
  * Startup hero, responsive to terminal width:
  * wide → wordmark with info column beside it; medium → stacked; narrow → one-liner.
  */
-export function renderBrandHeader(theme, version, cwd, width, notice, announcement) {
-    const name = theme.bold(theme.fg("text", `${BRAND_NAME} v${version}`)) +
-        (notice ? `  ${theme.fg("accent", notice)}` : "");
+export function renderBrandHeader(theme, input) {
+    const name = theme.bold(theme.fg("text", `${BRAND_NAME} v${input.version}`)) +
+        (input.notice ? `  ${theme.fg("accent", input.notice)}` : "");
     const brand = theme.fg("muted", `${BRAND_CN} · ${BRAND_TAGLINE}`);
-    const dir = theme.fg("dim", `cwd: ${formatHomePath(cwd)}`);
+    const dir = theme.fg("dim", `cwd: ${formatHomePath(input.cwd)}`);
     const hints = theme.fg("dim", "/help · Ctrl+V 图片 · Shift+Enter 换行 · Esc 中断");
-    if (width < ART_WIDTH + 4) {
+    if (input.width < ART_WIDTH + 4) {
         const lines = ["", ` ${theme.fg("accent", "✻")} ${name}  ${theme.fg("muted", BRAND_CN)}`, `   ${dir}`, `   ${theme.fg("dim", "Ctrl+V 粘贴图片")}`];
-        if (announcement)
-            lines.push(`   ${announcementLine(theme, announcement)}`);
+        if (input.announcement)
+            lines.push(`   ${announcementLine(theme, input.announcement)}`);
         return [...lines, ""];
     }
     const art = HERO_ART.map((line) => ` ${paintArt(theme, line)}`);
     // widest info row (hints) needs 46 cols beside the 28-col wordmark
-    if (width < ART_WIDTH + 46) {
+    if (input.width < ART_WIDTH + 46) {
         const lines = ["", ...art, "", ` ${name}  ${brand}`, ` ${dir}`, ` ${hints}`];
-        if (announcement)
-            lines.push(` ${announcementLine(theme, announcement)}`);
+        if (input.announcement)
+            lines.push(` ${announcementLine(theme, input.announcement)}`);
         return [...lines, ""];
     }
     const rows = [...art];
@@ -78,8 +78,8 @@ export function renderBrandHeader(theme, version, cwd, width, notice, announceme
     rows[3] += `${gap}${dir}`;
     rows[4] += `${gap}${hints}`;
     // 公告放信息列末尾(最后一行字模旁),够醒目又不挤掉常规信息
-    if (announcement)
-        rows[5] += `${gap}${announcementLine(theme, announcement)}`;
+    if (input.announcement)
+        rows[5] += `${gap}${announcementLine(theme, input.announcement)}`;
     return ["", ...rows, ""];
 }
 export function printConsoleBanner(version) {