u1s1-cli 1.2.3 → 1.2.4

package/dist/agent-setup.js

@@ -81,8 +81,8 @@ export function applyModelThinkingDefaults(settings, models) {
     const defaults = models.filter((model) => model.thinking !== undefined);
     if (defaults.length === 0)
         return false;
-    const levels = { ...(recordValue(settings["modelThinkingLevels"]) ?? {}) };
-    const managed = { ...(recordValue(settings[THINKING_DEFAULTS_MARKER]) ?? {}) };
+    const levels = { ...recordValue(settings["modelThinkingLevels"]) };
+    const managed = { ...recordValue(settings[THINKING_DEFAULTS_MARKER]) };
     let changed = false;
     for (const model of defaults) {
         const key = `${PROVIDER_ID}/${model.id}`;

package/dist/api.d.ts

@@ -105,7 +105,11 @@ export interface SearchResponse {
     results: SearchResult[];
 }
 /** 联网搜索走网关代理（上游 key 只在服务端）。maxResults 不传时由服务端决定默认值。 */
-export declare function searchWeb(cfg: Pick<CliConfig, "baseUrl" | "apiKey">, query: string, maxResults?: number, signal?: AbortSignal): Promise<SearchResponse>;
+export declare function searchWeb(cfg: Pick<CliConfig, "baseUrl" | "apiKey">, input: {
+    query: string;
+    maxResults?: number;
+    signal?: AbortSignal;
+}): Promise<SearchResponse>;
 /** web_fetch 直连失败时的回退:网关用 Cloudflare Browser Rendering 渲染后转 markdown。 */
 export declare function renderPage(cfg: Pick<CliConfig, "baseUrl" | "apiKey">, url: string, signal?: AbortSignal): Promise<{
     url: string;

package/dist/api.js

@@ -56,7 +56,7 @@ export async function loadCustomEndpoints(cfg) {
     }
 }
 /** 联网搜索走网关代理（上游 key 只在服务端）。maxResults 不传时由服务端决定默认值。 */
-export async function searchWeb(cfg, query, maxResults, signal) {
+export async function searchWeb(cfg, input) {
     if (!cfg.apiKey)
         throw new Error("没有配置 API Key");
     let resp;
@@ -67,8 +67,8 @@ export async function searchWeb(cfg, query, maxResults, signal) {
                 "x-u1s1-version": VERSION,
                 "content-type": "application/json",
             },
-            body: JSON.stringify({ query, max_results: maxResults }),
-            signal: signal ?? AbortSignal.timeout(60_000),
+            body: JSON.stringify({ query: input.query, max_results: input.maxResults }),
+            signal: input.signal ?? AbortSignal.timeout(60_000),
         });
     }
     catch {
@@ -171,10 +171,10 @@ export async function generateImage(cfg, req, signal) {
             signal: signal ?? AbortSignal.timeout(180_000),
         });
     }
-    catch (e) {
+    catch (error) {
         if (signal?.aborted)
-            throw e;
-        throw new Error(`连不上 ${cfg.baseUrl},检查一下网络?`);
+            throw error;
+        throw new Error(`连不上 ${cfg.baseUrl},检查一下网络?`, { cause: error });
     }
     if (resp.status === 401)
         throw new Error("登录已失效,请重新运行 u1s1 login");

package/dist/bench-report.d.ts

@@ -0,0 +1,22 @@
+import { type BenchResult, type BenchRunMeta } from "./bench-types.js";
+export interface ModelSummary {
+    totalScore: number;
+    totalMax: number;
+    totalLatency: number;
+    totalCost: number;
+    errors: number;
+    total: number;
+    okTokensOut: number;
+    okLatency: number;
+}
+export declare function resultModelLabel(result: BenchResult): string;
+export declare function formatDuration(ms: number): string;
+export declare function costStr(usd: number): string;
+export declare function renderTable(header: string[], rows: string[][]): string;
+export declare function scoreBar(score: number, maxScore: number, width?: number): string;
+export declare function summarizeByModel(results: BenchResult[], keyOf: (result: BenchResult) => string): Map<string, ModelSummary>;
+export declare function sortByScore(entries: [string, ModelSummary][]): [string, ModelSummary][];
+export declare function scoreRate(summary: ModelSummary): string;
+export declare function errorRate(summary: ModelSummary): string;
+export declare function tokPerSec(summary: ModelSummary): string;
+export declare function generateReport(run: BenchRunMeta): string;

package/dist/bench-report.js

@@ -0,0 +1,158 @@
+import { PROVIDER_ID } from "./config.js";
+import { DEFAULT_MAX_SCORE } from "./bench-types.js";
+const MAX_ANSWER_CHARS = 2_000;
+export function resultModelLabel(result) {
+    return result.providerName === PROVIDER_ID
+        ? result.modelId
+        : `${result.providerName}:${result.modelId}`;
+}
+export function formatDuration(ms) {
+    return ms < 1_000 ? `${ms.toFixed(0)}ms` : `${(ms / 1_000).toFixed(1)}s`;
+}
+export function costStr(usd) {
+    if (usd === 0)
+        return "—";
+    if (usd < 0.001)
+        return "<$0.001";
+    return `$${usd.toFixed(4)}`;
+}
+export function renderTable(header, rows) {
+    const columnWidths = header.map((heading, index) => Math.max(heading.length, ...rows.map((row) => row[index]?.length ?? 0)));
+    const separator = "─".repeat(columnWidths.reduce((sum, width) => sum + width + 3, 1));
+    const renderRow = (cells) => ` ${cells.map((cell, index) => cell.padEnd(columnWidths[index])).join(" │ ")} `;
+    return [renderRow(header), separator, ...rows.map(renderRow)].join("\n");
+}
+export function scoreBar(score, maxScore, width = 8) {
+    const ratio = maxScore > 0 ? Math.min(score / maxScore, 1) : 0;
+    const filled = Math.round(ratio * width);
+    const bar = "█".repeat(filled) + "░".repeat(width - filled);
+    if (ratio >= 0.8)
+        return `\x1b[32m${bar}\x1b[0m`;
+    if (ratio >= 0.5)
+        return `\x1b[33m${bar}\x1b[0m`;
+    return `\x1b[31m${bar}\x1b[0m`;
+}
+function emptySummary() {
+    return {
+        totalScore: 0,
+        totalMax: 0,
+        totalLatency: 0,
+        totalCost: 0,
+        errors: 0,
+        total: 0,
+        okTokensOut: 0,
+        okLatency: 0,
+    };
+}
+export function summarizeByModel(results, keyOf) {
+    const summaries = new Map();
+    for (const result of results) {
+        const key = keyOf(result);
+        const summary = summaries.get(key) ?? emptySummary();
+        summary.totalScore += result.score ?? 0;
+        summary.totalMax += result.maxScore ?? DEFAULT_MAX_SCORE;
+        summary.totalLatency += result.latencyMs;
+        summary.totalCost += result.costUsd;
+        if (result.error)
+            summary.errors++;
+        else {
+            summary.okTokensOut += result.tokensOut;
+            summary.okLatency += result.latencyMs;
+        }
+        summary.total++;
+        summaries.set(key, summary);
+    }
+    return summaries;
+}
+export function sortByScore(entries) {
+    return entries.sort((left, right) => right[1].totalScore - left[1].totalScore);
+}
+export function scoreRate(summary) {
+    return summary.totalMax > 0
+        ? ((summary.totalScore / summary.totalMax) * 100).toFixed(1)
+        : "0.0";
+}
+export function errorRate(summary) {
+    return ((summary.errors / summary.total) * 100).toFixed(0);
+}
+export function tokPerSec(summary) {
+    return summary.okLatency > 0
+        ? ((summary.okTokensOut / summary.okLatency) * 1_000).toFixed(1)
+        : "—";
+}
+function groupResultsBy(results, keyOf) {
+    const groups = new Map();
+    for (const result of results) {
+        const key = keyOf(result);
+        const group = groups.get(key) ?? [];
+        group.push(result);
+        groups.set(key, group);
+    }
+    return groups;
+}
+function appendScoreDetails(lines, results, maxScore) {
+    const scored = results.filter((result) => result.scoreDetails?.length);
+    if (scored.length === 0)
+        return;
+    lines.push("", "<details><summary>评分明细</summary>", "");
+    for (const result of scored) {
+        lines.push(`**${resultModelLabel(result)}:** ${result.score}/${maxScore}`);
+        for (const detail of result.scoreDetails) {
+            lines.push(`- ${detail.passed ? "✅" : "❌"} ${detail.check} (+${detail.earned}/${detail.points})`);
+        }
+        lines.push("");
+    }
+    lines.push("</details>", "");
+}
+function appendAnswers(lines, results) {
+    for (const result of results) {
+        lines.push(`<details><summary>${resultModelLabel(result)} 的回答</summary>`, "", "```");
+        lines.push(result.response.slice(0, MAX_ANSWER_CHARS));
+        if (result.response.length > MAX_ANSWER_CHARS)
+            lines.push("...(截断)");
+        lines.push("```", "</details>", "");
+    }
+}
+function appendQuestion(lines, questionId, results) {
+    const first = results[0];
+    const maxScore = first.maxScore ?? DEFAULT_MAX_SCORE;
+    lines.push(`### ${questionId}`, `> ${first.prompt}`, "");
+    const rows = results.map((result) => [
+        resultModelLabel(result),
+        result.score === undefined ? "—" : `${result.score}/${maxScore}`,
+        formatDuration(result.latencyMs),
+        String(result.tokensOut),
+        result.error ? `❌ ${result.error.slice(0, 40)}` : "✅",
+    ]);
+    lines.push("```", renderTable(["模型", `得分/${maxScore}`, "延迟", "输出tok", "状态"], rows), "```");
+    appendScoreDetails(lines, results, maxScore);
+    appendAnswers(lines, results);
+}
+function appendSummary(lines, results) {
+    const rows = sortByScore([...summarizeByModel(results, resultModelLabel)]).map(([model, summary]) => [
+        model,
+        `${summary.totalScore}/${summary.totalMax}`,
+        `${scoreRate(summary)}%`,
+        formatDuration(summary.totalLatency / summary.total),
+        tokPerSec(summary),
+        costStr(summary.totalCost),
+        summary.errors > 0 ? `${errorRate(summary)}%` : "0%",
+    ]);
+    lines.push("## 汇总", "", "```", renderTable(["模型", "总分", "得分率", "平均延迟", "速度(tok/s)", "总花费", "错误率"], rows), "```");
+}
+export function generateReport(run) {
+    const lines = [
+        `# Bench 报告: ${run.suiteName}`,
+        `运行时间: ${run.timestamp}`,
+        `模型: ${run.models.join(", ")}`,
+        "",
+    ];
+    for (const [category, categoryResults] of groupResultsBy(run.results, (result) => result.category || "未分类")) {
+        lines.push(`## ${category}`, "");
+        for (const [questionId, questionResults] of groupResultsBy(categoryResults, (result) => result.questionId)) {
+            appendQuestion(lines, questionId, questionResults);
+        }
+    }
+    appendSummary(lines, run.results);
+    return lines.join("\n");
+}

package/dist/bench-scoring.d.ts

@@ -0,0 +1,8 @@
+import type { BenchCheck, ScoreDetail } from "./bench-types.js";
+/** Execute one declarative benchmark check. */
+export declare function runCheck(response: string, check: BenchCheck): boolean;
+/** Score one response against all declared checks. */
+export declare function scoreResponse(response: string, checks: BenchCheck[], maxScore: number): {
+    score: number;
+    details: ScoreDetail[];
+};

package/dist/bench-scoring.js

@@ -0,0 +1,101 @@
+/** Remove Markdown fences before parsing a JSON answer. */
+function stripCodeFences(text) {
+    return text.replace(/```json\s*\n?/gi, "").replace(/\n?```/g, "").trim();
+}
+function containsExpectedText(response, check) {
+    const needle = String(check.value);
+    const text = check.ignoreCase ? response.toLowerCase() : response;
+    const search = check.ignoreCase ? needle.toLowerCase() : needle;
+    const found = text.includes(search);
+    return check.type === "contains" ? found : !found;
+}
+function isValidRegex(response, value) {
+    try {
+        new RegExp(String(value));
+        return true;
+    }
+    catch {
+        const literal = response.match(/\/(.+)\/[gimsuy]*/);
+        if (literal) {
+            new RegExp(literal[1]);
+            return true;
+        }
+        const cleaned = response.replace(/```[\s\S]*?```/g, "").trim();
+        const parts = cleaned.startsWith("/") ? cleaned.match(/^\/(.+)\/([gimsuy]*)$/) : null;
+        if (!parts)
+            return false;
+        new RegExp(parts[1]);
+        return true;
+    }
+}
+function hasJsonKey(response, value) {
+    try {
+        const parsed = JSON.parse(stripCodeFences(response));
+        return parsed[String(value)] !== undefined;
+    }
+    catch {
+        return false;
+    }
+}
+function usesAllowedWords(response, allowed) {
+    const tokens = response.split(/[\s，。！？、；：""''（）[【】\]「」]+/).filter(Boolean);
+    return tokens.every((token) => /^[，。！？、；：""''（）《》\s.。，,、!?;:…\-—]+$/.test(token) || allowed.includes(token));
+}
+function hasExpectedFiveCharacterLines(response, expected) {
+    let count = 0;
+    for (const line of response.split("\n")) {
+        const chars = line.replace(/[，。！？、；：""''（）《》\s，.\-,;:!?\dA-Za-z]/g, "").trim();
+        if (/^[\u4e00-\u9fff]{5}$/.test(chars))
+            count++;
+    }
+    return count >= expected;
+}
+/** Execute one declarative benchmark check. */
+export function runCheck(response, check) {
+    switch (check.type) {
+        case "contains":
+        case "not_contains":
+            return containsExpectedText(response, check);
+        case "contains_code_block":
+            return /```[\s\S]*?```/.test(response) || /`[^`]+`/.test(response);
+        case "parse_json": {
+            try {
+                JSON.parse(stripCodeFences(response));
+                return true;
+            }
+            catch {
+                return false;
+            }
+        }
+        case "valid_regex":
+            return isValidRegex(response, check.value);
+        case "has_json_key":
+            return hasJsonKey(response, check.value);
+        case "max_length":
+            return response.length <= Number(check.value);
+        case "min_length":
+            return response.length >= Number(check.value);
+        case "allowed_words_only":
+            return usesAllowedWords(response, check.value);
+        case "has_chinese":
+            return /[\u4e00-\u9fff\u3400-\u4dbf]/.test(response);
+        case "line_count_5chars":
+            return hasExpectedFiveCharacterLines(response, Number(check.value));
+        default:
+            return false;
+    }
+}
+/** Score one response against all declared checks. */
+export function scoreResponse(response, checks, maxScore) {
+    const details = checks.map((check) => {
+        const passed = runCheck(response, check);
+        return {
+            check: check.description,
+            passed,
+            points: check.points,
+            earned: passed ? check.points : 0,
+        };
+    });
+    const total = details.reduce((sum, detail) => sum + detail.earned, 0);
+    return { score: Math.min(total, maxScore), details };
+}

package/dist/bench-types.d.ts

@@ -0,0 +1,50 @@
+export interface BenchCheck {
+    type: string;
+    /** Depends on type: string | number | boolean | string[]. */
+    value: unknown;
+    points: number;
+    description: string;
+    ignoreCase?: boolean;
+}
+export interface BenchQuestion {
+    id: string;
+    category: string;
+    prompt: string;
+    maxScore?: number;
+    checks?: BenchCheck[];
+}
+export interface BenchSuite {
+    name: string;
+    description: string;
+    questions: BenchQuestion[];
+}
+export interface ScoreDetail {
+    check: string;
+    passed: boolean;
+    points: number;
+    earned: number;
+}
+export interface BenchResult {
+    questionId: string;
+    category: string;
+    prompt: string;
+    modelId: string;
+    providerName: string;
+    response: string;
+    latencyMs: number;
+    tokensIn: number;
+    tokensOut: number;
+    costUsd: number;
+    error?: string;
+    score?: number;
+    maxScore?: number;
+    scoreDetails?: ScoreDetail[];
+}
+export interface BenchRunMeta {
+    id: string;
+    timestamp: string;
+    suiteName: string;
+    models: string[];
+    results: BenchResult[];
+}
+export declare const DEFAULT_MAX_SCORE = 10;

package/dist/bench-types.js

@@ -0,0 +1 @@
+export const DEFAULT_MAX_SCORE = 10;