pi-distill 1.3.0 → 1.4.0

package/README.md

@@ -193,7 +193,7 @@ Configuration-file fields take precedence over environment variables. Unspecifie
 
 Use `/distill:stats` to view distillation statistics for the current Pi session. Statistics are kept in memory, reset when the session starts, and never store raw tool output.
 
-The report includes tool-result counts, success/failure/fallback counts, model attempts, original and summary character totals, compression ratio, `chars/4` estimated original/summary tokens, estimated tokens saved, and model-reported input/output/cache/total tokens and cost. Counts use compact `k` and `m` units at 1,000 and 1,000,000; durations use `ms`, `s`, or `min` based on their value. Estimated context tokens are labeled as estimates; usage and cost fields are shown as unavailable when the provider does not return usage data.
+The report includes tool-result counts, success/failure/fallback counts, model attempts, original and summary character totals, compression ratio, estimated original/summary tokens (heuristic: CJK characters count ~1 token each, other text ~4 chars/token), estimated tokens saved, and model-reported input/output/cache/total tokens and cost. Counts use compact `k` and `m` units at 1,000 and 1,000,000; durations use `ms`, `s`, or `min` based on their value. Estimated context tokens are labeled as estimates; usage and cost fields are shown as unavailable when the provider does not return usage data.
 
 The main environment variables are `PI_DISTILL_MODEL`, `PI_DISTILL_MIN_CHARS`, `PI_DISTILL_MAX_CHARS`, `PI_DISTILL_MAX_OUTPUT_CHARS`, `PI_DISTILL_TIMEOUT_SECONDS`, `PI_DISTILL_TIMEOUT_RETRY_COUNT`, `PI_DISTILL_ERROR_RETRY_COUNT`, `PI_DISTILL_MISSED_COMPRESSION_RATIO`, and `PI_DISTILL_SUMMARIZE_ERRORS`.
 

package/README.zh-CN.md

@@ -196,7 +196,7 @@ Agent 消费更适合当前决策的结果，并获得可审计的处理诊断
 
 使用 `/distill:stats` 查看当前 Pi 会话的提炼统计。统计只保存在内存中，在会话开始时重置，不保存原始工具输出。
 
-统计包括工具结果数量、成功/失败/回退次数、模型尝试次数、原始与摘要字符数、压缩比、按 `chars/4` 估算的原文/摘要 Token、预计节省 Token、提炼实际消耗的 input/output/cache/total Token 和成本。数量达到 1,000 或 1,000,000 时分别使用 `k` 或 `m` 紧凑显示；耗时会根据数值显示为 `ms`、`s` 或 `min`。原文/摘要 Token 是估算值；provider 未返回 usage 时，提炼消耗 Token 或成本字段显示为不可用。
+统计包括工具结果数量、成功/失败/回退次数、模型尝试次数、原始与摘要字符数、压缩比、估算的原文/摘要 Token（启发式：CJK 字符约每字 1 token，其余文本约 4 字符 1 token）、预计节省 Token、提炼实际消耗的 input/output/cache/total Token 和成本。数量达到 1,000 或 1,000,000 时分别使用 `k` 或 `m` 紧凑显示；耗时会根据数值显示为 `ms`、`s` 或 `min`。原文/摘要 Token 是估算值；provider 未返回 usage 时，提炼消耗 Token 或成本字段显示为不可用。
 
 主要环境变量包括 `PI_DISTILL_MODEL`、`PI_DISTILL_MIN_CHARS`、`PI_DISTILL_MAX_CHARS`、`PI_DISTILL_MAX_OUTPUT_CHARS`、`PI_DISTILL_TIMEOUT_SECONDS`、`PI_DISTILL_TIMEOUT_RETRY_COUNT`、`PI_DISTILL_ERROR_RETRY_COUNT`、`PI_DISTILL_MISSED_COMPRESSION_RATIO` 和 `PI_DISTILL_SUMMARIZE_ERRORS`。
 

package/locales/fallback-renderer.json

@@ -1,80 +1,64 @@
 {
-  "summarized": {
-    "zh-CN": "✓ 已提炼",
-    "en-US": "✓ Summarized"
-  },
   "summaryFallback": {
-    "zh-CN": "↺ 已回退原文",
-    "en-US": "↺ Original restored"
+    "zh-CN": "已回退原文",
+    "en-US": "Original restored"
   },
   "disabled": {
-    "zh-CN": "○ 已禁用",
-    "en-US": "○ Disabled"
+    "zh-CN": "已禁用",
+    "en-US": "Disabled"
   },
   "off": {
-    "zh-CN": "○ 关闭",
-    "en-US": "○ Off"
+    "zh-CN": "关闭",
+    "en-US": "Off"
   },
   "original": {
-    "zh-CN": "○ 原始输出",
-    "en-US": "○ Original"
+    "zh-CN": "原始输出",
+    "en-US": "Original"
   },
   "raw": {
-    "zh-CN": "↺ RAW",
-    "en-US": "↺ RAW"
+    "zh-CN": "RAW",
+    "en-US": "RAW"
   },
   "belowThreshold": {
-    "zh-CN": "– 低于阈值",
-    "en-US": "– Below threshold"
+    "zh-CN": "低于阈值",
+    "en-US": "Below threshold"
   },
   "nonTextOutput": {
     "zh-CN": "非文本结果",
     "en-US": "Non-text output"
   },
   "readFailed": {
-    "zh-CN": "! 读取失败",
-    "en-US": "! Read failed"
+    "zh-CN": "读取失败",
+    "en-US": "Read failed"
   },
   "summaryFailed": {
-    "zh-CN": "✕ 提炼失败",
-    "en-US": "✕ Summary failed"
+    "zh-CN": "提炼失败",
+    "en-US": "Summary failed"
   },
   "chars": {
     "zh-CN": "字符",
     "en-US": "chars"
   },
   "tokens": {
-    "zh-CN": "Token",
-    "en-US": "tokens"
+    "zh-CN": "tok",
+    "en-US": "tok"
   },
   "compressionTokens": {
-    "zh-CN": "压缩消耗",
-    "en-US": "compression"
-  },
-  "tokensSaved": {
-    "zh-CN": "节省",
-    "en-US": "saved"
-  },
-  "savedPercent": {
-    "zh-CN": "节省",
-    "en-US": "saved"
+    "zh-CN": "用量",
+    "en-US": "usage"
   },
   "tool": {
     "zh-CN": "工具",
-    "en-US": "Tool"
+    "en-US": "tool"
   },
   "distill": {
     "zh-CN": "提炼",
-    "en-US": "Distill"
+    "en-US": "distill"
   },
   "expand": {
     "zh-CN": " • Ctrl+O 展开",
     "en-US": " • Ctrl+O to expand"
   },
-  "header": {
-    "zh-CN": "◇ Distill  {status}",
-    "en-US": "◇ Distill  {status}"
-  },
   "summary": {
     "zh-CN": "摘要",
     "en-US": "Summary"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-distill",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Pi tool-output distillation with file-first configuration",
   "type": "module",
   "main": "./index.ts",

package/src/fallback-renderer.ts

@@ -17,7 +17,10 @@ type AuditTone = "success" | "muted" | "dim" | "warning" | "error";
 
 type DistillAuditView = {
   lines: string[];
-  statusLabel: string;
+  /** 状态图标同时承担品牌位（替换原 ◇），是语言无关符号，不走 locale。 */
+  statusIcon: string;
+  /** 状态文字；summarized 等自解释状态为空。 */
+  statusText: string;
   statusTone: AuditTone;
 };
 
@@ -34,6 +37,8 @@ function getString(value: unknown): string | undefined {
   return typeof value === "string" && value.trim() ? value : undefined;
 }
 
+const MIN_TOOL_DURATION_DISPLAY_MS = 50;
+
 function formatDuration(milliseconds: number): string {
   return `${(milliseconds / 1000).toFixed(1)}s`;
 }
@@ -65,13 +70,13 @@ function formatCompactCount(value: number): string {
 
 function renderDistillAuditLine(audit: DistillAuditView, line: string, index: number, theme: RenderTheme): string {
   if (index === 0) {
-    const title = theme.fg("accent", theme.bold("◇ Distill"));
-    const afterTitle = line.slice("◇ Distill".length);
-    const statusIndex = afterTitle.indexOf(audit.statusLabel);
-    if (statusIndex < 0) return `${title}${theme.fg("muted", afterTitle)}`;
-    const beforeStatus = afterTitle.slice(0, statusIndex);
-    const afterStatus = afterTitle.slice(statusIndex + audit.statusLabel.length);
-    return `${title}${theme.fg("toolTitle", beforeStatus)}${theme.fg(audit.statusTone, audit.statusLabel)}${theme.fg("muted", afterStatus)}`;
+    const title = `${theme.fg(audit.statusTone, theme.bold(audit.statusIcon))}${theme.fg("accent", theme.bold(" Distill"))}`;
+    const afterTitle = line.slice(`${audit.statusIcon} Distill`.length);
+    const statusPrefix = audit.statusText ? `  ${audit.statusText}` : "";
+    if (statusPrefix && afterTitle.startsWith(statusPrefix)) {
+      return `${title}${theme.fg(audit.statusTone, statusPrefix)}${theme.fg("muted", afterTitle.slice(statusPrefix.length))}`;
+    }
+    return `${title}${theme.fg("muted", afterTitle)}`;
   }
 
   const section = line.match(/^([├└]─ )([^ ]+)(  )(.*)$/);
@@ -179,17 +184,17 @@ export function buildDistillAuditLines(
   const status = getString(details.outputSummaryStatus);
   if (!status) return undefined;
 
-  const statusViews: Record<string, { label: string; tone: AuditTone }> = {
-    summarized: { label: i18n.t("summarized"), tone: "success" },
-    "summary-fallback": { label: i18n.t("summaryFallback"), tone: "warning" },
-    disabled: { label: i18n.t("disabled"), tone: "dim" },
-    "disabled-by-config": { label: i18n.t("off"), tone: "dim" },
-    "not-requested": { label: i18n.t("original"), tone: "muted" },
-    "full-output": { label: i18n.t("raw"), tone: "warning" },
-    "below-threshold": { label: i18n.t("belowThreshold"), tone: "dim" },
-    "non-text-output": { label: i18n.t("nonTextOutput"), tone: "muted" },
-    "diagnostic-failed": { label: i18n.t("readFailed"), tone: "warning" },
-    "summary-failed": { label: i18n.t("summaryFailed"), tone: "error" },
+  const statusViews: Record<string, { icon: string; textKey?: string; tone: AuditTone }> = {
+    summarized: { icon: "✓", tone: "success" },
+    "summary-fallback": { icon: "↺", textKey: "summaryFallback", tone: "warning" },
+    disabled: { icon: "○", textKey: "disabled", tone: "dim" },
+    "disabled-by-config": { icon: "○", textKey: "off", tone: "dim" },
+    "not-requested": { icon: "○", textKey: "original", tone: "muted" },
+    "full-output": { icon: "↺", textKey: "raw", tone: "warning" },
+    "below-threshold": { icon: "–", textKey: "belowThreshold", tone: "dim" },
+    "non-text-output": { icon: "○", textKey: "nonTextOutput", tone: "muted" },
+    "diagnostic-failed": { icon: "!", textKey: "readFailed", tone: "warning" },
+    "summary-failed": { icon: "✕", textKey: "summaryFailed", tone: "error" },
   };
   const anomalies = Array.isArray(details.outputSummaryAnomalies)
     ? details.outputSummaryAnomalies.filter((value): value is string => typeof value === "string")
@@ -202,43 +207,45 @@ export function buildDistillAuditLines(
   const estimatedSummaryTokens = getFiniteNumber(details.estimatedSummaryTokens);
   const estimatedTokensSaved = getFiniteNumber(details.estimatedTokensSaved);
   const summaryTotalTokens = getFiniteNumber(details.summaryTotalTokens);
-  const compressionRatio = getFiniteNumber(details.compressionRatio);
   const compressionSavedPercent = getFiniteNumber(details.compressionSavedPercent);
   const toolExecutionMs = getFiniteNumber(details.toolExecutionMs);
   const summaryDurationMs = getFiniteNumber(details.summaryDurationMs);
   const fullOutputPath = getString(details.fullOutputPath);
   const outputRequest = getString(details.outputSummaryPrompt);
   const summaryText = getString(details.summaryText);
-  const statusView = statusViews[status] ?? { label: status, tone: "muted" as const };
-  const metrics: string[] = [];
+  const statusView = statusViews[status] ?? { icon: "○", tone: "muted" as const };
+  const statusText = statusView.textKey ? i18n.t(statusView.textKey) : status in statusViews ? "" : status;
 
+  let mainMetric = "";
   if (estimatedOriginalTokens !== undefined && estimatedSummaryTokens !== undefined) {
-    const tokenMetric = `≈${formatCompactCount(estimatedOriginalTokens)} → ${formatCompactCount(estimatedSummaryTokens)} ${i18n.t("tokens")}`;
-    const savedMetric = estimatedTokensSaved !== undefined && estimatedTokensSaved > 0
-      ? `（${i18n.t("tokensSaved")} ≈${formatCompactCount(estimatedTokensSaved)}${compressionSavedPercent !== undefined ? `，${compressionSavedPercent.toFixed(1)}%` : ""}）`
-      : "";
-    metrics.push(`${tokenMetric}${savedMetric}`);
+    mainMetric = `${formatCompactCount(estimatedOriginalTokens)} → ${formatCompactCount(estimatedSummaryTokens)} ${i18n.t("tokens")}`;
+    if (estimatedTokensSaved !== undefined && estimatedTokensSaved > 0 && compressionSavedPercent !== undefined) {
+      mainMetric += ` ↓${compressionSavedPercent.toFixed(1)}%`;
+    }
   } else if (originalChars !== undefined && summaryChars !== undefined) {
-    metrics.push(`${formatCount(originalChars)} → ${formatCount(summaryChars)} ${i18n.t("chars")}`);
-    if (compressionRatio !== undefined) metrics.push(`${compressionRatio.toFixed(2)}×`);
+    mainMetric = `${formatCount(originalChars)} → ${formatCount(summaryChars)} ${i18n.t("chars")}`;
     if (compressionSavedPercent !== undefined) {
-      metrics.push(`${compressionSavedPercent.toFixed(1)}% ${i18n.t("savedPercent")}`);
+      mainMetric += ` ↓${compressionSavedPercent.toFixed(1)}%`;
     }
   } else if (originalChars !== undefined) {
-    metrics.push(`${formatCount(originalChars)} ${i18n.t("chars")}`);
+    mainMetric = `${formatCount(originalChars)} ${i18n.t("chars")}`;
   }
+
+  const secondaryMetrics: string[] = [];
   if (summaryTotalTokens !== undefined && summaryTotalTokens > 0) {
-    metrics.push(`${i18n.t("compressionTokens")} ${formatCompactCount(summaryTotalTokens)} ${i18n.t("tokens")}`);
+    secondaryMetrics.push(`${i18n.t("compressionTokens")} ${formatCompactCount(summaryTotalTokens)}`);
   }
-  if (toolExecutionMs !== undefined && toolExecutionMs >= 50) {
-    metrics.push(`${i18n.t("tool")} ${formatDuration(toolExecutionMs)}`);
+  if (toolExecutionMs !== undefined && toolExecutionMs >= MIN_TOOL_DURATION_DISPLAY_MS) {
+    secondaryMetrics.push(`${i18n.t("tool")} ${formatDuration(toolExecutionMs)}`);
   }
-  if (summaryDurationMs !== undefined) metrics.push(`${i18n.t("distill")} ${formatDuration(summaryDurationMs)}`);
+  if (summaryDurationMs !== undefined) secondaryMetrics.push(`${i18n.t("distill")} ${formatDuration(summaryDurationMs)}`);
 
   const expandHint = expanded ? "" : i18n.t("expand");
-  const lines = [
-    i18n.t("header", { status: `${statusView.label}${metrics.length > 0 ? `  ${metrics.join(" · ")}` : ""}${expandHint}` }),
-  ];
+  const metricParts = [mainMetric, ...secondaryMetrics].filter((part) => part.length > 0);
+  const headerSegments = [`${statusView.icon} Distill`];
+  if (statusText) headerSegments.push(statusText);
+  if (metricParts.length > 0) headerSegments.push(metricParts.join(" · "));
+  const lines = [`${headerSegments.join("  ")}${expandHint}`];
   if (expanded) {
     const sections: Array<{ label: string; text: string }> = [];
     if (render.showPrompt && outputRequest) sections.push({ label: i18n.t("outputRequest"), text: outputRequest });
@@ -263,7 +270,8 @@ export function buildDistillAuditLines(
 
   return {
     lines,
-    statusLabel: statusView.label,
+    statusIcon: statusView.icon,
+    statusText,
     statusTone: statusView.tone,
   };
 }

package/src/index.ts

@@ -61,6 +61,7 @@ import {
   type DistillToolConfig,
   type OutputSummaryDecision,
 } from "./summary-utils.ts";
+import { estimateHeuristicTokens } from "./token-estimator.ts";
 
 const i18n = createTranslator(loadCatalog(new URL("../locales/index.json", import.meta.url)));
 
@@ -182,7 +183,7 @@ type SummaryDiagnostics = {
   summaryCacheWriteTokens?: number;
   summaryTotalTokens?: number;
   summaryCost?: number;
-  /** 原始输出与最终摘要的 Token 数是按 Pi 的 chars/4 规则估算的。 */
+  /** 原始输出与最终摘要的展示用 Token 数，由分段启发式估算（CJK 约每字 1 token，其余约 4 字符 1 token）。 */
   estimatedOriginalOutputTokens?: number;
   estimatedSummaryTokens?: number;
   estimatedTokensSaved?: number;
@@ -275,18 +276,12 @@ function getSummaryUsageDiagnostics(usage: SummaryUsage | undefined): Pick<
   };
 }
 
-const ESTIMATED_CHARS_PER_TOKEN = 4;
-
-function estimateTextTokens(text: string): number {
-  return Math.ceil(text.length / ESTIMATED_CHARS_PER_TOKEN);
-}
-
 function getTokenCompressionDiagnostics(
   originalOutput: string,
   finalOutput: string,
 ): Pick<SummaryDiagnostics, "estimatedOriginalOutputTokens" | "estimatedSummaryTokens" | "estimatedTokensSaved"> {
-  const estimatedOriginalOutputTokens = estimateTextTokens(originalOutput);
-  const estimatedSummaryTokens = estimateTextTokens(finalOutput);
+  const estimatedOriginalOutputTokens = estimateHeuristicTokens(originalOutput);
+  const estimatedSummaryTokens = estimateHeuristicTokens(finalOutput);
   return {
     estimatedOriginalOutputTokens,
     estimatedSummaryTokens,

package/src/token-estimator.ts

@@ -0,0 +1,40 @@
+/**
+ * 展示用 token 估算。跨模型没有通用的精确 tokenizer（实测误差 24%~65%），
+ * Anthropic 官方 JS 包的词表也停留在 Claude 2 时代，所以不引入第三方
+ * tokenizer 依赖，统一使用分段启发式：
+ *
+ * - CJK、假名、谚文和全角字符：现代 BPE 分词器通常每字 1 token；
+ * - 其余字符：约 4 字符 1 token（与 Pi 的 chars/4 规则一致）。
+ *
+ * 相比纯 chars/4，中文内容不再被低估约 4 倍。
+ */
+
+/** 现代 BPE 分词器通常按单字编码的 Unicode 区间（CJK/假名/谚文/全角）。 */
+const DENSE_SCRIPT_RANGES: ReadonlyArray<readonly [number, number]> = [
+  [0x2e80, 0x9fff], // CJK 部首补充、假名、CJK 统一表意文字等
+  [0xac00, 0xd7af], // 谚文音节
+  [0xf900, 0xfaff], // CJK 兼容表意文字
+  [0xff00, 0xffef], // 全角字符
+  [0x20000, 0x2fa1f], // CJK 扩展 B–F 及兼容补充（代理对区间）
+];
+
+/** 判断码点是否属于 dense script 区间（这些文字通常每字 1 token）。 */
+function isDenseScript(codePoint: number): boolean {
+  return DENSE_SCRIPT_RANGES.some(([start, end]) => codePoint >= start && codePoint <= end);
+}
+
+const ASCII_CHARS_PER_TOKEN = 4;
+
+/**
+ * 分段启发式估算 token 数：dense script 字符按 1 token/字，其余按
+ * 4 字符 1 token。结果是展示用估算值，不保证与任何具体模型 tokenizer 一致。
+ */
+export function estimateHeuristicTokens(text: string): number {
+  let dense = 0;
+  let rest = 0;
+  for (const char of text) {
+    if (isDenseScript(char.codePointAt(0) ?? 0)) dense += 1;
+    else rest += 1;
+  }
+  return dense + Math.ceil(rest / ASCII_CHARS_PER_TOKEN);
+}