recallnest 2.3.0 → 2.5.4

package/src/memory-lint.ts

@@ -9,6 +9,9 @@
  * 2. Duplicates — near-identical memories by vector cosine similarity
  * 3. Stale — memories never or rarely accessed and old enough to review
  * 4. Orphans — memories with missing scope or broken consolidation links
+ * 5. Weak lessons — workflow patterns lacking any action verbs (inspired by
+ *    KarryViber/Orb memory-lint heuristic — a "lesson" should prescribe an
+ *    action, not describe a phenomenon)
  *
  * Produces a health score (0-100) and a human-readable report.
  */
@@ -16,6 +19,7 @@
 import type { MemoryEntry, MemoryStore } from "./store.js";
 import { cosineSimilarity } from "./multi-vector.js";
 import { parseEvolution, isActiveMemory } from "./memory-evolution.js";
+import { deriveUsageStatus, isUsageSignalActive } from "./usage-tracker.js";
 
 // ---------------------------------------------------------------------------
 // Types
@@ -24,7 +28,7 @@ import { parseEvolution, isActiveMemory } from "./memory-evolution.js";
 export type LintSeverity = "info" | "warning" | "error";
 
 export interface LintFinding {
-  check: string;     // e.g., "contradiction", "duplicate", "stale", "orphan"
+  check: string;     // e.g., "contradiction", "duplicate", "stale", "orphan", "weakLesson"
   severity: LintSeverity;
   detail: string;
   memoryIds: string[];
@@ -34,17 +38,22 @@ export interface MemoryLintReport {
   findings: LintFinding[];
   healthScore: number;  // 0-100
   totalScanned: number;
+  /** True if the scan hit the 10000-entry cap (older entries were not analyzed). */
+  scanLimited: boolean;
   timestamp: string;
   summary: {
     contradictions: number;
     duplicates: number;
     staleMemories: number;
     orphans: number;
+    weakLessons: number;
+    /** P0 B-1: cold 记忆数(被反复 surface 却从未被引用)。 */
+    coldMemories: number;
   };
 }
 
 export interface LintDeps {
-  store: Pick<MemoryStore, "list">;
+  store: Pick<MemoryStore, "list" | "getVectors">;
   scope?: string;
   verbose?: boolean;
 }
@@ -78,6 +87,32 @@ const STALE_DAYS = 90;
 
 const STALE_MS = STALE_DAYS * 24 * 60 * 60 * 1000;
 
+/**
+ * Categories where the action-verb lesson-quality heuristic applies.
+ *
+ * Workflow `patterns` are meant to be reusable "do it this way next time"
+ * prescriptions. A pattern whose text contains no imperative/action verb is
+ * usually too abstract for the agent to reuse and is flagged as a weak lesson.
+ *
+ * We intentionally do NOT apply this to `cases` (problem/solution pairs whose
+ * text may be descriptive), `events` (append-only log), `profile`/`preferences`
+ * (static facts), or `entities` (noun-centric).
+ *
+ * Inspired by KarryViber/Orb `lib/holographic/memory-lint.py`.
+ */
+const LESSON_CATEGORIES = new Set(["patterns"]);
+
+/**
+ * Regex matching action/imperative verbs used to judge whether a workflow
+ * pattern is actionable. Bilingual: English word-bounded; Chinese substring
+ * (CJK has no word boundaries).
+ *
+ * Intentionally wide — false-negatives here are benign (a weak-lesson flag is
+ * an `info` hint, not an error). We only want to catch patterns that are pure
+ * observations/descriptions with no prescriptive action at all.
+ */
+const ACTION_VERBS = /\b(check|verify|validate|ensure|use|avoid|implement|run|call|set|add|remove|update|refactor|test|document|commit|push|rollback|escalate|stop|retry|install|configure|create|delete|write|read|spawn|enable|disable|init|start|restart|launch|store|fetch|query|log|match|replace|generate|prefer|require|invoke|route|build|deploy|review|fix|parse|apply|assert|include|exclude|skip|catch|throw|wrap|mock|mount|import|export|inject|extract|prepend|append|return|raise|cache|clear|reset|refresh|merge|split|rename|move|copy|scan|patrol|trigger|schedule|observe|record|emit|publish|subscribe|notify|confirm|choose|select|filter|sort|group|reduce|map|forward|resolve|reject)\b|使用|避免|检查|验证|确认|添加|删除|修改|改为|回滚|停止|重试|安装|配置|运行|调用|启用|禁用|生成|初始化|启动|重启|存储|获取|查询|记录|匹配|替换|创建|写入|读取|强制|禁止|必须|应当|保留|抛出|捕获|重构|部署|审查|修复|解析|应用|断言|包含|排除|跳过|合并|拆分|重命名|移动|复制|刷新|重置|清理|缓存|注入|提取|追加|遍历|订阅|发布|通知|选择|筛选|排序|触发|观察/iu;
+
 // ---------------------------------------------------------------------------
 // Contradiction Detection (inline copy from consolidation-engine.ts)
 // ---------------------------------------------------------------------------
@@ -279,6 +314,87 @@ function findOrphans(entries: MemoryEntry[]): LintFinding[] {
   return findings;
 }
 
+// ---------------------------------------------------------------------------
+// Check 5: Weak Lessons (action-verb heuristic)
+// ---------------------------------------------------------------------------
+
+/**
+ * Identify "weak lessons" — workflow `patterns` whose text contains no
+ * imperative/action verb. Such entries are usually descriptive rather than
+ * prescriptive, so the agent cannot act on them next time.
+ *
+ * Severity is `info` (not warning/error): this is a quality hint, not a bug.
+ * Low penalty (see computeHealthScore) reflects that.
+ *
+ * Inspired by KarryViber/Orb `lib/holographic/memory-lint.py`.
+ */
+function findWeakLessons(entries: MemoryEntry[]): LintFinding[] {
+  const findings: LintFinding[] = [];
+
+  for (const entry of entries) {
+    if (!LESSON_CATEGORIES.has(entry.category)) continue;
+
+    // Empty/whitespace text is caught elsewhere; skip to avoid double-flagging.
+    if (!entry.text || entry.text.trim() === "") continue;
+
+    if (ACTION_VERBS.test(entry.text)) continue;
+
+    findings.push({
+      check: "weakLesson",
+      severity: "info",
+      detail: `Pattern without action verbs (too abstract to reuse): "${truncate(entry.text, 60)}"`,
+      memoryIds: [entry.id],
+    });
+  }
+
+  return findings;
+}
+
+/**
+ * Check 6 (P0 B-1 观察): cold memories — 被反复 surface(accessCount ≥ 6)却
+ * 从未被 reconstruction [src:ID] 引用的记忆,召回位污染信号。
+ *
+ * 聚合为单条 finding(cold 可能有几千条,逐条 finding 会淹没报告);
+ * memoryIds 取 injection 最高的前 20 条供抽查。severity 永远是 info:
+ * B-1 是纯观察阶段,降权/归档要等观察数据校准后的 B-2。
+ */
+function findColdMemories(entries: MemoryEntry[]): { findings: LintFinding[]; coldCount: number } {
+  // use 信号未采集(reconstruction flag 关)→ cold 全是假阳性,跳过且不扣健康分。
+  if (!isUsageSignalActive()) {
+    return { findings: [], coldCount: 0 };
+  }
+
+  const cold: Array<{ id: string; injection: number }> = [];
+
+  for (const entry of entries) {
+    if (deriveUsageStatus(entry) !== "cold") continue;
+    let injection = 0;
+    try {
+      const meta = JSON.parse(entry.metadata || "{}") as Record<string, unknown>;
+      injection = typeof meta.accessCount === "number" ? meta.accessCount : 0;
+    } catch { /* keep 0 */ }
+    cold.push({ id: entry.id, injection });
+  }
+
+  if (cold.length === 0) return { findings: [], coldCount: 0 };
+
+  const pct = entries.length > 0 ? ((cold.length / entries.length) * 100).toFixed(1) : "0";
+  const topIds = cold
+    .sort((a, b) => b.injection - a.injection)
+    .slice(0, 20)
+    .map(c => c.id);
+
+  return {
+    coldCount: cold.length,
+    findings: [{
+      check: "coldMemory",
+      severity: "info",
+      detail: `${cold.length} cold memories (${pct}% of scanned): repeatedly surfaced (accessCount >= 6) but never cited by reconstruction. Top-20 by injection in memoryIds. Observation only (B-1) — no action taken.`,
+      memoryIds: topIds,
+    }],
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Health Score
 // ---------------------------------------------------------------------------
@@ -291,16 +407,36 @@ function findOrphans(entries: MemoryEntry[]): LintFinding[] {
  * - Duplicates: -5 each (waste and confusion risk)
  * - Stale: -0.5 each (minor, many are expected)
  * - Orphans: -3 each (moderate, broken references)
+ * - Weak lessons: -2 each (pattern lacks action verbs, reuse-unfriendly)
+ * - Cold memories (P0 B-1): 按占比而非条数 — 超过 10% 的部分每个百分点 -1,
+ *   封顶 -10。cold 是反复 surface 零引用的召回污染信号,量大但单条危害小,
+ *   按条扣会瞬间扣穿。纯观察反馈,不触发任何动作。
+ *
+ * `weakLessons`/`coldMemories` are optional on the input type for backward
+ * compatibility — callers constructing `summary` without them still get a
+ * correct score.
  */
 export function computeHealthScore(
-  summary: MemoryLintReport["summary"],
-  _total: number,
+  summary: {
+    contradictions: number;
+    duplicates: number;
+    staleMemories: number;
+    orphans: number;
+    weakLessons?: number;
+    coldMemories?: number;
+  },
+  total: number,
 ): number {
+  const coldPct = total > 0 ? ((summary.coldMemories ?? 0) / total) * 100 : 0;
+  const coldPenalty = Math.min(10, Math.max(0, Math.floor(coldPct - 10)));
+
   const penalty =
     summary.contradictions * 10 +
     summary.duplicates * 5 +
     Math.floor(summary.staleMemories * 0.5) +
-    summary.orphans * 3;
+    summary.orphans * 3 +
+    (summary.weakLessons ?? 0) * 2 +
+    coldPenalty;
 
   return Math.max(0, Math.min(100, 100 - penalty));
 }
@@ -318,30 +454,42 @@ export function computeHealthScore(
  */
 export async function runMemoryLint(deps: LintDeps): Promise<MemoryLintReport> {
   const scopeFilter = deps.scope ? [deps.scope] : undefined;
-  const allEntries = await deps.store.list(scopeFilter, undefined, 10000, 0);
+  const SCAN_LIMIT = 10000;
+  const allEntries = await deps.store.list(scopeFilter, undefined, SCAN_LIMIT, 0);
+  const scanLimited = allEntries.length >= SCAN_LIMIT;
 
   // Filter to active entries only
-  const entries = allEntries.filter(e => isActiveMemory(e.metadata));
+  const activeEntries = allEntries.filter(e => isActiveMemory(e.metadata));
+
+  // list() 为性能不返回向量(vector: []);矛盾/去重检查靠向量算相似度,否则恒为 0、
+  // 静默失效并给出虚假 all-clear。补回真实向量再检查(stale/orphan/weakLesson 不依赖向量)。
+  const vectorMap = await deps.store.getVectors(activeEntries.map(e => e.id));
+  const entries = activeEntries.map(e => ({ ...e, vector: vectorMap.get(e.id) ?? [] }));
 
   // Run all checks
   const contradictions = findContradictions(entries);
   const duplicates = findDuplicates(entries);
   const stale = findStaleMemories(entries);
   const orphans = findOrphans(entries);
+  const weakLessons = findWeakLessons(entries);
+  const coldResult = findColdMemories(entries);
 
-  const findings = [...contradictions, ...duplicates, ...stale, ...orphans];
+  const findings = [...contradictions, ...duplicates, ...stale, ...orphans, ...weakLessons, ...coldResult.findings];
 
   const summary = {
     contradictions: contradictions.length,
     duplicates: duplicates.length,
     staleMemories: stale.length,
     orphans: orphans.length,
+    weakLessons: weakLessons.length,
+    coldMemories: coldResult.coldCount,
   };
 
   return {
     findings,
     healthScore: computeHealthScore(summary, entries.length),
     totalScanned: entries.length,
+    scanLimited,
     timestamp: new Date().toISOString(),
     summary,
   };
@@ -358,7 +506,7 @@ export function formatMemoryLintReport(report: MemoryLintReport): string {
   // All clear case
   if (report.findings.length === 0) {
     lines.push("Memory Lint: All Clear!");
-    lines.push(`Scanned: ${report.totalScanned} active memories`);
+    lines.push(`Scanned: ${report.totalScanned} active memories${report.scanLimited ? " (达扫描上限 10000，更早记忆未分析)" : ""}`);
     lines.push(`Health Score: ${report.healthScore}/100`);
     return lines.join("\n");
   }
@@ -366,7 +514,7 @@ export function formatMemoryLintReport(report: MemoryLintReport): string {
   // Header
   lines.push(`Memory Lint Report (${report.timestamp})`);
   lines.push("=".repeat(44));
-  lines.push(`Scanned: ${report.totalScanned} active memories`);
+  lines.push(`Scanned: ${report.totalScanned} active memories${report.scanLimited ? " (达扫描上限 10000，更早记忆未分析)" : ""}`);
   lines.push("");
 
   // Contradictions
@@ -422,6 +570,31 @@ export function formatMemoryLintReport(report: MemoryLintReport): string {
     lines.push("");
   }
 
+  // Weak Lessons
+  if (report.summary.weakLessons > 0) {
+    const items = report.findings.filter(f => f.check === "weakLesson");
+    lines.push(`Weak Lessons (${items.length}):`);
+    if (items.length <= 5) {
+      for (const f of items) {
+        lines.push(`  - ${f.detail} [${f.memoryIds.map(id => id.slice(0, 8)).join(", ")}]`);
+      }
+    } else {
+      lines.push(`  - ${items.length} patterns without action verbs (consider rewriting or promoting)`);
+    }
+    lines.push("");
+  }
+
+  // Cold Memories (P0 B-1 observation)
+  if (report.summary.coldMemories > 0) {
+    const item = report.findings.find(f => f.check === "coldMemory");
+    lines.push(`Cold Memories (${report.summary.coldMemories}):`);
+    if (item) {
+      lines.push(`  - ${item.detail}`);
+      lines.push(`  - top by injection: ${item.memoryIds.map(id => id.slice(0, 8)).join(", ")}`);
+    }
+    lines.push("");
+  }
+
   // Health score
   lines.push(`Health Score: ${report.healthScore}/100`);
 

package/src/memory-output.ts

@@ -5,6 +5,7 @@ import { extractMemoryProvenance } from "./memory-boundaries.js";
 import { extractMultiVectorText } from "./multi-vector.js";
 import { parseNarrative } from "./narrative-schema.js";
 import { getConfidence, getConfidenceMetadata } from "./confidence-tracker.js";
+import { estimateTokens } from "./context-collapse-renderer.js";
 
 interface MemoryMetadata {
   source?: string;
@@ -217,6 +218,49 @@ function pickBestSnippet(query: string, text: string): string {
   return cleanSnippet(bestSentence);
 }
 
+/**
+ * adaptive 专用 query-aware snippet：取匹配词周围窗口（而非整句/开头），保证匹配证据可见。
+ * 解决 pickBestSnippet 的两个退化（Codex 审点4 C 版 P2）：① 短缩写 query（如 "CI"）
+ * extractTerms 提取不到词 → 退回开头；② 匹配在超长句/log 行后段 → cleanSnippet 从头截断丢匹配。
+ * 只给 adaptive 用，不改 pickBestSnippet（避免影响 normal/explain 的既有行为）。
+ */
+function adaptiveSnippet(query: string, text: string): string {
+  const clean = text.replace(/\s+/g, " ").trim();
+  const lower = clean.toLowerCase();
+  // 短缩写时 extractTerms 可能为空，用 raw query 兜底直接定位。
+  const terms = extractTerms(query);
+  const isFallback = terms.length === 0;
+  const searchTerms = (isFallback ? [query.toLowerCase().trim()] : terms).filter(Boolean);
+
+  let matchIdx = -1;
+  for (const term of searchTerms) {
+    // term 定位策略：
+    // - ASCII 字母数字短缩写（CI/PR）用 word boundary，避免误匹配 "specific" 里的 "ci"（Codex C P2）
+    // - CJK 单字 / 符号 query（猫 / C++ / C#）：JS \b 只认 ASCII、对它们永不匹配，回到 substring（收尾审 P2-1）
+    // - 正常 extractTerms 词：substring（容忍词根 / 词形）
+    // 已知限制（收尾审 P2-2，留后续）：多 term 取最早出现，未按 term rarity/coverage 给窗口打分——
+    // 泛词在前、判别词在窗口外时可能漏掉判别词。第一版接受。
+    let idx: number;
+    if (isFallback && /^[a-z0-9]+$/i.test(term)) {
+      const m = new RegExp(`\\b${term}\\b`, "i").exec(clean);
+      idx = m ? m.index : -1;
+    } else {
+      idx = lower.indexOf(term);
+    }
+    if (idx >= 0 && (matchIdx < 0 || idx < matchIdx)) matchIdx = idx;
+  }
+  if (matchIdx < 0) return pickBestSnippet(query, text); // 完全无匹配 → 退回整句最佳
+
+  const WINDOW_BEFORE = 60;
+  const WINDOW_AFTER = 180;
+  const start = Math.max(0, matchIdx - WINDOW_BEFORE);
+  const end = Math.min(clean.length, matchIdx + WINDOW_AFTER);
+  let snip = clean.slice(start, end).trim();
+  if (start > 0) snip = "…" + snip;
+  if (end < clean.length) snip = snip + "…";
+  return snip;
+}
+
 function normalizeRecallText(result: RetrievalResult): string {
   const source = getSourceLabel(result);
   if (source !== "asset") return result.entry.text;
@@ -282,7 +326,7 @@ function buildSearchRow(index: number, query: string, result: RetrievalResult):
   return [
     String(index + 1).padEnd(2),
     result.entry.id.slice(0, 8).padEnd(8),
-    `${(result.score * 100).toFixed(0)}%`.padEnd(5),
+    `${(result.score * 100).toFixed(1)}%`.padEnd(6),
     getCategoryLabel(result).padEnd(12),
     getTierLabel(result).padEnd(10),
     getSourceLabel(result).padEnd(7),
@@ -308,7 +352,7 @@ export function formatBriefResults(
   for (let i = 0; i < results.length; i++) {
     const r = results[i];
     const excerpt = extractBriefExcerpt(r);
-    lines.push(`#${i + 1} ${r.entry.id.slice(0, 8)} ${Math.round(r.score * 100)}% — ${excerpt}`);
+    lines.push(`#${i + 1} ${r.entry.id.slice(0, 8)} ${(r.score * 100).toFixed(1)}% — ${excerpt}`);
   }
   return lines.join("\n");
 }
@@ -382,6 +426,73 @@ export function formatSearchResults(
   return lines.join("\n");
 }
 
+const ADAPTIVE_TOKEN_BUDGET = 8000;
+const ADAPTIVE_FULL_SCORE = 0.85;
+
+/**
+ * P-fidelity (点4): adaptive 保真度渲染 — 借鉴 RepoPrompt CE「保真度阶梯」的内核
+ * （按相关性分配保真度 + token 预算），但 search-native 实现。
+ *
+ * 为什么不复用 resume_context 的 collapseResults（CC+Codex 共识，Codex 三轮 review）：
+ * collapse 的 l0 floor / 按 score 重排 / query-agnostic fallback 都是为 resume 场景定制，
+ * 搬到 search 连续撞三个 P2（隐藏过滤 / 覆盖 highlight 排序 / 丢失匹配证据）。search 的语义
+ * 不同——结果是"为什么这条命中"，必须 query-aware；故第一版自实现两档：
+ *   - 高相关（score ≥ 0.85）→ 全文
+ *   - 其余 → query-aware snippet（pickBestSnippet，显示匹配证据）
+ * 按 handler 传入顺序（normal=score / highlight=contextual）依次填入 token 预算，超预算时
+ * 全文降级 snippet、snippet 仍超则停止（剩余计入 omitted）。完整三档保真度阶梯（每条记忆预存
+ * 多档摘要 + 预缓存 token 成本）留作后续大改。
+ */
+export function formatCollapsedResults(
+  results: RetrievalResult[],
+  context: RenderContext,
+): string {
+  if (results.length === 0) return "No results found.";
+
+  const rendered: string[] = [];
+  let tokensUsed = 0;
+  let shown = 0;
+
+  // 保留传入顺序（不重排）：handler 已按 normal=score / highlight=contextual 排好。
+  for (const r of results) {
+    const isFull = r.score >= ADAPTIVE_FULL_SCORE;
+    let level = isFull ? "FULL" : "SNIP";
+    // query-aware：snippet 取匹配词周围窗口、显示匹配证据（search 是"为什么命中"）。
+    let text = isFull ? r.entry.text : adaptiveSnippet(context.query, r.entry.text);
+    let tokens = estimateTokens(text);
+
+    if (tokensUsed + tokens > ADAPTIVE_TOKEN_BUDGET) {
+      if (isFull) {
+        // 全文超预算 → 降级为 query-aware snippet
+        text = adaptiveSnippet(context.query, r.entry.text);
+        level = "SNIP";
+        tokens = estimateTokens(text);
+      }
+      if (tokensUsed + tokens > ADAPTIVE_TOKEN_BUDGET) break; // snippet 仍超 → 停止，剩余计入 omitted
+    }
+    tokensUsed += tokens;
+    shown++;
+
+    // 定位信息：id / score / category / source（Codex 约束：search 结果必须可定位）
+    const id = r.entry.id.slice(0, 8);
+    const scorePct = `${(r.score * 100).toFixed(0)}%`;
+    rendered.push(`[${level}] ${id}  ${scorePct}  ${getCategoryLabel(r)}  ${getSourceLabel(r)}`);
+    rendered.push(text);
+    rendered.push("");
+  }
+
+  const omitted = results.length - shown;
+  const lines = [
+    `Query   : ${context.query}`,
+    `Profile : ${context.profile}`,
+    `Mode    : adaptive — full text for score ≥ ${ADAPTIVE_FULL_SCORE}, query-aware snippet otherwise, ${ADAPTIVE_TOKEN_BUDGET}-token budget`,
+    `Shown   : ${shown} of ${results.length}${omitted > 0 ? ` (${omitted} omitted: token budget)` : ""}`,
+    "",
+    ...rendered,
+  ];
+  return lines.join("\n").trimEnd();
+}
+
 export function formatExplainResults(
   results: RetrievalResult[],
   context: RenderContext,
@@ -398,7 +509,7 @@ export function formatExplainResults(
 
   for (let i = 0; i < results.length; i++) {
     const result = results[i];
-    const score = `${(result.score * 100).toFixed(0)}%`;
+    const score = `${(result.score * 100).toFixed(1)}%`;
     const retrieval = getRetrievalPath(result);
     const file = getFileLabel(result);
     const session = getSessionLabel(result);