recallnest 2.6.1 → 3.0.1

package/src/capture-engine.ts

@@ -461,6 +461,15 @@ export async function writeDurableEntry(
   entry: MemoryEntry;
   disposition: "stored" | "updated" | "deduped" | "promoted" | "conflict";
   conflictId?: string;
+  /**
+   * True when this write REPLACED the content of an existing canonical row
+   * (a belief version was archived). Two different paths can do that — the
+   * latest-wins branch reports `updated`, while the deterministic-id branch
+   * below still reports `stored` — so disposition alone cannot tell them
+   * apart. Consumers that mirror memory content elsewhere (KG triples) must
+   * drop the old projection when this is set.
+   */
+  supersededPrevious?: boolean;
 }> {
   const matches = await findCanonicalMatches(deps.store, params.canonicalKey);
   const categoryMatches = matches.filter((entry) => entry.category === params.category);
@@ -605,6 +614,7 @@ export async function writeDurableEntry(
     return {
       entry: updated,
       disposition: params.promotedFrom ? "promoted" : "updated",
+      supersededPrevious: true,
     };
   }
 
@@ -617,6 +627,7 @@ export async function writeDurableEntry(
     ? deterministicId(params.scope, params.canonicalKey)
     : undefined;
   let metadataForStore = params.metadata;
+  let supersededPrevious = false;
 
   if (targetId && deps.store.getById) {
     const rowAtTargetId = await deps.store.getById(targetId);
@@ -629,6 +640,7 @@ export async function writeDurableEntry(
         { now },
       );
       metadataForStore = buildSupersedingBeliefMetadata(params.metadata, previousEvo, historyId, now);
+      supersededPrevious = true;
     }
   }
 
@@ -647,6 +659,7 @@ export async function writeDurableEntry(
   return {
     entry: stored,
     disposition: params.promotedFrom ? "promoted" : "stored",
+    supersededPrevious,
   };
 }
 
@@ -1106,7 +1119,7 @@ export async function persistMemory(
     metadata = JSON.stringify(parsed);
   }
 
-  const { entry, disposition, conflictId } = await writeDurableEntry(deps, {
+  const { entry, disposition, conflictId, supersededPrevious } = await writeDurableEntry(deps, {
     text: input.text,
     vector,
     category: input.category,
@@ -1124,9 +1137,18 @@ export async function persistMemory(
   const entryPrivacyTier = parsePrivacyTier(metadata);
   const kgAllowed = entryPrivacyTier !== "ephemeral" && entryPrivacyTier !== "private";
   if (deps.kgExtractor && disposition !== "deduped" && kgAllowed) {
-    deps.kgExtractor
-      .extractAndStore(input.text, entry.id, resolvedScope)
-      .catch(() => {}); // Silently ignore — KG extraction must never block memory writes
+    const kgExtractor = deps.kgExtractor;
+    // When this write replaced existing content, the edges extracted from the OLD
+    // text are still in the graph: triple ids are content-derived, so re-extracting
+    // merely adds new rows beside the stale ones and both compete during traversal.
+    // Purge first, then re-extract — strictly ordered, or the purge would race ahead
+    // and delete what we just wrote.
+    const purged = supersededPrevious
+      ? kgExtractor.purgeForMemory(entry.id)
+      : Promise.resolve();
+    purged
+      .then(() => kgExtractor.extractAndStore(input.text, entry.id, resolvedScope))
+      .catch(() => {}); // Silently ignore — KG upkeep must never block memory writes
   }
 
   // F-1: Audit log — record store operation (non-blocking, silent on failure)

package/src/cli.ts

@@ -38,9 +38,18 @@ import {
 } from "./ingest.js";
 import { runDoctor, formatDoctorResults } from "./doctor.js";
 import { persistCaseMemory, persistMemory, persistWorkflowPattern } from "./capture-engine.js";
-import { scanMemoryPromotions, buildPromoteScanDeps, formatPromoteScanResult } from "./memory-promotion.js";
+import {
+  scanMemoryPromotions,
+  scanSynthesisPromotions,
+  buildPromoteScanDeps,
+  formatPromoteScanResult,
+  formatSynthesisPromoteResult,
+} from "./memory-promotion.js";
 import { runDream, formatDreamResult, formatDreamMetrics, DEFAULT_DREAM_CONFIG, classifyDreamFailure, shouldBlockDreamRun, partitionAutoDreamScopes } from "./dream-pipeline.js";
-import { listScopesAboveThreshold } from "./activity-counter.js";
+import { dreamBudgetMs } from "./env-config.js";
+import { listScopesAboveThreshold, pruneWriteCounts } from "./activity-counter.js";
+import { isTranscriptScope } from "./memory-boundaries.js";
+import { maybeRunGc } from "./auto-gc.js";
 import {
   CaseMemoryInputSchema,
   type CaseMemoryInput,
@@ -62,6 +71,7 @@ import { WorkflowObservationOutcomeSchema } from "./workflow-observation-schema.
 import { buildWorkflowEvidence, buildWorkflowObservationRecord, inspectWorkflowDashboard, inspectWorkflowHealth } from "./workflow-observation-engine.js";
 import { formatWorkflowEvidencePack, formatWorkflowHealthDashboard, formatWorkflowHealthReport, formatWorkflowObservationSaved } from "./workflow-observation-output.js";
 import { WorkflowObservationStore } from "./workflow-observation-store.js";
+import { applyUtilityToMeta, computeMemoryUtility, formatMemoryUtility } from "./memory-utility.js";
 import {
   applyWorkflowObservationScopeSuggestions,
   formatWorkflowObservationScopeReview,
@@ -702,6 +712,28 @@ program
     console.log(formatPromoteScanResult(result));
   });
 
+program
+  .command("promote-synthesis")
+  .description("扫描 dream 合成出的结论（evidence 层），把证据充分的那些晋升出一份 durable 副本")
+  .requiredOption("--scope <scope>", "要扫描的 scope，如 memory")
+  .option("--apply", "执行晋升；默认 dry-run 只预览")
+  .option("--min-evidence <n>", "最少几条仍存活的不同证据（默认 2）", "2")
+  .option("--min-importance <n>", "合成条目自身的最低 importance（默认 0.6）", "0.6")
+  .action(async (options) => {
+    const config = loadConfig();
+    const { store, embedder } = createComponents(config);
+    const conflictStore = new ConflictCandidateStore();
+    const deps = buildPromoteScanDeps({ store, embedder, conflictStore });
+    const minDistinctEvidence = Number(options.minEvidence);
+    const minImportance = Number(options.minImportance);
+    const result = await scanSynthesisPromotions(deps, options.scope, {
+      dryRun: !options.apply,
+      ...(Number.isFinite(minDistinctEvidence) ? { minDistinctEvidence } : {}),
+      ...(Number.isFinite(minImportance) ? { minImportance } : {}),
+    });
+    console.log(formatSynthesisPromoteResult(result));
+  });
+
 program
   .command("export <query>")
   .description("导出检索与蒸馏结果到 markdown/json")
@@ -1360,7 +1392,7 @@ program
 program
   .command("ingest")
   .description("导入对话记录到索引")
-  .option("-s, --source <source>", "数据源: cc / codex / gemini / kimi / memory / desktop / all", "all")
+  .option("-s, --source <source>", "数据源: cc / codex / gemini / kimi / memory / desktop / minis / all", "all")
   .option("-l, --limit <n>", "限制处理文件数（调试用）")
   .option("-v, --verbose", "详细输出")
   .option("--no-dedup", "跳过向量去重")
@@ -1503,6 +1535,27 @@ program
       }
     }
 
+    // Minis (iPhone) —— 手机 agent 自己把对话写进这个目录，ingest 顺带扫走。
+    // 为什么是独立源而不是复用 desktop：scope 前缀决定了 memory-boundaries 怎么降权，
+    // 混在 cc: 里的话来源在库里就分辨不出来（那正是 2026-08-12 起挂着的缺口）。
+    if (source === "all" || source === "minis") {
+      const minisSource = config.sources.minis;
+      if (minisSource) {
+        const minisPath = resolveSourcePath(minisSource.path, "minis");
+        if (existsSync(minisPath)) {
+          console.log("📱 导入 Minis 对话...");
+          const r = await ingestCCTranscripts(store, embedder, minisPath, {
+            ...ingestOpts,
+            scopePrefix: "minis",
+          });
+          results.push(r);
+          console.log(`  ✅ Minis: ${formatIngestSummary(r)}`);
+        } else if (source === "minis") {
+          console.log(`⚠️  Minis 目录不存在: ${minisPath}`);
+        }
+      }
+    }
+
     // Memory markdown files
     if (source === "all" || source === "memory") {
       console.log("📚 导入记忆文件...");
@@ -1587,10 +1640,11 @@ program
 program
   .command("dream")
   .description("运行 dream 离线整固管线 (Orient/Gather/Consolidate/Prune)")
-  .option("--scope <scope>", "目标 scope(前缀或精确)", "cc")
+  .option("--scope <scope>", "目标 scope(默认前缀家族匹配;加 --exact-scope 变精确)", "cc")
+  .option("--exact-scope", "把 --scope 当成一个具体 scope 名精确匹配,不做前缀家族展开(调度入口必带)")
   .option("--auto", "扫描 activity-counter 中写计数达标的所有 scope，逐个跑 dream（单 scope 失败不阻断其他）")
   .option("--force", "跳过 min-writes 门槛强制运行")
-  .action(async (options: { scope: string; auto?: boolean; force?: boolean }) => {
+  .action(async (options: { scope: string; auto?: boolean; force?: boolean; exactScope?: boolean }) => {
     const config = loadConfig();
     const { store, embedder, llm } = createComponents(config);
     // Same activity-stats file the store writes per-scope counts to (beside the LanceDB dir).
@@ -1602,20 +1656,76 @@ program
       const eligible = listScopesAboveThreshold(DEFAULT_DREAM_CONFIG.minWritesForDream, { statsPath: activityStatsPath });
       // 巨型 scope 交给专用 schedule,不占日常轮次的 wall-clock 预算(理由见
       // DreamConfig.autoExcludeScopes:07-30 那轮 6h 只跑了 15/507,而 memory 独占 79% 的条数)。
-      const { run: scopes, deferred } = partitionAutoDreamScopes(eligible, DEFAULT_DREAM_CONFIG.autoExcludeScopes);
+      const { run: nonMemoryScopes, deferred } = partitionAutoDreamScopes(eligible, DEFAULT_DREAM_CONFIG.autoExcludeScopes);
       if (deferred.length > 0) {
         // 不静默:被挪走的 scope 必须在日志里看得见,否则"今天怎么没跑 memory"只能靠翻代码回答。
         // 前缀刻意用 `[dream]` 而不是 `dream --auto:`——后者是 dream-checkup.sh 抓「达标 scope 数」
         // 的锚点,多一行同前缀的会混进它的取数。
         console.log(`[dream] ${deferred.length} 个 scope 交给专用 schedule，本轮不跑 → ${deferred.join(", ")}`);
       }
+
+      // 2026-08-16 策略排除（互审 C2/K5）：与上面的 deferred **语义不同,别合并**——
+      // deferred 是"别处会跑",这里是"没有任何 schedule 会跑"。日志措辞必须分开,
+      // 否则将来查"pivot 怎么没巩固"会被误导去找那个并不存在的专用 job。
+      // 同时清写计数:runDream 对它返回 skipped 不会 resetWriteCount,不清则计数无限
+      // 累积、每天都进达标队列白跑一次(这正是 07-02 那个"处理完不出队"bug 的形状)。
+      const neverDream = DEFAULT_DREAM_CONFIG.neverDreamScopes;
+      const policyExcluded = nonMemoryScopes.filter(s => neverDream.includes(s));
+      const afterPolicy = nonMemoryScopes.filter(s => !neverDream.includes(s));
+      if (policyExcluded.length > 0) {
+        const prunedPolicy = await pruneWriteCounts(s => neverDream.includes(s), { statsPath: activityStatsPath });
+        console.log(
+          `[dream] ${policyExcluded.length} 个 scope 按策略不参与巩固（neverDreamScopes，无专用 schedule）` +
+          ` → ${policyExcluded.join(", ")}；已清写计数 ${prunedPolicy.length} 个`,
+        );
+      }
+
+      // 2026-08-14 transcript scope 整体出队（dream 写爆库根治的准入侧）：
+      // transcript 是原文层（shared-behaviors §5.3——原文归 Deja/ingest 管，提炼归
+      // pivot 池），历史会话被 ingest 批量灌入就把写计数打过门槛，2026-08 实测达标
+      // 队列 95% 是 codex/cc 会话 scope、dream 派生产物 96.6% 落在不参与日常检索的
+      // transcript scope 上——巩固算力几乎全部空转，还把 _versions 写到 7.7GB。
+      // 出队同时清扫计数（含未达标条目——历史会话只增不减，不清则 stats 文件无限膨胀；
+      // ingest 白天会再写回、次日 04:00 再清，稳态保持小）。判定用 memory-boundaries
+      // 的 isTranscriptScope（接新端时那边本来就要求同步前缀清单）。
+      const scopes = afterPolicy.filter(s => !isTranscriptScope(s));
+      const transcriptEligible = afterPolicy.length - scopes.length;
+      const prunedTranscripts = await pruneWriteCounts(isTranscriptScope, { statsPath: activityStatsPath });
+      if (prunedTranscripts.length > 0 || transcriptEligible > 0) {
+        console.log(
+          `[dream] transcript 原文层不参与巩固（§5.3）：清扫 ${prunedTranscripts.length} 个 scope 的写计数` +
+          `（其中达标 ${transcriptEligible} 个）→ 样例: ${prunedTranscripts.slice(0, 5).join(", ")}`,
+        );
+      }
+
+      // 2026-08-14 全库 GC 兜底（互审 C5-P0）：dream 是 auto-gc 的唯一生产触发点，而
+      // maybeRunGc 藏在 runDreamInner 里 —— transcript 出队后 durable 队列可为空，若在
+      // 进循环前就 exit，全库 GC 检查会静默消失、归档管线停转。这里无条件补调一次：
+      // GC 自带 24h stamp + 跨进程锁，循环内已触发过则本次 too_soon 跳过，幂等零成本。
+      // 失败只记日志不并入 blocked：GC 是 best-effort 维护，明天还有机会。
+      const runGcBackstop = async (): Promise<void> => {
+        try {
+          const gc = await maybeRunGc(store, DEFAULT_DREAM_CONFIG.gc);
+          console.log(
+            `[dream] 全库 GC 兜底: ${gc.triggered
+              ? `archived ${gc.archivedCount}, dissolved version groups ${gc.dissolvedVersionGroups}`
+              : gc.dissolvedVersionGroups > 0
+                ? `dissolved version groups ${gc.dissolvedVersionGroups}; archive skipped — ${gc.reason}`
+              : `skipped — ${gc.reason}`}`,
+          );
+        } catch (err) {
+          console.error(`[dream] 全库 GC 兜底失败（不阻断本轮判定）: ${err instanceof Error ? err.message : String(err)}`);
+        }
+      };
+
       if (scopes.length === 0) {
-        // 两种空:一种是真没人达标,一种是达标的全被挪走了——混成一句会让排障从这里开始跑偏。
+        // 三种空分开说:真没人达标 / 达标的全被挪走或全是 transcript——混成一句会让排障跑偏。
         console.log(
           eligible.length === 0
             ? "dream --auto: 无达标 scope（写计数均低于门槛）"
-            : "dream --auto: 达标 scope 全部交给专用 schedule，本轮无事可做",
+            : "dream --auto: 达标 scope 均为 transcript（已清扫）、已交专用 schedule 或按策略排除，本轮无事可做",
         );
+        await runGcBackstop();
         console.log("[[DREAM_STATUS]] skip");
         process.exit(0);
       }
@@ -1625,7 +1735,7 @@ program
       // 把 07-25~27 三天的调度全堵死（上一次没结束，launchd 起不了新的）。
       // 天花板见 DreamConfig.autoRunBudgetMs 的注释——截断会让靠后 scope 系统性饿死，
       // 真解是 session scope TTL，不是把预算调大。
-      const budgetMs = Number(process.env.RECALLNEST_DREAM_BUDGET_MS ?? DEFAULT_DREAM_CONFIG.autoRunBudgetMs);
+      const budgetMs = dreamBudgetMs(DEFAULT_DREAM_CONFIG.autoRunBudgetMs);
       const deadline = Date.now() + budgetMs;
 
       let fatalFailures = 0;
@@ -1647,7 +1757,11 @@ program
         }
         processed++;
         try {
-          const result = await runDream({ store, llm, embedder, scope, force: Boolean(options.force), activityStatsPath, kgStore });
+          // scopeMatch: "exact" —— 这些 scope 名直接来自 activity-stats 的键，是具体
+          // scope 不是 family selector。无冒号的键（实测 11 个，含 `cc` / `global` /
+          // `memory`）走前缀会把整个家族卷进来：`cc` 一旦达标就以 `LIKE 'cc%'` 把 08-14
+          // 刚出队的 transcript 原文层整批捞回（2026-08-16 互审 K4）。
+          const result = await runDream({ store, llm, embedder, scope, force: Boolean(options.force), activityStatsPath, kgStore, scopeMatch: "exact" });
           tally[result.output.kind]++;
           effectsTotal += result.output.effectsWritten;
           if (result.output.degraded) degradedScopes.push(`${scope}=${result.output.degraded}`);
@@ -1665,6 +1779,10 @@ program
         }
       }
 
+      // 全库 GC 兜底：循环内的 per-scope GC 若已触发过，这里 too_soon 幂等跳过；
+      // 若全部 scope 都 skip/失败没走到 prune 阶段，这里补上当天唯一一次 GC 检查。
+      await runGcBackstop();
+
       // 旧语义是「任一 scope 抛异常即 blocked」，在 475 个 scope 的规模下等于「有一个撞上
       // store-write 锁就整轮红」，而整轮红会让 shell 脚本重试 → 重跑全量。改为：代码/数据
       // 缺陷一票否决，环境性失败要成规模才算整轮失败。比例按已处理数算，不按总数。
@@ -1702,6 +1820,9 @@ program
         force: Boolean(options.force),
         activityStatsPath,
         kgStore,
+        // 手动调用默认保留 family（`--scope cc` 扫 `cc:*` 是既有用法）；
+        // 调度入口（weekly job）必须显式带 --exact-scope。
+        scopeMatch: options.exactScope ? "exact" : "family",
       });
 
       console.log(formatDreamResult(result));
@@ -1936,6 +2057,92 @@ program
 
 // ─── doctor ──────────────────────────────────────────────────────────────────
 
+program
+  .command("memory-utility")
+  .description("价值回溯：查一条 memory 参与过的任务成功率与 utility（不传 id 时按 scope 批量）")
+  .argument("[memoryId]", "memory id（完整 UUID 或前缀）")
+  .option("--scope <scope>", "批量模式：扫这个 scope 下的条目")
+  .option("--category <category>", "批量模式：只看这个 category")
+  .option("--limit <n>", "批量模式扫描上限", "500")
+  .option("--min-samples <n>", "批量模式只列出有效样本量 ≥ 此值的条目", "0")
+  .option("--links <n>", "展示最近 N 条参与记录", "5")
+  .option("--apply", "把算出的 utility 写回 metadata（默认只读，不写）")
+  .option("--observations-dir <dir>", "从指定目录读 workflow observations（默认生产目录）")
+  .option("--json", "输出 JSON")
+  .action(async (memoryId, options) => {
+    const { store } = createComponents(loadConfig());
+    const observationStore = options.observationsDir
+      ? new WorkflowObservationStore(expandHome(options.observationsDir))
+      : new WorkflowObservationStore();
+    // 全量拉 observation：join 要看的是历史全集，不是最近 200 条。
+    const observations = await observationStore.listRecent({ limit: 100_000 });
+    const showLinks = Number(options.links) || 0;
+
+    if (memoryId) {
+      const entry = await store.getById(memoryId);
+      if (!entry) {
+        console.error(`未找到 memory: ${memoryId}`);
+        process.exitCode = 1;
+        return;
+      }
+      const summary = computeMemoryUtility(entry, observations);
+      if (options.apply) {
+        await store.patchMetadata(entry.id, (meta) => applyUtilityToMeta(meta, summary));
+      }
+      if (options.json) {
+        console.log(JSON.stringify({ ...summary, applied: Boolean(options.apply) }, null, 2));
+        return;
+      }
+      console.log(formatMemoryUtility(summary, { showLinks }));
+      console.log(`  observations 池: ${observations.length} 条`);
+      if (options.apply) console.log("  已写回 metadata.utility");
+      return;
+    }
+
+    if (!options.scope) {
+      console.error("需要 <memoryId> 或 --scope");
+      process.exitCode = 1;
+      return;
+    }
+
+    const limit = Number(options.limit) || 500;
+    const minSamples = Number(options.minSamples) || 0;
+    const entries = await store.listPage({
+      scopeFilter: [options.scope],
+      category: options.category,
+      limit,
+      offset: 0,
+    });
+
+    const summaries = entries
+      .map((entry) => computeMemoryUtility(entry, observations))
+      .filter((s) => s.effectiveSamples >= minSamples && s.total > 0)
+      .sort((a, b) => (b.utility ?? 0) - (a.utility ?? 0));
+
+    if (options.apply) {
+      for (const summary of summaries) {
+        await store.patchMetadata(summary.memoryId, (meta) => applyUtilityToMeta(meta, summary));
+      }
+    }
+
+    if (options.json) {
+      console.log(JSON.stringify({ scanned: entries.length, matched: summaries.length, summaries }, null, 2));
+      return;
+    }
+
+    console.log(`scope=${options.scope} 扫描 ${entries.length} 条，其中 ${summaries.length} 条有参与记录`);
+    console.log(`observations 池: ${observations.length} 条`);
+    for (const summary of summaries.slice(0, 20)) {
+      const utility = summary.utility === null ? "  null" : summary.utility.toFixed(4).padStart(7);
+      console.log(
+        `${utility}  samples=${summary.effectiveSamples.toFixed(2).padStart(6)}`
+        + `  exact=${String(summary.exactCount).padStart(3)} session=${String(summary.sessionCount).padStart(3)}`
+        + `  ${summary.memoryId.slice(0, 8)}`,
+      );
+    }
+    if (options.apply) console.log(`已写回 ${summaries.length} 条的 metadata.utility`);
+  });
+
 program
   .command("workflow-observe")
   .description("记录一条 workflow primitive observation，作为自进化观测层的 append-only 输入")
@@ -1949,9 +2156,14 @@ program
   .option("--tags <tags>", "逗号分隔 tags")
   .option("--tools <tools>", "逗号分隔 tools")
   .option("--idempotency-key <key>", "同一请求重试时用于替换旧 observation 的稳定键")
+  .option("--reader-id <id>", "P0 join key：读者身份（默认本进程）。CLI 每次是新进程，跨会话关联要显式传")
+  .option("--recalled-ids <ids>", "P0 join key：逗号分隔的 memory id，本次任务实际用到的记忆")
   .option("--json", "输出 JSON")
   .action(async (workflowId, summary, options) => {
     const store = new WorkflowObservationStore();
+    // CLI 是一次性进程，检索账本必然是空的（这次进程没检索过），所以不做自动补，
+    // 只认显式传入——自动补在这里只会写出一个恒空的字段，看着像记了其实没记。
+    const recalledIds = splitCsvOption(options.recalledIds);
     const record = buildWorkflowObservationRecord({
       workflowId,
       summary,
@@ -1963,6 +2175,8 @@ program
       tags: splitCsvOption(options.tags),
       tools: splitCsvOption(options.tools),
       idempotencyKey: options.idempotencyKey,
+      readerId: options.readerId,
+      ...(recalledIds.length > 0 ? { recalledIds } : {}),
     });
     const stored = await store.save(record);
     if (options.json) {