recallnest 3.0.1 → 4.0.0

package/src/ingest-plan.ts

@@ -0,0 +1,56 @@
+/**
+ * `ingest` 命令这一轮跑哪些来源。
+ *
+ * 2026-10-06 起对话原文默认不再入库（开关 `RECALLNEST_TRANSCRIPT_INGEST`，见 env-config.ts `transcriptIngest`）。
+ * 入库有两个触发入口——launchd 定时那一轮、pull-from-macbook.sh 拉完之后那一轮——两个都跑
+ * `scripts/incremental-ingest.sh` → `cli.ts ingest --source all`，所以闸放在命令内部，而不是去改脚本或卸定时任务。
+ *
+ * 关着的时候三件事不能跟着停，这也是把决定单独抽成一个函数、逐格测到的原因：
+ *   - 记忆文件对账（memory 来源）照常；
+ *   - Minis 投递目录里的文件照样挪进 data/minis-archive：Deja 靠那个存档读 Minis 的对话，而挪存档原先拿
+ *     「台账里记着已入库」当闸——只跳过入库的话，文件永远不会被标成已处理，也就永远不会被挪，而且不报错；
+ *   - 显式点名一个对话来源（`--source cc`）时要说清没有执行，不能安静地什么都不做。
+ */
+
+export const CONVERSATION_SOURCES = ["cc", "codex", "kimi", "gemini", "desktop", "minis"] as const;
+export type ConversationSource = (typeof CONVERSATION_SOURCES)[number];
+
+export type MinisMode =
+  /** 入库（切片、嵌入），再把台账里记着已入库的文件挪进存档——开关打开时的行为 */
+  | "ingest-then-archive"
+  /** 不入库；能解析出对话的文件直接挪进存档——开关关着时的行为 */
+  | "archive-only"
+  /** 这一轮不碰 Minis 投递目录 */
+  | "skip";
+
+export interface IngestPlan {
+  /** 这一轮要切片、嵌入、写库的对话来源（不含 minis，它单独看 `minis`） */
+  transcriptSources: Exclude<ConversationSource, "minis">[];
+  minis: MinisMode;
+  /** 记忆文件（memory 来源） */
+  memory: boolean;
+  /** 待处理队列里攒的是没有 LLM 时跳过的对话切片，回填等于入库，跟着开关走 */
+  drainPendingQueue: boolean;
+  /** 这一轮点到了、但因为开关关着而没有入库的对话来源（含 minis），日志里要说出来 */
+  skippedTranscriptSources: ConversationSource[];
+}
+
+export function isConversationSource(source: string): source is ConversationSource {
+  return (CONVERSATION_SOURCES as readonly string[]).includes(source);
+}
+
+export function planIngest(source: string, transcriptIngestOn: boolean): IngestPlan {
+  const wants = (name: string) => source === "all" || source === name;
+  const requested = CONVERSATION_SOURCES.filter((name) => wants(name));
+  const requestedTranscripts = requested.filter(
+    (name): name is Exclude<ConversationSource, "minis"> => name !== "minis",
+  );
+
+  return {
+    transcriptSources: transcriptIngestOn ? requestedTranscripts : [],
+    minis: !wants("minis") ? "skip" : transcriptIngestOn ? "ingest-then-archive" : "archive-only",
+    memory: wants("memory"),
+    drainPendingQueue: transcriptIngestOn,
+    skippedTranscriptSources: transcriptIngestOn ? [] : requested,
+  };
+}

package/src/ingest.ts

@@ -13,6 +13,7 @@ import { join, basename, resolve, dirname } from "node:path";
 import { homedir } from "node:os";
 import { detectLang, tokenizeFts } from "./language-hook.js";
 import { redactSecrets } from "./pii-detector.js";
+import { normalizeDedupText } from "./text-fingerprint.js";
 import type { MemoryStore, MemoryEntry } from "./store.js";
 import type { Embedder } from "./embedder.js";
 import { chunkDocument, type ChunkerConfig } from "./chunker.js";
@@ -35,6 +36,8 @@ import {
 import { compressToolOutput } from "./tool-output-compressor.js";
 import { tagNarrativeIfEnabled } from "./narrative-tagger.js";
 
+export { normalizeDedupText };
+
 // ============================================================================
 // Types
 // ============================================================================
@@ -53,6 +56,8 @@ export interface IngestResult {
   chunksDeduped: number;
   dedupReasonCounts: DedupReasonCounts;
   errors: string[];
+  /** 子 agent 独立会话文件被整文件跳过的数量（目前只有 Codex 分支会产生；见 isCodexSubagentSessionFile）。 */
+  subagentFilesSkipped?: number;
 }
 
 interface ConversationTurn {
@@ -111,19 +116,21 @@ export type DedupReason = "hard" | "exact" | "llm-skip" | "llm-merge" | "unique"
 
 export type DedupReasonCounts = Record<DedupReason, number>;
 
-/** Secondary action on an existing memory during dedup (e.g., delete outdated entries). */
-export interface DedupAction {
-  id: string;
-  action: "delete";
-  reason: string;
-}
-
+/**
+ * 去重结论只回答「这条进来的切片存不存」，不带任何指向已有记忆的动作。
+ *
+ * 2026-10-05 之前这里还有一个 `secondaryDeletes`：模型可以顺手标记「过时」的已有记忆，
+ * 导入调用点原样 `store.delete`——候选不分 scope，不写审计、不记日志、错误吞掉。
+ * 2026-09-22T16:38:03Z 记忆文件导入就这样删掉了当天手写进人工维护那一层的一条偏好，
+ * 11 秒后同一轮导入写入了内容相近的文档切片；到 10-05 核对时，显式存过、不在主表、又没有
+ * forget 记录的手写记忆有 17 条。导入只新增和跳过，删除一律走 forget_memory（有隐私级别
+ * 检查与审计）。经过与数字见 CHANGELOG「Ingest dedup can no longer delete existing memories」；
+ * 回归用例 `ingest-dedup-no-deletes.test.ts`。
+ */
 export interface DedupCheckResult {
   action: "store" | "skip";
   reason: DedupReason;
   existingText?: string;
-  /** Secondary actions on other existing memories (only populated with LLM dedup). */
-  secondaryDeletes?: DedupAction[];
 }
 
 function createDedupReasonCounts(): DedupReasonCounts {
@@ -143,26 +150,6 @@ function recordDedupDecision(result: IngestResult, decision: DedupCheckResult):
   }
 }
 
-/**
- * Execute secondary delete actions from a dedup decision.
- * Errors are isolated per-action to avoid cascading failures.
- */
-async function executeSecondaryDeletes(
-  store: MemoryStore,
-  deletes: DedupAction[],
-): Promise<number> {
-  let executed = 0;
-  for (const del of deletes) {
-    try {
-      await store.delete(del.id);
-      executed++;
-    } catch {
-      // Per-action error isolation: log and continue
-    }
-  }
-  return executed;
-}
-
 export function getDedupSkippedCount(result: IngestResult): number {
   return result.dedupReasonCounts.hard
     + result.dedupReasonCounts.exact
@@ -181,22 +168,6 @@ export function formatDedupReasonSummary(result: IngestResult): string {
   return `hard:${counts.hard}, exact:${counts.exact}, llm-skip:${counts["llm-skip"]}, llm-merge:${counts["llm-merge"]}`;
 }
 
-/**
- * Two-stage dedup: vector pre-filter + optional LLM semantic decision.
- *
- * Returns: "store" | "skip"
- *
- * Note: when the LLM says MERGE, we currently keep the new chunk instead of
- * dropping it. We do not have a structured ingest-time merge path yet, and
- * swallowing "same topic + new information" transcript chunks is worse for
- * recall fidelity than storing an incremental near-duplicate.
- */
-function normalizeDedupText(value: string): string {
-  return value
-    .replace(/^\[(用户|助手)\]\s*/gm, "")
-    .replace(/\s+/g, " ")
-    .trim();
-}
 
 /**
  * Detect content that carries temporal events, file operations, or explicit
@@ -306,6 +277,16 @@ function shouldForceCreateToolChoicePreference(
     : { shouldForceCreate: false };
 }
 
+/**
+ * Two-stage dedup: vector pre-filter + optional LLM semantic decision.
+ *
+ * Returns: "store" | "skip"
+ *
+ * Note: when the LLM says MERGE, we currently keep the new chunk instead of
+ * dropping it. We do not have a structured ingest-time merge path yet, and
+ * swallowing "same topic + new information" transcript chunks is worse for
+ * recall fidelity than storing an incremental near-duplicate.
+ */
 export async function dedupCheck(
   store: MemoryStore,
   vector: number[],
@@ -415,8 +396,6 @@ export async function dedupCheck(
         const hasMulti = typeof (llm as any).dedupDecisionMulti === "function";
 
         let primaryAction: "CREATE" | "MERGE" | "SKIP" = "CREATE";
-        let primaryReason = "";
-        const secondaryDeletes: DedupAction[] = [];
 
         if (hasMulti) {
           const candidates = results
@@ -424,48 +403,20 @@ export async function dedupCheck(
             .slice(0, DEDUP_CANDIDATE_LIMIT)
             .map(r => ({ id: r.entry.id, text: r.entry.text }));
 
+          // 只取「存 / 并 / 跳」这一个结论。候选来自全库（不分 scope），模型对它们没有处置权，
+          // 见 DedupCheckResult 上方的说明。
           const decision = await llm.dedupDecisionMulti(text, candidates);
           primaryAction = decision.action;
-          primaryReason = decision.reason;
-
-          // Build secondary deletes from validated actions (resolve IDs at parse time)
-          if (decision.actions) {
-            for (const act of decision.actions) {
-              const target = candidates[act.match_index - 1];
-              if (target) {
-                secondaryDeletes.push({
-                  id: target.id,
-                  action: "delete",
-                  reason: typeof act.reason === "string" ? act.reason : "",
-                });
-              }
-            }
-          }
         } else {
           const decision = await llm.dedupDecision(text, existingText);
           primaryAction = decision.action;
-          primaryReason = decision.reason;
         }
 
         if (primaryAction === "SKIP") {
-          return {
-            action: "skip",
-            reason: "llm-skip",
-            existingText,
-            ...(secondaryDeletes.length > 0 ? { secondaryDeletes } : {}),
-          };
+          return { action: "skip", reason: "llm-skip", existingText };
         }
         if (primaryAction === "MERGE") {
-          return {
-            action: "store",
-            reason: "llm-merge",
-            existingText,
-            ...(secondaryDeletes.length > 0 ? { secondaryDeletes } : {}),
-          };
-        }
-        // CREATE — still execute secondary deletes if any
-        if (secondaryDeletes.length > 0) {
-          return { action: "store", reason: "unique", secondaryDeletes };
+          return { action: "store", reason: "llm-merge", existingText };
         }
       } catch {
         // LLM failed, fall through to store
@@ -516,7 +467,7 @@ function fallbackExtraction(text: string): SmartExtraction {
  * Uses LLM 6-category extraction when available, falls back to heuristic.
  * Returns: category + L0 + L1 + importance for each text.
  */
-async function smartExtractBatch(
+export async function smartExtractBatch(
   texts: string[],
   llm?: LLMClient | null,
 ): Promise<SmartExtraction[]> {
@@ -545,7 +496,7 @@ function isCoreSummaryEnabled(): boolean {
  * Only runs when RECALLNEST_CORE_SUMMARY=true and LLM is available.
  * Returns null array when disabled (no overhead).
  */
-async function generateCoreSummaries(
+export async function generateCoreSummaries(
   texts: string[],
   llm?: LLMClient | null,
 ): Promise<(string | null)[]> {
@@ -715,7 +666,7 @@ function initialTier(extraction: Pick<SmartExtraction, "category" | "importance"
   return "peripheral";
 }
 
-function buildIngestedEntry(params: {
+export function buildIngestedEntry(params: {
   source: string;
   scope: string;
   text: string;
@@ -884,6 +835,11 @@ export function parseCCTranscript(filePath: string): ConversationTurn[] {
   for (const line of lines) {
     try {
       const obj = JSON.parse(line);
+      // 子 agent 侧链（isSidechain:true）的 user 行是主 agent 写给子 agent 的任务信封，不是人说的话。
+      // 今天 CC 把侧链分文件存在 <session>/subagents/agent-*.jsonl，而本函数的调用方只读项目目录顶层
+      // 的 .jsonl，所以这一行当前不会命中；留着是防 CC 哪天把侧链内联回主文件（2026-09-08 借鉴 ECC
+      // observe.sh 的「按结构字段认谁在说、不认 role」）。
+      if (obj.isSidechain === true) continue;
       // 这一行里所有的图片信号。用户亲手贴的那部分在下面单独精确认，
       // 其余的一律按补集归入 AI 产图——不枚举它是截图、读图、生图还是配图。
       const lineImageSignals = countImageSignals(obj);
@@ -1101,6 +1057,28 @@ export function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
 // Codex Session Parser
 // ============================================================================
 
+/**
+ * Codex multi_agent 派出的子 agent 会话是独立的 rollout 文件，首行 session_meta 带 parent_thread_id
+ * （另有 agent_nickname / agent_role / multi_agent_version）。文件里 role=user 的 input_text 是主 agent
+ * 派的任务，不是人说的话——2026-09-08 实测本机 5,597 个 Codex 会话文件里 2,555 个（45.6%）是子 agent 文件，
+ * 其 user turn 占全部 user turn 的 46.7%（近 7 天 63.7%）。ingest 前整文件跳过，与 Kimi 分支只读
+ * agents/main/wire.jsonl 同一个判据：按结构字段认「谁在说」，不认 role。只读首行，不整读文件。
+ */
+export function isCodexSubagentSessionFile(filePath: string): boolean {
+  try {
+    const content = readFileSync(filePath, "utf-8");
+    const nl = content.indexOf("\n");
+    const first = (nl === -1 ? content : content.slice(0, nl)).trim();
+    if (!first) return false;
+    const obj = JSON.parse(first);
+    if (obj?.type !== "session_meta") return false;
+    const parent = obj.payload?.parent_thread_id;
+    return typeof parent === "string" && parent.length > 0;
+  } catch {
+    return false;
+  }
+}
+
 export function parseCodexSession(filePath: string): ConversationTurn[] {
   const turns: ConversationTurn[] = [];
   const content = readFileSync(filePath, "utf-8");
@@ -1266,6 +1244,14 @@ export async function ingestCodexSessions(
         continue;
       }
 
+      // 子 agent 独立会话整文件跳过（判据与理由见 isCodexSubagentSessionFile）。标记 processed 是为了
+      // 下一轮非 --recent 的增量不再重读它；子 agent 的结论本就该由主 agent 落盘，原文由 Deja 全量索引兜底。
+      if (isCodexSubagentSessionFile(filePath)) {
+        result.subagentFilesSkipped = (result.subagentFilesSkipped ?? 0) + 1;
+        markProcessed(filePath, stat.size, 0, stat.mtimeMs);
+        continue;
+      }
+
       const turns = parseCodexSession(filePath);
       if (turns.length === 0) {
         result.chunksSkipped++;
@@ -1300,9 +1286,6 @@ export async function ingestCodexSessions(
             if (!options.noDedup) {
               const decision = await dedupCheck(store, vector, batch[j], options.llm);
               recordDedupDecision(result, decision);
-              if (decision.secondaryDeletes?.length) {
-                await executeSecondaryDeletes(store, decision.secondaryDeletes);
-              }
               if (decision.action === "skip") continue;
             }
             dedupedTexts.push(batch[j]);
@@ -1601,9 +1584,6 @@ export async function ingestKimiSessions(
             if (!options.noDedup) {
               const decision = await dedupCheck(store, vector, batch[j], options.llm);
               recordDedupDecision(result, decision);
-              if (decision.secondaryDeletes?.length) {
-                await executeSecondaryDeletes(store, decision.secondaryDeletes);
-              }
               if (decision.action === "skip") continue;
             }
             dedupedTexts.push(batch[j]);
@@ -1806,9 +1786,6 @@ export async function ingestGeminiSessions(
             if (!options.noDedup) {
               const decision = await dedupCheck(store, vector, batch[j], options.llm);
               recordDedupDecision(result, decision);
-              if (decision.secondaryDeletes?.length) {
-                await executeSecondaryDeletes(store, decision.secondaryDeletes);
-              }
               if (decision.action === "skip") continue;
             }
             dedupedTexts.push(batch[j]);
@@ -1876,7 +1853,7 @@ export async function ingestGeminiSessions(
  * Split a markdown file by headings (## or #) into chunks.
  * Each chunk includes the heading + content under it.
  */
-function parseMarkdown(filePath: string): Array<{ text: string; heading: string }> {
+export function parseMarkdown(filePath: string): Array<{ text: string; heading: string }> {
   const content = readFileSync(filePath, "utf-8");
   const chunks: Array<{ text: string; heading: string }> = [];
 
@@ -2042,9 +2019,6 @@ export async function ingestCCTranscripts(
             if (!options.noDedup) {
               const decision = await dedupCheck(store, vector, batch[j], options.llm);
               recordDedupDecision(result, decision);
-              if (decision.secondaryDeletes?.length) {
-                await executeSecondaryDeletes(store, decision.secondaryDeletes);
-              }
               if (decision.action === "skip") continue;
             }
 
@@ -2196,9 +2170,6 @@ export async function ingestMarkdownFiles(
             if (!options.noDedup) {
               const decision = await dedupCheck(store, vector, batch[j], options.llm);
               recordDedupDecision(result, decision);
-              if (decision.secondaryDeletes?.length) {
-                await executeSecondaryDeletes(store, decision.secondaryDeletes);
-              }
               if (decision.action === "skip") continue;
             }
             dedupedTexts.push(batch[j]);
@@ -2330,9 +2301,6 @@ export async function ingestGenericText(
             if (!options.noDedup) {
               const decision = await dedupCheck(store, vector, batch[j], options.llm);
               recordDedupDecision(result, decision);
-              if (decision.secondaryDeletes?.length) {
-                await executeSecondaryDeletes(store, decision.secondaryDeletes);
-              }
               if (decision.action === "skip") continue;
             }
             dedupedTexts.push(batch[j]);
@@ -2449,9 +2417,6 @@ export async function ingestConnectorFile(
         if (!options.noDedup) {
           const decision = await dedupCheck(store, vector, texts[j], options.llm);
           recordDedupDecision(result, decision);
-          if (decision.secondaryDeletes?.length) {
-            await executeSecondaryDeletes(store, decision.secondaryDeletes);
-          }
           if (decision.action === "skip") continue;
         }
         dedupedTexts.push(texts[j]);