recallnest 2.3.0 → 2.6.0

package/src/ingest.ts

@@ -12,6 +12,7 @@ import { readFileSync, readdirSync, existsSync, statSync, writeFileSync } from "
 import { join, basename, resolve, dirname } from "node:path";
 import { homedir } from "node:os";
 import { detectLang, tokenizeFts } from "./language-hook.js";
+import { redactSecrets } from "./pii-detector.js";
 import type { MemoryStore, MemoryEntry } from "./store.js";
 import type { Embedder } from "./embedder.js";
 import { chunkDocument, type ChunkerConfig } from "./chunker.js";
@@ -20,8 +21,10 @@ import { isConnectorOutputV1 } from "./connector-types.js";
 import { isObsidianVault, scanVault } from "./obsidian-connector.js";
 import { segmentEvents, type EventSegmenterConfig, DEFAULT_EVENT_SEGMENTER_CONFIG } from "./event-segmenter.js";
 import { isProcessed, markProcessed } from "./tracker.js";
+import * as envConfig from "./env-config.js";
 import type { LLMClient, SmartExtraction } from "./llm-client.js";
 import { resolveIngestBoundary } from "./memory-boundaries.js";
+import { isActiveMemory } from "./memory-evolution.js";
 import { isNoise } from "./noise-filter.js";
 import {
   inferReplyStylePreferenceSlot,
@@ -287,8 +290,12 @@ export async function dedupCheck(
   llm?: LLMClient | null,
 ): Promise<DedupCheckResult> {
   try {
-    // Stage 1: vector similarity check
-    const results = await store.vectorSearch(vector, DEDUP_CANDIDATE_LIMIT, DEDUP_SOFT_THRESHOLD);
+    // Stage 1: vector similarity check. store.vectorSearch() has no status filter, so
+    // superseded belief-history rows surface here — dedupe against one and incoming
+    // content gets dropped as a duplicate of a belief that is no longer held (returning
+    // to an earlier position would be silently discarded).
+    const results = (await store.vectorSearch(vector, DEDUP_CANDIDATE_LIMIT, DEDUP_SOFT_THRESHOLD))
+      .filter((result) => isActiveMemory(result.entry.metadata));
     if (results.length === 0) {
       return { action: "store", reason: "unique" }; // Clearly unique
     }
@@ -507,7 +514,7 @@ async function smartExtractBatch(
  * Tier 3.1: Check if core summary generation is enabled.
  */
 function isCoreSummaryEnabled(): boolean {
-  return process.env.RECALLNEST_CORE_SUMMARY === "true";
+  return envConfig.coreSummary();
 }
 
 /**
@@ -625,7 +632,8 @@ export async function drainPendingQueue(
   let processed = 0, errors = 0;
   for (let i = 0; i < pending.length; i += 20) {
     const batch = pending.slice(i, i + 20);
-    const texts = batch.map(c => c.text);
+    // F-3b: scrub before the LLM extraction call and downstream embedding
+    const texts = batch.map(c => redactSecrets(c.text).text);
     const extractions = await smartExtractBatch(texts, llm);
     const embeddingTexts = extractions.map(e => e.l1 || e.l0);
     const vectors = await embedder.embedBatchPassage(embeddingTexts);
@@ -860,7 +868,7 @@ function filterNoiseTurns(turns: ConversationTurn[]): ConversationTurn[] {
   return turns.filter((turn) => !isNoise(turn.text));
 }
 
-function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
+export function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
   text: string;
   timestamp: string;
   sessionId: string;
@@ -933,7 +941,18 @@ function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
     }
   }
 
-  return chunks;
+  // P1 defence-in-depth: re-filter AFTER splitting. Long turns pass the
+  // turn-level filterNoiseTurns gate, but chunking can peel their tail into
+  // severed context-JSON fragments that the turn-level check never saw.
+  // F-3b: scrub secrets at the chunk boundary — this is the shared text
+  // finalization point for all session-ingest paths (CC/Codex/Gemini), before
+  // any chunk reaches embedding, LLM extraction, or storage.
+  return chunks
+    .filter((c) => !isNoise(c.text))
+    .map((c) => {
+      const r = redactSecrets(c.text);
+      return r.redacted > 0 ? { ...c, text: r.text } : c;
+    });
 }
 
 // ============================================================================
@@ -1027,9 +1046,14 @@ export async function ingestCodexSessions(
     errors: [],
   };
 
-  const baseDir = expandHome("~/.codex/sessions");
-  if (!existsSync(baseDir)) {
-    result.errors.push(`Codex sessions directory not found: ${baseDir}`);
+  // 同时扫 sessions（活跃）+ archived_sessions（App 内归档会把文件移到这里，否则会漏）
+  const baseDirs = [
+    expandHome("~/.codex/sessions"),
+    expandHome("~/.codex/archived_sessions"),
+  ];
+  const existingDirs = baseDirs.filter((d) => existsSync(d));
+  if (existingDirs.length === 0) {
+    result.errors.push(`Codex sessions directory not found: ${baseDirs.join(", ")}`);
     return result;
   }
 
@@ -1048,7 +1072,7 @@ export async function ingestCodexSessions(
       }
     }
   }
-  walk(baseDir);
+  for (const dir of existingDirs) walk(dir);
   allFiles.sort();
 
   // --recent: only keep files modified within N hours
@@ -1171,6 +1195,276 @@ export async function ingestCodexSessions(
   return result;
 }
 
+// ============================================================================
+// Kimi Code Session Parser
+// ============================================================================
+// Kimi Code 的 wire.jsonl 是事件流（非 role/content 行），对话内容只在三类事件里：
+//   turn.prompt (origin.kind="user")          → 用户输入，.input 是 text part 数组
+//   context.append_message                    → 消息；用户消息只收 origin.kind="user"
+//                                               （injection/skill_activation 等是系统注入，跳过）
+//   context.append_loop_event content.part    → 助手文本（part.type==="text"；think 跳过）
+// 其余类型（llm.tools_snapshot / mcp.tools_discovered / config.update / llm.request /
+// usage.record / permission.* 等，约占 86% 体积）是运行事件，一律跳过。
+function parseKimiSession(filePath: string): ConversationTurn[] {
+  const turns: ConversationTurn[] = [];
+  const content = readFileSync(filePath, "utf-8");
+  const lines = content.split("\n").filter((l) => l.trim());
+
+  // sessionId 从路径取：.../sessions/<workDirKey>/<sessionId>/agents/main/wire.jsonl
+  // 去掉 "session_" 前缀，否则 scope 统一变成 kimi:session_
+  const pathParts = filePath.split("/");
+  const rawId = pathParts.length >= 4 ? pathParts[pathParts.length - 4] : basename(filePath, ".jsonl");
+  const sessionId = rawId.replace(/^session_/, "");
+
+  const pushTurn = (role: "user" | "assistant", text: string, timestamp: string) => {
+    const trimmed = text.trim();
+    // 中文 8 字符已是完整句；用户阈值与 codex/cc 对齐
+    if (trimmed.length <= (role === "user" ? 10 : 8)) return;
+    // turn.prompt 与随后的 context.append_message 重复记录同一用户输入（同 codex 相邻去重）
+    const lastTurn = turns[turns.length - 1];
+    if (lastTurn && lastTurn.role === role && lastTurn.text === trimmed) return;
+    turns.push({ role, text: trimmed, timestamp, sessionId });
+  };
+
+  for (const line of lines) {
+    try {
+      const obj = JSON.parse(line);
+      const timestamp = typeof obj.time === "number" ? new Date(obj.time).toISOString() : "";
+
+      if (obj.type === "turn.prompt") {
+        if (obj.origin?.kind !== "user") continue;
+        const parts = Array.isArray(obj.input) ? obj.input : [];
+        const text = parts
+          .filter((c: any) => c?.type === "text")
+          .map((c: any) => c.text || "")
+          .join("\n");
+        if (text) pushTurn("user", text, timestamp);
+        continue;
+      }
+
+      if (obj.type === "context.append_message") {
+        const msg = obj.message;
+        if (!msg || (msg.role !== "user" && msg.role !== "assistant")) continue;
+        // 系统注入（system-reminder、skill 激活、后台任务通知）不是用户对话内容
+        if (msg.role === "user" && msg.origin?.kind !== "user") continue;
+        let text = "";
+        if (typeof msg.content === "string") {
+          text = msg.content;
+        } else if (Array.isArray(msg.content)) {
+          text = msg.content
+            .filter((c: any) => c.type === "text")
+            .map((c: any) => c.text || "")
+            .join("\n");
+        }
+        if (text) pushTurn(msg.role, text, timestamp);
+        continue;
+      }
+
+      if (obj.type === "context.append_loop_event") {
+        const part = obj.event?.type === "content.part" ? obj.event.part : null;
+        if (part?.type === "text" && typeof part.text === "string") {
+          pushTurn("assistant", part.text, timestamp);
+        }
+      }
+    } catch {
+      // Skip malformed lines
+    }
+  }
+
+  return turns;
+}
+
+export async function ingestKimiSessions(
+  store: MemoryStore,
+  embedder: Embedder,
+  options: { limit?: number; verbose?: boolean; noDedup?: boolean; llm?: LLMClient | null; recentHours?: number } = {},
+): Promise<IngestResult> {
+  const result: IngestResult = {
+    source: "kimi",
+    filesProcessed: 0,
+    chunksIngested: 0,
+    chunksSkipped: 0,
+    chunksDeduped: 0,
+    dedupReasonCounts: createDedupReasonCounts(),
+    errors: [],
+  };
+
+  const baseDir = expandHome("~/.kimi-code/sessions");
+  const indexPath = expandHome("~/.kimi-code/session_index.jsonl");
+
+  // 文件发现：优先 session_index.jsonl（精确），再 walk sessions 目录兜底（索引缺漏时）。
+  // 只收 agents/main/wire.jsonl，子 agent 的 agents/agent-N/wire.jsonl 与主对话重复，不采。
+  const seen = new Set<string>();
+  const allFiles: string[] = [];
+  const addFile = (p: string) => {
+    if (!seen.has(p) && existsSync(p)) {
+      seen.add(p);
+      allFiles.push(p);
+    }
+  };
+
+  if (existsSync(indexPath)) {
+    try {
+      for (const line of readFileSync(indexPath, "utf-8").split("\n")) {
+        if (!line.trim()) continue;
+        try {
+          const entry = JSON.parse(line);
+          if (typeof entry.sessionDir === "string" && entry.sessionDir) {
+            addFile(join(entry.sessionDir, "agents", "main", "wire.jsonl"));
+          }
+        } catch {
+          // Skip malformed index lines
+        }
+      }
+    } catch {
+      // 索引读不出来就走下面的目录扫描
+    }
+  }
+
+  if (existsSync(baseDir)) {
+    const mainWireSuffix = join("agents", "main", "wire.jsonl");
+    const walk = (dir: string) => {
+      for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        const full = join(dir, entry.name);
+        if (entry.isDirectory()) {
+          walk(full);
+        } else if (full.endsWith(mainWireSuffix)) {
+          addFile(full);
+        }
+      }
+    };
+    walk(baseDir);
+  }
+
+  if (allFiles.length === 0) {
+    result.errors.push(`Kimi Code sessions not found: ${baseDir} (index: ${indexPath})`);
+    return result;
+  }
+  allFiles.sort();
+
+  const recentMode = options.recentHours !== undefined;
+  const cutoffMs = recentMode ? Date.now() - options.recentHours! * 3600_000 : 0;
+
+  // --recent: only keep files modified within N hours
+  const filteredFiles = recentMode
+    ? allFiles.filter((f) => { try { return statSync(f).mtimeMs >= cutoffMs; } catch { return false; } })
+    : allFiles;
+
+  const filesToProcess = options.limit ? filteredFiles.slice(0, options.limit) : filteredFiles;
+  const total = filesToProcess.length;
+
+  for (let fi = 0; fi < filesToProcess.length; fi++) {
+    const filePath = filesToProcess[fi];
+
+    try {
+      const stat = statSync(filePath);
+      if (stat.size < 200) {
+        result.chunksSkipped++;
+        continue;
+      }
+
+      // In --recent mode, skip ingested-files check — files may have new content appended
+      if (!recentMode && isProcessed(filePath, stat.size, stat.mtimeMs)) {
+        result.chunksSkipped++;
+        continue;
+      }
+
+      const turns = parseKimiSession(filePath);
+      if (turns.length === 0) {
+        result.chunksSkipped++;
+        markProcessed(filePath, stat.size, 0, stat.mtimeMs);
+        continue;
+      }
+
+      const chunks = groupTurnsIntoChunks(turns);
+      const texts = chunks.map((c) => c.text);
+      const batchSize = 32;
+      let fileChunks = 0;
+
+      for (let i = 0; i < texts.length; i += batchSize) {
+        const batch = texts.slice(i, i + batchSize);
+        const batchChunks = chunks.slice(i, i + batchSize);
+
+        try {
+          const vectors = await embedder.embedBatchPassage(batch);
+          const toStore: Array<{text: string; vector: number[]; category: string; scope: string; importance: number; metadata: string}> = [];
+
+          // Dedup + L0 batch
+          const dedupedTexts: string[] = [];
+          const dedupedVectors: number[][] = [];
+          const dedupedChunks: typeof batchChunks = [];
+
+          for (let j = 0; j < batch.length; j++) {
+            const vector = vectors[j];
+            if (!vector || vector.length === 0) {
+              result.chunksSkipped++;
+              continue;
+            }
+            if (!options.noDedup) {
+              const decision = await dedupCheck(store, vector, batch[j], options.llm);
+              recordDedupDecision(result, decision);
+              if (decision.secondaryDeletes?.length) {
+                await executeSecondaryDeletes(store, decision.secondaryDeletes);
+              }
+              if (decision.action === "skip") continue;
+            }
+            dedupedTexts.push(batch[j]);
+            dedupedVectors.push(vector);
+            dedupedChunks.push(batchChunks[j]);
+          }
+
+          if (dedupedTexts.length > 0) {
+            if (!options.llm) {
+              queueForLaterExtraction(
+                dedupedChunks.map(c => ({ text: c.text, scope: `kimi:${c.sessionId.slice(0, 8)}` }))
+              );
+              result.chunksSkipped += dedupedTexts.length;
+            } else {
+              const extractions = await smartExtractBatch(dedupedTexts, options.llm);
+              const coreSummaries = await generateCoreSummaries(dedupedTexts, options.llm);
+              for (let j = 0; j < dedupedTexts.length; j++) {
+                const chunk = dedupedChunks[j];
+                const ext = extractions[j];
+                toStore.push(buildIngestedEntry({
+                  source: "kimi",
+                  scope: `kimi:${chunk.sessionId.slice(0, 8)}`,
+                  text: dedupedTexts[j],
+                  vector: dedupedVectors[j],
+                  extraction: ext,
+                  sessionId: chunk.sessionId,
+                  file: basename(filePath),
+                  coreSummary: coreSummaries[j],
+                }));
+              }
+            }
+          }
+
+          if (toStore.length > 0) {
+            await store.storeBatch(toStore);
+            result.chunksIngested += toStore.length;
+            fileChunks += toStore.length;
+          }
+        } catch (err: any) {
+          result.errors.push(`Embedding batch error: ${err.message}`);
+        }
+      }
+
+      markProcessed(filePath, stat.size, fileChunks, stat.mtimeMs);
+      result.filesProcessed++;
+
+      if ((fi + 1) % 10 === 0 || fi + 1 === total) {
+        console.log(
+          `  Kimi: ${fi + 1}/${total} files, ${result.chunksIngested} chunks ingested`,
+        );
+      }
+    } catch (err: any) {
+      result.errors.push(`Error processing ${basename(filePath)}: ${err.message}`);
+    }
+  }
+
+  return result;
+}
+
 // ============================================================================
 // Gemini Session Parser
 // ============================================================================
@@ -1650,7 +1944,8 @@ export async function ingestMarkdownFiles(
         continue;
       }
 
-      const texts = sections.map((s) => s.text);
+      // F-3b: scrub before embedding and storage
+      const texts = sections.map((s) => redactSecrets(s.text).text);
       const batchSize = 32;
       let fileChunks = 0;
 
@@ -1788,7 +2083,8 @@ export async function ingestGenericText(
 
       const chunkResult = chunkDocument(textToChunk, CONVERSATION_CHUNK_CONFIG);
 
-      const texts = chunkResult.chunks;
+      // F-3b: scrub before embedding and storage
+      const texts = chunkResult.chunks.map((t) => redactSecrets(t).text);
       const batchSize = 32;
 
       for (let i = 0; i < texts.length; i += batchSize) {

package/src/kg-extractor.ts

@@ -8,6 +8,7 @@ import type { LLMClient } from "./llm-client.js";
 import type { KGStore } from "./kg-store.js";
 import { logInfo, logWarn } from "./stderr-log.js";
 import { detectLang, getKgPromptHook } from "./language-hook.js";
+import * as envConfig from "./env-config.js";
 
 // ============================================================================
 // Types
@@ -39,7 +40,7 @@ interface LlmExtractionResponse {
 
 /** Check if KG mode is enabled (env var) */
 export function isKGModeEnabled(): boolean {
-  return process.env.RECALLNEST_KG_MODE === "true";
+  return envConfig.kgMode();
 }
 
 // ============================================================================
@@ -95,7 +96,13 @@ const DEFAULT_KG_SYSTEM = `You are a knowledge graph extraction assistant. Extra
 const DEFAULT_KG_USER_TEMPLATE = `Extract knowledge graph triples from the following text.
 Rules:
 - Use simple predicates: uses, created_by, works_with, is_a, part_of, has, located_in, belongs_to, depends_on, implements, extends, related_to, caused_by, results_in, precedes, follows
-- Assign confidence 0.0-1.0
+- confidence must be EXACTLY one of these five values (models collapse continuous ranges — pick the one matching the evidence):
+  1.0  = stated explicitly in the text
+  0.85 = strong inference (clearly implied, never stated)
+  0.75 = reasonable inference (requires interpretation)
+  0.65 = weak inference (thematic association only)
+  0.55 = speculative (surface co-occurrence)
+  Never use 0.5 or any value outside this list. If unsure between two tiers, pick the lower.
 - Extract 3-8 triples
 - Return JSON: { "triples": [{ "subject": "", "predicate": "", "object": "", "confidence": 0.0 }] }
 
@@ -163,8 +170,11 @@ export class KGExtractor {
         prompt,
       );
     } catch (err) {
+      // Transport/API failure is NOT "zero triples" — propagate so callers can
+      // distinguish (backfill must not journal it as processed). The capture
+      // path's fire-and-forget .catch() keeps memory writes unaffected.
       logWarn(`[KG] LLM extraction failed: ${String(err)}`);
-      return [];
+      throw err instanceof Error ? err : new Error(String(err));
     }
 
     if (!response?.triples || !Array.isArray(response.triples)) {