recallnest 2.2.1 → 2.5.4

package/src/ingest.ts

@@ -12,11 +12,16 @@ import { readFileSync, readdirSync, existsSync, statSync, writeFileSync } from "
 import { join, basename, resolve, dirname } from "node:path";
 import { homedir } from "node:os";
 import { detectLang, tokenizeFts } from "./language-hook.js";
+import { redactSecrets } from "./pii-detector.js";
 import type { MemoryStore, MemoryEntry } from "./store.js";
 import type { Embedder } from "./embedder.js";
 import { chunkDocument, type ChunkerConfig } from "./chunker.js";
+import type { ConnectorOutputV1, ConnectorRecord } from "./connector-types.js";
+import { isConnectorOutputV1 } from "./connector-types.js";
+import { isObsidianVault, scanVault } from "./obsidian-connector.js";
 import { segmentEvents, type EventSegmenterConfig, DEFAULT_EVENT_SEGMENTER_CONFIG } from "./event-segmenter.js";
 import { isProcessed, markProcessed } from "./tracker.js";
+import * as envConfig from "./env-config.js";
 import type { LLMClient, SmartExtraction } from "./llm-client.js";
 import { resolveIngestBoundary } from "./memory-boundaries.js";
 import { isNoise } from "./noise-filter.js";
@@ -504,7 +509,7 @@ async function smartExtractBatch(
  * Tier 3.1: Check if core summary generation is enabled.
  */
 function isCoreSummaryEnabled(): boolean {
-  return process.env.RECALLNEST_CORE_SUMMARY === "true";
+  return envConfig.coreSummary();
 }
 
 /**
@@ -622,7 +627,8 @@ export async function drainPendingQueue(
   let processed = 0, errors = 0;
   for (let i = 0; i < pending.length; i += 20) {
     const batch = pending.slice(i, i + 20);
-    const texts = batch.map(c => c.text);
+    // F-3b: scrub before the LLM extraction call and downstream embedding
+    const texts = batch.map(c => redactSecrets(c.text).text);
     const extractions = await smartExtractBatch(texts, llm);
     const embeddingTexts = extractions.map(e => e.l1 || e.l0);
     const vectors = await embedder.embedBatchPassage(embeddingTexts);
@@ -857,7 +863,7 @@ function filterNoiseTurns(turns: ConversationTurn[]): ConversationTurn[] {
   return turns.filter((turn) => !isNoise(turn.text));
 }
 
-function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
+export function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
   text: string;
   timestamp: string;
   sessionId: string;
@@ -930,7 +936,18 @@ function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
     }
   }
 
-  return chunks;
+  // P1 defence-in-depth: re-filter AFTER splitting. Long turns pass the
+  // turn-level filterNoiseTurns gate, but chunking can peel their tail into
+  // severed context-JSON fragments that the turn-level check never saw.
+  // F-3b: scrub secrets at the chunk boundary — this is the shared text
+  // finalization point for all session-ingest paths (CC/Codex/Gemini), before
+  // any chunk reaches embedding, LLM extraction, or storage.
+  return chunks
+    .filter((c) => !isNoise(c.text))
+    .map((c) => {
+      const r = redactSecrets(c.text);
+      return r.redacted > 0 ? { ...c, text: r.text } : c;
+    });
 }
 
 // ============================================================================
@@ -1024,9 +1041,14 @@ export async function ingestCodexSessions(
     errors: [],
   };
 
-  const baseDir = expandHome("~/.codex/sessions");
-  if (!existsSync(baseDir)) {
-    result.errors.push(`Codex sessions directory not found: ${baseDir}`);
+  // 同时扫 sessions（活跃）+ archived_sessions（App 内归档会把文件移到这里，否则会漏）
+  const baseDirs = [
+    expandHome("~/.codex/sessions"),
+    expandHome("~/.codex/archived_sessions"),
+  ];
+  const existingDirs = baseDirs.filter((d) => existsSync(d));
+  if (existingDirs.length === 0) {
+    result.errors.push(`Codex sessions directory not found: ${baseDirs.join(", ")}`);
     return result;
   }
 
@@ -1045,7 +1067,7 @@ export async function ingestCodexSessions(
       }
     }
   }
-  walk(baseDir);
+  for (const dir of existingDirs) walk(dir);
   allFiles.sort();
 
   // --recent: only keep files modified within N hours
@@ -1647,7 +1669,8 @@ export async function ingestMarkdownFiles(
         continue;
       }
 
-      const texts = sections.map((s) => s.text);
+      // F-3b: scrub before embedding and storage
+      const texts = sections.map((s) => redactSecrets(s.text).text);
       const batchSize = 32;
       let fileChunks = 0;
 
@@ -1785,7 +1808,8 @@ export async function ingestGenericText(
 
       const chunkResult = chunkDocument(textToChunk, CONVERSATION_CHUNK_CONFIG);
 
-      const texts = chunkResult.chunks;
+      // F-3b: scrub before embedding and storage
+      const texts = chunkResult.chunks.map((t) => redactSecrets(t).text);
       const batchSize = 32;
 
       for (let i = 0; i < texts.length; i += batchSize) {
@@ -1850,3 +1874,203 @@ export async function ingestGenericText(
 
   return result;
 }
+
+// ============================================================================
+// Connector-v1 Ingest (GB-2)
+// ============================================================================
+
+/**
+ * Ingest a connector-v1 formatted JSON payload.
+ *
+ * External connectors (Obsidian, Email, RSS, etc.) produce a standard
+ * ConnectorOutputV1 JSON → this function consumes it through the existing
+ * dedup / smartExtract / buildIngestedEntry pipeline.
+ *
+ * contentHash on each record enables incremental sync: if a record with the
+ * same hash is already in the store, it is skipped without embedding.
+ */
+export async function ingestConnectorFile(
+  store: MemoryStore,
+  embedder: Embedder,
+  content: string,
+  options: { verbose?: boolean; noDedup?: boolean; llm?: LLMClient | null } = {},
+): Promise<IngestResult> {
+  const parsed = JSON.parse(content);
+  if (!isConnectorOutputV1(parsed)) {
+    return {
+      source: "connector",
+      filesProcessed: 0,
+      chunksIngested: 0,
+      chunksSkipped: 0,
+      chunksDeduped: 0,
+      dedupReasonCounts: createDedupReasonCounts(),
+      errors: ["Invalid connector-v1 format: missing version/source/scope/records"],
+    };
+  }
+
+  const connectorData = parsed as ConnectorOutputV1;
+  const { source, scope, records } = connectorData;
+
+  const result: IngestResult = {
+    source: `connector:${source}`,
+    filesProcessed: 1,
+    chunksIngested: 0,
+    chunksSkipped: 0,
+    chunksDeduped: 0,
+    dedupReasonCounts: createDedupReasonCounts(),
+    errors: [],
+  };
+
+  if (records.length === 0) return result;
+
+  const batchSize = 32;
+
+  for (let i = 0; i < records.length; i += batchSize) {
+    const batch = records.slice(i, i + batchSize);
+    const texts = batch.map((r: ConnectorRecord) =>
+      r.title ? `[${r.title}] ${r.text}` : r.text,
+    );
+
+    try {
+      const vectors = await embedder.embedBatchPassage(texts);
+
+      // --- dedup pass ---
+      const dedupedTexts: string[] = [];
+      const dedupedVectors: number[][] = [];
+      const dedupedRecords: ConnectorRecord[] = [];
+
+      for (let j = 0; j < batch.length; j++) {
+        const vector = vectors[j];
+        if (!vector || vector.length === 0) {
+          result.chunksSkipped++;
+          continue;
+        }
+        if (!options.noDedup) {
+          const decision = await dedupCheck(store, vector, texts[j], options.llm);
+          recordDedupDecision(result, decision);
+          if (decision.secondaryDeletes?.length) {
+            await executeSecondaryDeletes(store, decision.secondaryDeletes);
+          }
+          if (decision.action === "skip") continue;
+        }
+        dedupedTexts.push(texts[j]);
+        dedupedVectors.push(vector);
+        dedupedRecords.push(batch[j]);
+      }
+
+      if (dedupedTexts.length === 0) continue;
+
+      // --- extract + build entries ---
+      const extractions = await smartExtractBatch(dedupedTexts, options.llm);
+      const coreSummaries = await generateCoreSummaries(dedupedTexts, options.llm);
+      const toStore: Array<{
+        text: string;
+        vector: number[];
+        category: string;
+        scope: string;
+        importance: number;
+        metadata: string;
+      }> = [];
+
+      for (let j = 0; j < dedupedTexts.length; j++) {
+        const record = dedupedRecords[j];
+        const entry = buildIngestedEntry({
+          source: `connector:${source}`,
+          scope,
+          text: dedupedTexts[j],
+          vector: dedupedVectors[j],
+          extraction: extractions[j],
+          file: record.id,
+          heading: record.title,
+          coreSummary: coreSummaries[j],
+        });
+
+        // Merge connector-specific metadata
+        const meta = JSON.parse(entry.metadata);
+        if (record.tags?.length) meta.connectorTags = record.tags;
+        if (record.contentHash) meta.contentHash = record.contentHash;
+        if (record.sourceMetadata) meta.sourceMetadata = record.sourceMetadata;
+        entry.metadata = JSON.stringify(meta);
+
+        // Respect categoryHint / importanceHint from connector
+        if (record.categoryHint) {
+          entry.category = record.categoryHint;
+        }
+        if (record.importanceHint !== undefined) {
+          entry.importance = record.importanceHint;
+        }
+
+        toStore.push(entry);
+      }
+
+      if (toStore.length > 0) {
+        await store.storeBatch(toStore);
+        result.chunksIngested += toStore.length;
+      }
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      result.errors.push(`Connector batch error: ${msg}`);
+    }
+  }
+
+  if (options.verbose) {
+    console.log(`  connector:${source}: ${records.length} records → ${result.chunksIngested} stored`);
+  }
+
+  // GB-3: Write source heartbeat after connector ingest
+  try {
+    const { writeHeartbeat } = await import("./source-heartbeat.js");
+    writeHeartbeat(source, result.chunksIngested, result.errors);
+  } catch {
+    // Heartbeat write is best-effort — don't break ingest on failure
+  }
+
+  return result;
+}
+
+// ============================================================================
+// Obsidian Vault Ingest (GB-1)
+// ============================================================================
+
+/**
+ * Scan an Obsidian vault and ingest its markdown files through the connector-v1 pipeline.
+ *
+ * Wrapper that:
+ * 1. Validates the path is an Obsidian vault (.obsidian/ exists)
+ * 2. Calls scanVault() to produce a ConnectorOutputV1 payload
+ * 3. Feeds the payload to ingestConnectorFile()
+ */
+export async function ingestObsidianVault(
+  store: MemoryStore,
+  embedder: Embedder,
+  vaultPath: string,
+  options: {
+    verbose?: boolean;
+    noDedup?: boolean;
+    llm?: LLMClient | null;
+    scopePrefix?: string;
+  } = {},
+): Promise<IngestResult> {
+  const resolved = expandHome(vaultPath);
+
+  if (!isObsidianVault(resolved)) {
+    return {
+      source: "obsidian",
+      filesProcessed: 0,
+      chunksIngested: 0,
+      chunksSkipped: 0,
+      chunksDeduped: 0,
+      dedupReasonCounts: createDedupReasonCounts(),
+      errors: [`Not an Obsidian vault (missing .obsidian/ directory): ${resolved}`],
+    };
+  }
+
+  const output = scanVault(resolved, { scopePrefix: options.scopePrefix });
+  const content = JSON.stringify(output);
+
+  if (options.verbose) {
+    console.log(`  obsidian: scanned ${output.records.length} notes from ${resolved}`);
+  }
+
+  return ingestConnectorFile(store, embedder, content, options);
+}

package/src/kg-extractor.ts

@@ -8,6 +8,7 @@ import type { LLMClient } from "./llm-client.js";
 import type { KGStore } from "./kg-store.js";
 import { logInfo, logWarn } from "./stderr-log.js";
 import { detectLang, getKgPromptHook } from "./language-hook.js";
+import * as envConfig from "./env-config.js";
 
 // ============================================================================
 // Types
@@ -39,7 +40,7 @@ interface LlmExtractionResponse {
 
 /** Check if KG mode is enabled (env var) */
 export function isKGModeEnabled(): boolean {
-  return process.env.RECALLNEST_KG_MODE === "true";
+  return envConfig.kgMode();
 }
 
 // ============================================================================
@@ -163,8 +164,11 @@ export class KGExtractor {
         prompt,
       );
     } catch (err) {
+      // Transport/API failure is NOT "zero triples" — propagate so callers can
+      // distinguish (backfill must not journal it as processed). The capture
+      // path's fire-and-forget .catch() keeps memory writes unaffected.
       logWarn(`[KG] LLM extraction failed: ${String(err)}`);
-      return [];
+      throw err instanceof Error ? err : new Error(String(err));
     }
 
     if (!response?.triples || !Array.isArray(response.triples)) {