recallnest 2.3.0 → 2.6.0

package/src/capture-engine.ts

@@ -1,13 +1,15 @@
 import type { Embedder } from "./embedder.js";
 import { detectLang, tokenizeFts } from "./language-hook.js";
 import { generateAnchor } from "./anchor-generator.js";
-import { incrementWriteCount } from "./activity-counter.js";
+import { extractErrorSignatures } from "./error-signature.js";
 import { verifyWrite } from "./write-verifier.js";
 // batchInternalDedup is available in ingest.ts for large-batch scenarios;
 // persistMemoryBatch relies on per-item conflict detection (A-2) instead.
-import { defaultEvolution, buildSupersedeMetadata, buildSupersedeMetadataForNew, buildPendingReviewMetadata, isActiveMemory, patchEvolution } from "./memory-evolution.js";
+import { defaultEvolution, buildSupersedeMetadata, buildSupersedeMetadataForNew, buildPendingReviewMetadata, isActiveMemory, parseEvolution, patchEvolution } from "./memory-evolution.js";
+import { archiveBeliefVersion, buildSupersedingBeliefMetadata } from "./belief-history.js";
 import { detectTopicTag, injectTopicTag } from "./topic-tag.js";
 import { assignDefaultConfidence, type ConfidenceMetadata } from "./confidence-tracker.js";
+import { parseDependsOnInput } from "./freshness.js";
 import {
   type CaseMemoryInput,
   CaseMemoryInputSchema,
@@ -56,12 +58,13 @@ import {
 import { matchPreference, applyPreferenceMatch } from "./preference-matcher.js";
 import type { LLMClient } from "./llm-client.js";
 import type { KGExtractor } from "./kg-extractor.js";
-import { scanForPII } from "./pii-detector.js";
+import { scanForPII, redactSecrets } from "./pii-detector.js";
 import type { AuditLogger } from "./audit-log.js";
 import { checkAdmission, type ScopeRateLimiter, type AdmissionConfig } from "./admission-control.js";
+import { resolveNoisePrototypeBank, type NoisePrototypeBank } from "./noise-prototype-bank.js";
 import { tagNarrativeIfEnabled } from "./narrative-tagger.js";
 
-type StoreDeps = Pick<MemoryStore, "store"> & Partial<Pick<MemoryStore, "list" | "update" | "getById" | "get" | "vectorSearch">>;
+type StoreDeps = Pick<MemoryStore, "store"> & Partial<Pick<MemoryStore, "list" | "update" | "getById" | "get" | "vectorSearch" | "upsert">>;
 type ConflictStoreDeps = Pick<ConflictCandidateStore, "save" | "replace" | "getOpenByFingerprint" | "getLatestByFingerprint">;
 
 export interface PersistMemoryDeps {
@@ -78,9 +81,15 @@ export interface PersistMemoryDeps {
   rateLimiter?: ScopeRateLimiter | null;
   /** LME-8: Optional admission config overrides */
   admissionConfig?: Partial<AdmissionConfig>;
+  /**
+   * Noise-prototype shadow experiment (mlp borrow). undefined → env-governed
+   * shared instance (RECALLNEST_NOISE_PROTOTYPE, shadow default); null → off.
+   * Shadow-only: it observes and logs, it never rejects.
+   */
+  noisePrototypeBank?: NoisePrototypeBank | null;
 }
 
-interface DurableWriteInput {
+export interface DurableWriteInput {
   text: string;
   vector: number[];
   category: DurableMemoryCategory;
@@ -140,7 +149,7 @@ function mergeTags(...groups: string[][]): string[] {
   return merged.slice(0, 8);
 }
 
-function buildStructuredMetadata(params: {
+export function buildStructuredMetadata(params: {
   source: string;
   tags: string[];
   capture: string;
@@ -437,11 +446,15 @@ async function findCanonicalMatches(
   return entries.filter((entry) => {
     const boundary = extractBoundaryMetadata(entry.metadata);
     if (boundary?.layer !== "durable") return false;
+    // Belief-history rows carry the same canonicalKey as the live row they were copied
+    // from. Without this guard they'd be treated as dedupe/conflict targets — so rewriting
+    // a belief back to an older wording would look like a duplicate, not a change.
+    if (!isActiveMemory(entry.metadata)) return false;
     return extractCanonicalKey(entry.metadata) === canonicalKey;
   });
 }
 
-async function writeDurableEntry(
+export async function writeDurableEntry(
   deps: PersistMemoryDeps,
   params: DurableWriteInput,
 ): Promise<{
@@ -563,13 +576,26 @@ async function writeDurableEntry(
       }
     }
 
+    // latest-wins is where beliefs change. Archive the version being replaced first —
+    // overwriting in place used to delete the old text + vector outright, which is what
+    // made "what did I believe last month?" unanswerable no matter how retrieval queried.
+    const now = Date.now();
+    const previousEvo = parseEvolution(latest.metadata, latest.timestamp);
+    const { historyId } = await archiveBeliefVersion(
+      { store: deps.store, embedder: deps.embedder },
+      latest,
+      { now },
+    );
+
     const updated = await deps.store.update(latest.id, {
       text: params.text,
       vector: params.vector,
       importance: params.importance,
       category: params.category,
-      metadata: params.metadata,
-      timestamp: Date.now(),
+      // Merge rather than replace: a bare params.metadata would wipe the evolution block
+      // (version, access stats, supersede links) along with the text.
+      metadata: buildSupersedingBeliefMetadata(params.metadata, previousEvo, historyId, now),
+      timestamp: now,
       ...(params.language ? { language: params.language } : {}),
       ...(params.fts_text ? { fts_text: params.fts_text } : {}),
     });
@@ -582,16 +608,38 @@ async function writeDurableEntry(
     };
   }
 
+  // No canonical match was found, so this falls through to store(). With a canonicalKey the
+  // id is deterministic and store() upserts on id — so a row already sitting at that id is
+  // overwritten here without ever reaching the latest-wins branch above. findCanonicalMatches
+  // only scans the most recent CANONICAL_SCAN_LIMIT rows, so any canonical row older than that
+  // window lands on this path. Look it up by id directly, which no scan limit can hide.
+  const targetId = params.canonicalKey
+    ? deterministicId(params.scope, params.canonicalKey)
+    : undefined;
+  let metadataForStore = params.metadata;
+
+  if (targetId && deps.store.getById) {
+    const rowAtTargetId = await deps.store.getById(targetId);
+    if (rowAtTargetId && normalizeMemoryText(rowAtTargetId.text) !== normalizeMemoryText(params.text)) {
+      const now = Date.now();
+      const previousEvo = parseEvolution(rowAtTargetId.metadata, rowAtTargetId.timestamp);
+      const { historyId } = await archiveBeliefVersion(
+        { store: deps.store, embedder: deps.embedder },
+        rowAtTargetId,
+        { now },
+      );
+      metadataForStore = buildSupersedingBeliefMetadata(params.metadata, previousEvo, historyId, now);
+    }
+  }
+
   const stored = await deps.store.store({
-    id: params.canonicalKey
-      ? deterministicId(params.scope, params.canonicalKey)
-      : undefined,
+    id: targetId,
     text: params.text,
     vector: params.vector,
     category: params.category,
     scope: params.scope,
     importance: params.importance,
-    metadata: params.metadata,
+    metadata: metadataForStore,
     ...(params.language ? { language: params.language } : {}),
     ...(params.fts_text ? { fts_text: params.fts_text } : {}),
   });
@@ -699,6 +747,17 @@ function buildCaseMemoryMetadata(
       tools: input.tools,
     },
   };
+  // A1: error-signature 指纹（读写对称，P3-A 检索复用同一抽取器）。
+  // 在 build metadata 前算好作为 extra 写入，不 post-write update（Codex 验证要求）。
+  const errorSignature = extractErrorSignatures({
+    problem: input.problem,
+    context: input.context,
+    outcome: input.outcome,
+    solutionSteps: input.solutionSteps,
+  });
+  if (errorSignature.length > 0) cmExtra.error_signature = errorSignature;
+  // A1: break-loop 五维（可选，向后兼容；不提供则普通 case）。
+  if (input.debugFraming) cmExtra.debugFraming = input.debugFraming;
   const anchor = generateAnchor(input.title + ": " + input.problem, cmExtra);
   if (anchor) cmExtra.anchor = anchor;
   return buildStructuredMetadata({
@@ -746,7 +805,16 @@ export async function persistMemory(
   deps: PersistMemoryDeps,
   rawInput: unknown,
 ): Promise<StoredMemoryRecord> {
-  const input = StoreMemoryInputSchema.parse(rawInput);
+  let input = StoreMemoryInputSchema.parse(rawInput);
+
+  // F-3b: Scan THEN redact, before the text reaches any external surface —
+  // LLM summary/importance calls, embedding API, storage, audit log, KG.
+  // Scan first so the piiWarning below still reflects what the raw text held.
+  const piiScan = scanForPII(input.text);
+  const redaction = redactSecrets(input.text);
+  if (redaction.redacted > 0) {
+    input = { ...input, text: redaction.text };
+  }
 
   // F3: Extract temporal validity params before Zod strips them
   const rawObj = rawInput as Record<string, unknown> | null;
@@ -801,6 +869,28 @@ export async function persistMemory(
     deps.admissionConfig,
   );
   if (admission.verdict === "rejected") {
+    // Gemini #26523 lesson: rejected writes must stay queryable after the fact,
+    // not just echo back to the caller — otherwise they become invisible noise.
+    try {
+      deps.auditLogger?.log({
+        operation: "reject",
+        scope: admissionScope,
+        actor: input.source,
+        details: `${admission.reason ?? "unknown"}: ${input.text.slice(0, 100)}`,
+      });
+    } catch {
+      // Audit must never block memory writes
+    }
+    // Noise-prototype shadow experiment: learn ONLY from the regex noise
+    // filter's own verdict (mlp #914 lesson — a heuristic's low-importance
+    // self-judgment must not be recycled as a noise label). Embedding only
+    // happens on a repeat sighting under the bank cap, so this stays cheap.
+    if (admission.reason === "noise_detected") {
+      const bank = resolveNoisePrototypeBank(deps.noisePrototypeBank);
+      if (bank) {
+        await bank.learnFromRejection(input.text, (text) => deps.embedder.embedPassage(text));
+      }
+    }
     return {
       id: `rejected-${Date.now()}`,
       text: input.text,
@@ -812,10 +902,20 @@ export async function persistMemory(
       resolvedScope: admissionScope,
       storedAt: new Date().toISOString(),
       disposition: "rejected" as const,
+      ...(admission.reason ? { rejectionReason: admission.reason } : {}),
     };
   }
 
   const vector = await deps.embedder.embedPassage(input.text);
+
+  // Noise-prototype shadow experiment, matching side: reuses the write-path
+  // embedding just computed (zero extra embedding). Log-only — the verdict
+  // never blocks this write while the experiment is in shadow.
+  {
+    const bank = resolveNoisePrototypeBank(deps.noisePrototypeBank);
+    bank?.matchShadow(input.text, vector);
+  }
+
   const language = detectLang(input.text);
   const fts_text = tokenizeFts(input.text, language);
   const resolvedScope = resolveScope(input);
@@ -843,6 +943,7 @@ export async function persistMemory(
         matchResult,
         deps.store as MemoryStore,
         resolvedScope,
+        { embedder: deps.embedder },
       );
       if (applied.handled) {
         const entry = applied.entry ?? {
@@ -937,15 +1038,16 @@ export async function persistMemory(
   }
 
   // F-3: PII detection — inject warning into metadata if PII found.
-  // Non-blocking: PII does not prevent the write.
+  // Non-blocking: PII does not prevent the write. Uses the pre-redaction scan
+  // from the top of persistMemory (scanning here would see scrubbed text).
   let metadata = buildStoreMemoryMetadata(input, canonicalKey);
-  const piiResult = scanForPII(input.text);
-  if (piiResult.hasPII) {
+  if (piiScan.hasPII) {
     const parsed: Record<string, unknown> = JSON.parse(metadata);
     parsed.piiWarning = {
-      summary: piiResult.summary,
-      detections: piiResult.detections.length,
-      severity: piiResult.detections.some(d => d.severity === "high") ? "high" : "medium",
+      summary: piiScan.summary,
+      detections: piiScan.detections.length,
+      severity: piiScan.detections.some(d => d.severity === "high") ? "high" : "medium",
+      ...(redaction.redacted > 0 ? { redacted: redaction.redacted } : {}),
     };
     metadata = JSON.stringify(parsed);
   }
@@ -995,6 +1097,15 @@ export async function persistMemory(
     metadata = JSON.stringify(parsed);
   }
 
+  // Freshness (borrowed 2026-07-21): opt-in dependsOn declarations ride in metadata like
+  // validUntil/confidence do — pulled from raw input so StoreMemoryInputSchema stays untouched.
+  const dependsOn = parseDependsOnInput(rawObj?.dependsOn);
+  if (dependsOn) {
+    const parsed: Record<string, unknown> = JSON.parse(metadata);
+    parsed.dependsOn = dependsOn;
+    metadata = JSON.stringify(parsed);
+  }
+
   const { entry, disposition, conflictId } = await writeDurableEntry(deps, {
     text: input.text,
     vector,
@@ -1031,14 +1142,9 @@ export async function persistMemory(
     // Audit must never block memory writes
   }
 
-  // HP-3: Activity counter — track non-dedup writes for distill trigger
-  if (disposition !== "deduped") {
-    try {
-      incrementWriteCount();
-    } catch {
-      // Activity tracking must never block memory writes
-    }
-  }
+  // HP-3 activity counter moved to the store layer (store/upsert/storeBatch count per
+  // scope): that is the necessary chokepoint for every write source (capture/ingest/
+  // store_memory/consolidation), and per-scope counts drive per-scope dream activation.
 
   // HP-2: Post-write verification — async, non-blocking
   if (disposition !== "deduped" && deps.store.get) {
@@ -1063,7 +1169,7 @@ export async function persistMemory(
     input.category !== "preferences" &&
     inferImplicitUsageSlot(input.text)
   ) {
-    const prefCanonicalKey = buildDefaultCanonicalKey("preferences", input.text);
+    const prefCanonicalKey = buildDefaultCanonicalKey({ category: "preferences", text: input.text });
     const prefSlot = inferImplicitUsageSlot(input.text);
     const prefMetadata = buildStructuredMetadata({
       source: input.source,
@@ -1098,7 +1204,8 @@ export async function persistWorkflowPattern(
   const input = WorkflowPatternInputSchema.parse(rawInput);
   const resolvedScope = resolveScope(input);
   const tags = mergeTags(input.tags, ["workflow", "pattern"]);
-  const text = buildWorkflowPatternText(input);
+  // F-3b: scrub before embedding/storage (this path bypasses persistMemory)
+  const text = redactSecrets(buildWorkflowPatternText(input)).text;
   const vector = await deps.embedder.embedPassage(text);
   const language = detectLang(text);
   const fts_text = tokenizeFts(text, language);
@@ -1142,7 +1249,8 @@ export async function persistCaseMemory(
   const input = CaseMemoryInputSchema.parse(rawInput);
   const resolvedScope = resolveScope(input);
   const tags = mergeTags(input.tags, ["case", "solution"]);
-  const text = buildCaseMemoryText(input);
+  // F-3b: scrub before embedding/storage (this path bypasses persistMemory)
+  const text = redactSecrets(buildCaseMemoryText(input)).text;
   const vector = await deps.embedder.embedPassage(text);
   const language = detectLang(text);
   const fts_text = tokenizeFts(text, language);
@@ -1202,7 +1310,9 @@ export async function promoteMemory(
   }
 
   const category = inferPromotedCategory(input, sourceEntry);
-  const text = input.text || sourceEntry.text;
+  // F-3b: scrub caller-supplied text AND legacy stored text (pre-redaction
+  // era entries may still hold secrets; promotion is a natural wash point)
+  const text = redactSecrets(input.text || sourceEntry.text).text;
   const canonicalKey = resolveCanonicalKey({
     category,
     text,