recallnest 1.4.1 → 2.2.1

package/src/capture-engine.ts

@@ -1,4 +1,13 @@
 import type { Embedder } from "./embedder.js";
+import { detectLang, tokenizeFts } from "./language-hook.js";
+import { generateAnchor } from "./anchor-generator.js";
+import { incrementWriteCount } from "./activity-counter.js";
+import { verifyWrite } from "./write-verifier.js";
+// batchInternalDedup is available in ingest.ts for large-batch scenarios;
+// persistMemoryBatch relies on per-item conflict detection (A-2) instead.
+import { defaultEvolution, buildSupersedeMetadata, buildSupersedeMetadataForNew, buildPendingReviewMetadata, isActiveMemory, patchEvolution } from "./memory-evolution.js";
+import { detectTopicTag, injectTopicTag } from "./topic-tag.js";
+import { assignDefaultConfidence, type ConfidenceMetadata } from "./confidence-tracker.js";
 import {
   type CaseMemoryInput,
   CaseMemoryInputSchema,
@@ -17,6 +26,7 @@ import {
   type DurableMemoryCategory,
   DURABLE_MEMORY_CATEGORIES,
   type WriteDisposition,
+  parsePrivacyTier,
 } from "./memory-schema.js";
 import { deterministicId, type MemoryEntry, type MemoryStore } from "./store.js";
 import {
@@ -36,6 +46,7 @@ import { buildConflictCandidateRecord, buildConflictFingerprint, reopenConflictC
 import type { ConflictCandidateStore } from "./conflict-store.js";
 import {
   inferPreferenceSlot,
+  inferImplicitUsageSlot,
   samePreferenceSlot,
   type AtomicBrandItemPreferenceSlot,
   type PreferenceSlot,
@@ -45,11 +56,15 @@ import {
 import { matchPreference, applyPreferenceMatch } from "./preference-matcher.js";
 import type { LLMClient } from "./llm-client.js";
 import type { KGExtractor } from "./kg-extractor.js";
+import { scanForPII } from "./pii-detector.js";
+import type { AuditLogger } from "./audit-log.js";
+import { checkAdmission, type ScopeRateLimiter, type AdmissionConfig } from "./admission-control.js";
+import { tagNarrativeIfEnabled } from "./narrative-tagger.js";
 
 type StoreDeps = Pick<MemoryStore, "store"> & Partial<Pick<MemoryStore, "list" | "update" | "getById" | "get" | "vectorSearch">>;
 type ConflictStoreDeps = Pick<ConflictCandidateStore, "save" | "replace" | "getOpenByFingerprint" | "getLatestByFingerprint">;
 
-interface PersistMemoryDeps {
+export interface PersistMemoryDeps {
   store: StoreDeps;
   embedder: Pick<Embedder, "embedPassage">;
   conflictStore?: ConflictStoreDeps;
@@ -57,6 +72,12 @@ interface PersistMemoryDeps {
   llm?: LLMClient | null;
   /** Tier 4.1: Optional KG triple extractor (async, non-blocking) */
   kgExtractor?: KGExtractor | null;
+  /** F-1: Optional audit logger for recording store operations */
+  auditLogger?: AuditLogger | null;
+  /** LME-8: Optional rate limiter for admission control */
+  rateLimiter?: ScopeRateLimiter | null;
+  /** LME-8: Optional admission config overrides */
+  admissionConfig?: Partial<AdmissionConfig>;
 }
 
 interface DurableWriteInput {
@@ -71,6 +92,8 @@ interface DurableWriteInput {
   source: "manual" | "agent" | "api";
   sourceCategory?: string;
   sourceBoundary?: ReturnType<typeof extractBoundaryMetadata>;
+  language?: string;
+  fts_text?: string;
 }
 
 const CANONICAL_SCAN_LIMIT = 1000;
@@ -124,14 +147,28 @@ function buildStructuredMetadata(params: {
   category: DurableMemoryCategory;
   canonicalKey: string;
   extra?: Record<string, unknown>;
+  /** Optional: scope + text for narrative tagging (Phase 3) */
+  narrativeInput?: { scope: string; text: string; sessionId?: string };
 }): string {
+  // HP-narrative: Tag with autobiographical narrative metadata when enabled
+  const narrative = params.narrativeInput
+    ? tagNarrativeIfEnabled({
+        scope: params.narrativeInput.scope,
+        text: params.narrativeInput.text,
+        timestamp: Date.now(),
+        sessionId: params.narrativeInput.sessionId,
+      })
+    : null;
+
   return JSON.stringify({
     source: params.source,
     tags: params.tags,
     capture: params.capture,
     boundary: buildStructuredMemoryBoundary(params.category),
     canonicalKey: params.canonicalKey,
+    evolution: defaultEvolution(),
     ...params.extra,
+    ...(narrative ? { narrative } : {}),
   });
 }
 
@@ -533,6 +570,8 @@ async function writeDurableEntry(
       category: params.category,
       metadata: params.metadata,
       timestamp: Date.now(),
+      ...(params.language ? { language: params.language } : {}),
+      ...(params.fts_text ? { fts_text: params.fts_text } : {}),
     });
     if (!updated) {
       throw new Error(`Failed to update canonical memory ${latest.id}`);
@@ -553,6 +592,8 @@ async function writeDurableEntry(
     scope: params.scope,
     importance: params.importance,
     metadata: params.metadata,
+    ...(params.language ? { language: params.language } : {}),
+    ...(params.fts_text ? { fts_text: params.fts_text } : {}),
   });
 
   return {
@@ -599,13 +640,20 @@ function inferPromotedCategory(
 }
 
 function buildStoreMemoryMetadata(input: StoreMemoryInput, canonicalKey: string): string {
+  const anchor = generateAnchor(input.text);
+  const anchorExtra = anchor ? { anchor } : undefined;
+  // HP-ethics: Include privacyTier in metadata if non-default
+  const privacyExtra = input.privacyTier && input.privacyTier !== "durable"
+    ? { privacyTier: input.privacyTier }
+    : undefined;
   return buildStructuredMetadata({
     source: input.source,
     tags: input.tags,
     capture: "store_memory_schema_v1",
     category: input.category,
     canonicalKey,
-    extra: buildPreferenceSlotExtra(input.category, input.text),
+    extra: mergeExtra(buildPreferenceSlotExtra(input.category, input.text), anchorExtra, privacyExtra),
+    narrativeInput: { scope: input.scope, text: input.text },
   });
 }
 
@@ -614,21 +662,25 @@ function buildWorkflowPatternMetadata(
   tags: string[],
   canonicalKey: string,
 ): string {
+  const wpExtra: Record<string, unknown> = {
+    workflowPattern: {
+      title: input.title,
+      trigger: input.trigger,
+      steps: input.steps,
+      outcome: input.outcome,
+      tools: input.tools,
+    },
+  };
+  const anchor = generateAnchor(input.title + ": " + input.trigger, wpExtra);
+  if (anchor) wpExtra.anchor = anchor;
   return buildStructuredMetadata({
     source: input.source,
     tags,
     capture: "workflow_pattern_schema_v1",
     category: "patterns",
     canonicalKey,
-    extra: {
-      workflowPattern: {
-        title: input.title,
-        trigger: input.trigger,
-        steps: input.steps,
-        outcome: input.outcome,
-        tools: input.tools,
-      },
-    },
+    extra: wpExtra,
+    narrativeInput: { scope: input.scope, text: input.title + ": " + input.trigger },
   });
 }
 
@@ -637,22 +689,26 @@ function buildCaseMemoryMetadata(
   tags: string[],
   canonicalKey: string,
 ): string {
+  const cmExtra: Record<string, unknown> = {
+    caseMemory: {
+      title: input.title,
+      problem: input.problem,
+      context: input.context,
+      solutionSteps: input.solutionSteps,
+      outcome: input.outcome,
+      tools: input.tools,
+    },
+  };
+  const anchor = generateAnchor(input.title + ": " + input.problem, cmExtra);
+  if (anchor) cmExtra.anchor = anchor;
   return buildStructuredMetadata({
     source: input.source,
     tags,
     capture: "case_memory_schema_v1",
     category: "cases",
     canonicalKey,
-    extra: {
-      caseMemory: {
-        title: input.title,
-        problem: input.problem,
-        context: input.context,
-        solutionSteps: input.solutionSteps,
-        outcome: input.outcome,
-        tools: input.tools,
-      },
-    },
+    extra: cmExtra,
+    narrativeInput: { scope: input.scope, text: input.title + ": " + input.problem },
   });
 }
 
@@ -682,6 +738,7 @@ function buildPromotionMetadata(
         },
       },
     ),
+    narrativeInput: { scope: input.scope, text },
   });
 }
 
@@ -690,7 +747,77 @@ export async function persistMemory(
   rawInput: unknown,
 ): Promise<StoredMemoryRecord> {
   const input = StoreMemoryInputSchema.parse(rawInput);
+
+  // F3: Extract temporal validity params before Zod strips them
+  const rawObj = rawInput as Record<string, unknown> | null;
+  const validUntilRaw = rawObj?.validUntil;
+  const eventTimeRaw = rawObj?.eventTime;
+
+  // LC-P5: Large text gate — texts > 8000 chars get truncated to L0 summary.
+  // Prevents oversized entries from bloating the index and degrading embedding quality.
+  const LARGE_TEXT_THRESHOLD = 8000;
+  if (input.text.length > LARGE_TEXT_THRESHOLD && deps.llm) {
+    try {
+      const summary = await deps.llm.generateCoreSummary(input.text);
+      if (summary) {
+        // Store original length in metadata for reference, replace text with summary
+        const origLength = input.text.length;
+        input.text = summary;
+        input.tags = [...(input.tags ?? []), `truncated:${origLength}`];
+      }
+    } catch {
+      // Fallback: hard truncate if LLM fails
+      input.text = input.text.slice(0, LARGE_TEXT_THRESHOLD);
+    }
+  } else if (input.text.length > LARGE_TEXT_THRESHOLD) {
+    // No LLM available — hard truncate
+    input.text = input.text.slice(0, LARGE_TEXT_THRESHOLD);
+  }
+
+  // B-3: LLM importance assessment — refine default importance (0.7) via LLM.
+  // Only fires when importance is at the default value and LLM is available.
+  // Explicit importance values from the caller are respected as-is.
+  let pendingReview = false; // HP-6: track low-confidence assessment
+  if (input.importance === 0.7 && deps.llm) {
+    try {
+      const assessed = await deps.llm.assessImportance(input.text, input.category);
+      if (assessed !== null) {
+        input.importance = assessed;
+        // HP-6: Very low importance from LLM → pending review for distill to re-evaluate
+        if (assessed < 0.3) pendingReview = true;
+      }
+    } catch {
+      // LLM importance assessment must never block memory writes
+    }
+  }
+
+  // LME-8: Admission control — reject low-quality writes before expensive embedding
+  const admissionScope = input.scope || "unknown";
+  const admission = checkAdmission(
+    input.text,
+    input.importance,
+    admissionScope,
+    deps.rateLimiter ?? undefined,
+    deps.admissionConfig,
+  );
+  if (admission.verdict === "rejected") {
+    return {
+      id: `rejected-${Date.now()}`,
+      text: input.text,
+      category: input.category,
+      importance: input.importance,
+      scope: admissionScope,
+      source: input.source,
+      tags: input.tags ?? [],
+      resolvedScope: admissionScope,
+      storedAt: new Date().toISOString(),
+      disposition: "rejected" as const,
+    };
+  }
+
   const vector = await deps.embedder.embedPassage(input.text);
+  const language = detectLang(input.text);
+  const fts_text = tokenizeFts(input.text, language);
   const resolvedScope = resolveScope(input);
   const canonicalKey = resolveCanonicalKey({
     category: input.category,
@@ -738,24 +865,229 @@ export async function persistMemory(
     }
   }
 
+  // A-2: Evolution-aware semantic conflict detection.
+  // For non-preferences categories, check if a semantically similar memory
+  // already exists in the same scope. If so, use LLM to decide whether the
+  // old memory should be superseded.
+  // Requires both vectorSearch (for candidate lookup) and LLM (for judgment).
+  // Graceful fallback: if either is unavailable, skip detection entirely.
+  let supersededOldId: string | null = null; // HP-1: track for bidirectional link
+  let supersededReason: string | null = null;
+  if (
+    input.category !== "preferences" &&
+    deps.store.vectorSearch &&
+    deps.llm &&
+    deps.store.update
+  ) {
+    try {
+      const candidates = await deps.store.vectorSearch(vector, 3, 0.85, [resolvedScope]);
+      // Only consider same-category, active memories as supersede targets
+      const supersedeTargets = candidates.filter(
+        c => c.entry.category === input.category &&
+             c.entry.id !== deterministicId(resolvedScope, input.text) &&
+             isActiveMemory(c.entry.metadata),
+      );
+      if (supersedeTargets.length > 0) {
+        const decision = await deps.llm.dedupDecision(input.text, supersedeTargets[0].entry.text);
+        if (decision.action === "MERGE" || decision.action === "SKIP") {
+          // LLM says new info overlaps with existing → supersede the old one
+          // (MERGE = new has extra info → store new, mark old superseded)
+          // (SKIP  = identical → just dedup, handled by writeDurableEntry)
+          if (decision.action === "MERGE") {
+            const oldEntry = supersedeTargets[0].entry;
+            const supersededMeta = buildSupersedeMetadata(oldEntry.metadata, deterministicId(resolvedScope, input.text));
+            await deps.store.update(oldEntry.id, { metadata: supersededMeta });
+            // HP-1: Track for bidirectional link
+            supersededOldId = oldEntry.id;
+            supersededReason = decision.reason;
+          }
+        }
+        // CREATE or error → proceed normally (store new memory as-is)
+      }
+    } catch {
+      // Evolution conflict detection must never block memory writes
+    }
+  }
+
+  // F2: Interference pre-warning — if scope already has ≥5 high-similarity memories,
+  // mark the weakest as pending_review and record an interferenceNote.
+  if (deps.store.vectorSearch && deps.store.update) {
+    try {
+      const nearMatches = await deps.store.vectorSearch(vector, 6, 0.82, [resolvedScope]);
+      const activeNear = nearMatches.filter(
+        c => c.entry.id !== deterministicId(resolvedScope, input.text) &&
+             isActiveMemory(c.entry.metadata),
+      );
+      if (activeNear.length >= 5) {
+        // Find weakest by importance × confidence
+        const weakest = activeNear.reduce((min, cur) => {
+          const curScore = (cur.entry.importance ?? 0.5) * cur.score;
+          const minScore = (min.entry.importance ?? 0.5) * min.score;
+          return curScore < minScore ? cur : min;
+        });
+        const reviewMeta = buildPendingReviewMetadata(weakest.entry.metadata);
+        const parsed: Record<string, unknown> = JSON.parse(reviewMeta);
+        const evo = parsed.evolution as Record<string, unknown> | undefined;
+        if (evo) evo.evolutionNote = `F2: interference cluster detected (${activeNear.length} similar active memories)`;
+        await deps.store.update(weakest.entry.id, { metadata: JSON.stringify(parsed) });
+      }
+    } catch {
+      // Interference detection must never block memory writes
+    }
+  }
+
+  // F-3: PII detection — inject warning into metadata if PII found.
+  // Non-blocking: PII does not prevent the write.
+  let metadata = buildStoreMemoryMetadata(input, canonicalKey);
+  const piiResult = scanForPII(input.text);
+  if (piiResult.hasPII) {
+    const parsed: Record<string, unknown> = JSON.parse(metadata);
+    parsed.piiWarning = {
+      summary: piiResult.summary,
+      detections: piiResult.detections.length,
+      severity: piiResult.detections.some(d => d.severity === "high") ? "high" : "medium",
+    };
+    metadata = JSON.stringify(parsed);
+  }
+
+  // MP-1: Topic Tag — auto-detect or use explicit tag from caller
+  const topicTag = (input as Record<string, unknown>).topicTag as string | undefined
+    ?? detectTopicTag(input.text);
+  if (topicTag) {
+    metadata = injectTopicTag(metadata, topicTag);
+  }
+
+  // HP-1: Bidirectional supersede link — mark new memory as superseding old
+  if (supersededOldId) {
+    metadata = buildSupersedeMetadataForNew(metadata, supersededOldId, supersededReason ?? undefined);
+  }
+
+  // HP-6: Low-confidence entries → pending_review (distill prioritizes these)
+  if (pendingReview) {
+    metadata = buildPendingReviewMetadata(metadata);
+  }
+
+  // F3: Temporal validity — inject validUntil and eventTime into evolution metadata
+  if (validUntilRaw != null || eventTimeRaw != null) {
+    const temporalPatch: Record<string, number> = {};
+    if (validUntilRaw != null) {
+      temporalPatch.validUntil = typeof validUntilRaw === "number"
+        ? validUntilRaw
+        : new Date(String(validUntilRaw)).getTime();
+    }
+    if (eventTimeRaw != null) {
+      temporalPatch.eventTime = typeof eventTimeRaw === "number"
+        ? eventTimeRaw
+        : new Date(String(eventTimeRaw)).getTime();
+    }
+    metadata = patchEvolution(metadata, temporalPatch);
+  }
+
+  // F1: Assign structured confidence based on source (user can override via explicit param)
+  const confidenceRaw = rawObj?.confidence as Partial<ConfidenceMetadata> | number | undefined;
+  const explicitConfidence = typeof confidenceRaw === "number"
+    ? { score: confidenceRaw }
+    : (confidenceRaw ?? undefined);
+  const confidenceMeta = assignDefaultConfidence(input.source, explicitConfidence);
+  {
+    const parsed: Record<string, unknown> = JSON.parse(metadata);
+    parsed.confidence = confidenceMeta;
+    metadata = JSON.stringify(parsed);
+  }
+
   const { entry, disposition, conflictId } = await writeDurableEntry(deps, {
     text: input.text,
     vector,
     category: input.category,
     scope: resolvedScope,
     importance: input.importance,
-    metadata: buildStoreMemoryMetadata(input, canonicalKey),
+    metadata,
     canonicalKey,
     source: input.source,
+    language,
+    fts_text,
   });
 
   // Tier 4.1: Async KG triple extraction (non-blocking)
-  if (deps.kgExtractor && disposition !== "deduped") {
+  // HP-ethics: Skip KG extraction for ephemeral/private memories — they must not leave graph traces
+  const entryPrivacyTier = parsePrivacyTier(metadata);
+  const kgAllowed = entryPrivacyTier !== "ephemeral" && entryPrivacyTier !== "private";
+  if (deps.kgExtractor && disposition !== "deduped" && kgAllowed) {
     deps.kgExtractor
       .extractAndStore(input.text, entry.id, resolvedScope)
       .catch(() => {}); // Silently ignore — KG extraction must never block memory writes
   }
 
+  // F-1: Audit log — record store operation (non-blocking, silent on failure)
+  try {
+    deps.auditLogger?.log({
+      operation: "store",
+      scope: resolvedScope,
+      memoryId: entry.id,
+      actor: input.source,
+      details: `${input.category}: ${input.text.slice(0, 100)}`,
+    });
+  } catch {
+    // Audit must never block memory writes
+  }
+
+  // HP-3: Activity counter — track non-dedup writes for distill trigger
+  if (disposition !== "deduped") {
+    try {
+      incrementWriteCount();
+    } catch {
+      // Activity tracking must never block memory writes
+    }
+  }
+
+  // HP-2: Post-write verification — async, non-blocking
+  if (disposition !== "deduped" && deps.store.get) {
+    verifyWrite(deps.store as Pick<import("./store.js").MemoryStore, "get">, entry.id)
+      .then(result => {
+        if (!result.ok) {
+          console.error(
+            `[HP-2] Write verification issues for ${entry.id}: ${result.issues.join(", ")} (${result.durationMs}ms)`,
+          );
+        }
+      })
+      .catch(() => {}); // Must never block
+  }
+
+  // LME-1: Implicit preference dual-write — when a non-preference memory
+  // contains an implicit usage signal ("I use X", "I have X", etc.),
+  // store an additional preferences copy so it can be recalled via the
+  // low-threshold preference retrieval path (hardMinScore 0.25 vs 0.35).
+  // Async & non-blocking: the primary write is already done.
+  if (
+    disposition !== "deduped" &&
+    input.category !== "preferences" &&
+    inferImplicitUsageSlot(input.text)
+  ) {
+    const prefCanonicalKey = buildDefaultCanonicalKey("preferences", input.text);
+    const prefSlot = inferImplicitUsageSlot(input.text);
+    const prefMetadata = buildStructuredMetadata({
+      source: input.source,
+      tags: [...(input.tags ?? []), "derived-preference", `derived-from:${entry.id}`],
+      capture: "implicit_preference_dual_write_v1",
+      category: "preferences",
+      canonicalKey: prefCanonicalKey,
+      extra: prefSlot ? { preferenceSlot: prefSlot } : undefined,
+      narrativeInput: { scope: input.scope, text: input.text },
+    });
+    writeDurableEntry(deps, {
+      text: input.text,
+      vector,
+      category: "preferences",
+      scope: resolvedScope,
+      importance: Math.min(input.importance, 0.5),
+      metadata: prefMetadata,
+      canonicalKey: prefCanonicalKey,
+      source: input.source,
+      language,
+      fts_text,
+    }).catch(() => {}); // Dual-write must never block the primary write
+  }
+
   return toStoredRecord(input, entry, disposition, canonicalKey, conflictId);
 }
 
@@ -768,6 +1100,8 @@ export async function persistWorkflowPattern(
   const tags = mergeTags(input.tags, ["workflow", "pattern"]);
   const text = buildWorkflowPatternText(input);
   const vector = await deps.embedder.embedPassage(text);
+  const language = detectLang(text);
+  const fts_text = tokenizeFts(text, language);
   const canonicalKey = resolveCanonicalKey({
     category: "patterns",
     title: input.title,
@@ -783,6 +1117,8 @@ export async function persistWorkflowPattern(
     metadata: buildWorkflowPatternMetadata(input, tags, canonicalKey),
     canonicalKey,
     source: input.source,
+    language,
+    fts_text,
   });
 
   return {
@@ -808,6 +1144,8 @@ export async function persistCaseMemory(
   const tags = mergeTags(input.tags, ["case", "solution"]);
   const text = buildCaseMemoryText(input);
   const vector = await deps.embedder.embedPassage(text);
+  const language = detectLang(text);
+  const fts_text = tokenizeFts(text, language);
   const canonicalKey = resolveCanonicalKey({
     category: "cases",
     title: input.title,
@@ -823,6 +1161,8 @@ export async function persistCaseMemory(
     metadata: buildCaseMemoryMetadata(input, tags, canonicalKey),
     canonicalKey,
     source: input.source,
+    language,
+    fts_text,
   });
 
   return {
@@ -869,6 +1209,8 @@ export async function promoteMemory(
     canonicalKey: input.canonicalKey,
   });
   const vector = await deps.embedder.embedPassage(text);
+  const language = detectLang(text);
+  const fts_text = tokenizeFts(text, language);
   const resolvedScope = resolveScope(input);
   const { entry, disposition, conflictId } = await writeDurableEntry(deps, {
     text,
@@ -882,6 +1224,8 @@ export async function promoteMemory(
     source: input.source,
     sourceCategory: sourceEntry.category,
     sourceBoundary: boundary,
+    language,
+    fts_text,
   });
 
   return {
@@ -919,6 +1263,11 @@ export async function persistMemoryBatch(
     }),
   );
 
+  // Note: batch-internal cosine dedup (upstream #319) already runs in the ingest
+  // pipeline (ingest.ts:batchInternalDedup) for large offline batches. For MCP
+  // persistMemoryBatch (typically <10 items), per-item conflict detection (A-2)
+  // in persistMemory handles dedup adequately without double-embedding cost.
+
   const persisted: StoredMemoryRecord[] = [];
   for (const memory of normalizedItems) {
     persisted.push(await persistMemory(deps, memory));

package/src/cascade-forget.ts

@@ -19,6 +19,7 @@
  */
 
 import type { MemoryStore, MemoryEntry } from "./store.js";
+import { isActiveMemory } from "./memory-evolution.js";
 
 // ---------------------------------------------------------------------------
 // Config
@@ -85,10 +86,10 @@ export async function cascadeForget(
     // Skip the forgotten entry itself (may still be in index)
     if (entry.id === forgottenEntry.id) continue;
 
-    // Skip already-archived entries
+    // Skip already-archived/superseded entries (unified via evolution.status)
+    if (!isActiveMemory(entry.metadata)) continue;
     let meta: Record<string, any> = {};
     try { meta = JSON.parse(entry.metadata || "{}"); } catch { /* skip */ }
-    if (meta.state === "archived" || meta.state === "superseded") continue;
 
     // Proportional demotion: higher similarity → bigger cut
     // At sim=1.0 → full maxDemotion, at threshold → near-zero