recallnest 1.4.1 → 2.2.1

package/src/memory-lint.ts

@@ -0,0 +1,429 @@
+/**
+ * Memory Lint Engine — Content Quality Checker
+ *
+ * A read-only quality checker for memory content. Separate from data-checkup.ts
+ * which checks infrastructure integrity (vector dims, tier distribution, etc.),
+ * Memory Lint checks *content* quality:
+ *
+ * 1. Contradictions — memories that say opposite things about the same topic
+ * 2. Duplicates — near-identical memories by vector cosine similarity
+ * 3. Stale — memories never or rarely accessed and old enough to review
+ * 4. Orphans — memories with missing scope or broken consolidation links
+ *
+ * Produces a health score (0-100) and a human-readable report.
+ */
+
+import type { MemoryEntry, MemoryStore } from "./store.js";
+import { cosineSimilarity } from "./multi-vector.js";
+import { parseEvolution, isActiveMemory } from "./memory-evolution.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type LintSeverity = "info" | "warning" | "error";
+
+export interface LintFinding {
+  check: string;     // e.g., "contradiction", "duplicate", "stale", "orphan"
+  severity: LintSeverity;
+  detail: string;
+  memoryIds: string[];
+}
+
+export interface MemoryLintReport {
+  findings: LintFinding[];
+  healthScore: number;  // 0-100
+  totalScanned: number;
+  timestamp: string;
+  summary: {
+    contradictions: number;
+    duplicates: number;
+    staleMemories: number;
+    orphans: number;
+  };
+}
+
+export interface LintDeps {
+  store: Pick<MemoryStore, "list">;
+  scope?: string;
+  verbose?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/** Maximum entries to compare per scope+category group (performance guard). */
+const MAX_ENTRIES_PER_GROUP = 100;
+
+/** Cosine similarity threshold for duplicate detection. */
+const DUPLICATE_THRESHOLD = 0.92;
+
+/**
+ * Minimum vector similarity to even consider two entries as potential contradictions.
+ * Real contradictions are about the SAME topic but say opposite things,
+ * so they should have moderate semantic similarity.
+ */
+const CONTRADICTION_SIMILARITY_FLOOR = 0.45;
+
+/**
+ * Categories where contradictions are meaningful.
+ * Append-only categories (events, cases) naturally contain "opposite" entries
+ * from different points in time — those are not contradictions.
+ */
+const CONTRADICTION_CATEGORIES = new Set(["profile", "preferences", "entities", "patterns"]);
+
+/** Entries not accessed in this many days are candidates for staleness. */
+const STALE_DAYS = 90;
+
+const STALE_MS = STALE_DAYS * 24 * 60 * 60 * 1000;
+
+// ---------------------------------------------------------------------------
+// Contradiction Detection (inline copy from consolidation-engine.ts)
+// ---------------------------------------------------------------------------
+
+/**
+ * Heuristic contradiction detection between two memory texts.
+ * Checks for negation patterns and requires at least one shared significant
+ * term to reduce false positives.
+ */
+function detectContradiction(textA: string, textB: string): boolean {
+  const a = textA.toLowerCase();
+  const b = textB.toLowerCase();
+
+  const negationPairs: [RegExp, RegExp][] = [
+    [/\bnot\b/, /\b(?:always|must|should|is|are|was|were)\b/],
+    [/\bnever\b/, /\b(?:always|every|each)\b/],
+    [/\bdisable/, /\benable/],
+    [/不要|不用|别/, /必须|一定|总是/],
+    [/从不/, /每次|总是|一直/],
+  ];
+
+  for (const [negRe, posRe] of negationPairs) {
+    if ((negRe.test(a) && posRe.test(b)) || (negRe.test(b) && posRe.test(a))) {
+      // Require at least one shared significant term to reduce false positives
+      const wordsA = new Set(a.split(/\s+/).filter(w => w.length > 3));
+      const wordsB = new Set(b.split(/\s+/).filter(w => w.length > 3));
+      for (const w of wordsA) {
+        if (wordsB.has(w)) return true;
+      }
+    }
+  }
+  return false;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/** Truncate text for display in findings. */
+function truncate(text: string, maxLen = 60): string {
+  if (text.length <= maxLen) return text;
+  return text.slice(0, maxLen - 3) + "...";
+}
+
+/**
+ * Group entries by scope + category key.
+ * Within each group, sort by importance descending and cap at MAX_ENTRIES_PER_GROUP.
+ */
+function groupEntries(entries: MemoryEntry[]): Map<string, MemoryEntry[]> {
+  const groups = new Map<string, MemoryEntry[]>();
+
+  for (const entry of entries) {
+    const key = `${entry.scope ?? ""}::${entry.category}`;
+    if (!groups.has(key)) groups.set(key, []);
+    groups.get(key)!.push(entry);
+  }
+
+  // Sort each group by importance descending, cap at limit
+  for (const [key, group] of groups) {
+    group.sort((a, b) => b.importance - a.importance);
+    if (group.length > MAX_ENTRIES_PER_GROUP) {
+      groups.set(key, group.slice(0, MAX_ENTRIES_PER_GROUP));
+    }
+  }
+
+  return groups;
+}
+
+// ---------------------------------------------------------------------------
+// Check 1: Contradictions
+// ---------------------------------------------------------------------------
+
+function findContradictions(entries: MemoryEntry[]): LintFinding[] {
+  const findings: LintFinding[] = [];
+
+  // Only check merge-type categories where contradictions are meaningful
+  const eligible = entries.filter(e => CONTRADICTION_CATEGORIES.has(e.category));
+  const groups = groupEntries(eligible);
+
+  for (const [, group] of groups) {
+    if (group.length < 2) continue;
+
+    for (let i = 0; i < group.length; i++) {
+      for (let j = i + 1; j < group.length; j++) {
+        // Pre-filter: entries must be about the same topic (moderate vector similarity)
+        const sim = cosineSimilarity(group[i].vector, group[j].vector);
+        if (sim < CONTRADICTION_SIMILARITY_FLOOR) continue;
+
+        if (detectContradiction(group[i].text, group[j].text)) {
+          findings.push({
+            check: "contradiction",
+            severity: "warning",
+            detail: `"${truncate(group[i].text)}" vs "${truncate(group[j].text)}"`,
+            memoryIds: [group[i].id, group[j].id],
+          });
+        }
+      }
+    }
+  }
+
+  return findings;
+}
+
+// ---------------------------------------------------------------------------
+// Check 2: Duplicates
+// ---------------------------------------------------------------------------
+
+function findDuplicates(entries: MemoryEntry[]): LintFinding[] {
+  const findings: LintFinding[] = [];
+  const groups = groupEntries(entries);
+
+  for (const [, group] of groups) {
+    if (group.length < 2) continue;
+
+    for (let i = 0; i < group.length; i++) {
+      for (let j = i + 1; j < group.length; j++) {
+        const sim = cosineSimilarity(group[i].vector, group[j].vector);
+        if (sim >= DUPLICATE_THRESHOLD) {
+          findings.push({
+            check: "duplicate",
+            severity: "warning",
+            detail: `"${truncate(group[i].text)}" -- ${(sim * 100).toFixed(1)}% similar [${truncate(group[j].text)}]`,
+            memoryIds: [group[i].id, group[j].id],
+          });
+        }
+      }
+    }
+  }
+
+  return findings;
+}
+
+// ---------------------------------------------------------------------------
+// Check 3: Stale Memories
+// ---------------------------------------------------------------------------
+
+function findStaleMemories(entries: MemoryEntry[]): LintFinding[] {
+  const findings: LintFinding[] = [];
+  const now = Date.now();
+
+  for (const entry of entries) {
+    const evo = parseEvolution(entry.metadata, entry.timestamp);
+
+    // Stale = entry itself is old enough AND (lastAccessedAt is null or > 90 days ago) AND accessCount <= 1
+    const entryAge = now - entry.timestamp;
+    if (entryAge < STALE_MS) continue; // too new to be considered stale
+
+    const lastAccess = evo.lastAccessedAt;
+    const isOldAccess = lastAccess === null || (now - lastAccess > STALE_MS);
+
+    if (isOldAccess && evo.accessCount <= 1) {
+      const ageDays = Math.floor((now - entry.timestamp) / (24 * 60 * 60 * 1000));
+      findings.push({
+        check: "stale",
+        severity: "info",
+        detail: `${ageDays}d old, ${evo.accessCount} access(es): "${truncate(entry.text, 50)}"`,
+        memoryIds: [entry.id],
+      });
+    }
+  }
+
+  return findings;
+}
+
+// ---------------------------------------------------------------------------
+// Check 4: Orphans
+// ---------------------------------------------------------------------------
+
+function findOrphans(entries: MemoryEntry[]): LintFinding[] {
+  const findings: LintFinding[] = [];
+
+  // Build set of all loaded IDs for consolidation link checking
+  const allIds = new Set(entries.map(e => e.id));
+
+  for (const entry of entries) {
+    // Missing or schema scope
+    if (!entry.scope || entry.scope.trim() === "" || entry.scope === "__schema__") {
+      findings.push({
+        check: "orphan",
+        severity: "info",
+        detail: `Missing/empty scope: "${truncate(entry.text, 50)}"`,
+        memoryIds: [entry.id],
+      });
+      continue;
+    }
+
+    // Broken consolidation link
+    const evo = parseEvolution(entry.metadata, entry.timestamp);
+    if (evo.consolidatedInto && !allIds.has(evo.consolidatedInto)) {
+      findings.push({
+        check: "orphan",
+        severity: "warning",
+        detail: `Broken consolidation link -> ${evo.consolidatedInto.slice(0, 12)}...: "${truncate(entry.text, 40)}"`,
+        memoryIds: [entry.id],
+      });
+    }
+  }
+
+  return findings;
+}
+
+// ---------------------------------------------------------------------------
+// Health Score
+// ---------------------------------------------------------------------------
+
+/**
+ * Compute a 0-100 health score from finding counts.
+ *
+ * Weights:
+ * - Contradictions: -10 each (most dangerous, conflicting guidance)
+ * - Duplicates: -5 each (waste and confusion risk)
+ * - Stale: -0.5 each (minor, many are expected)
+ * - Orphans: -3 each (moderate, broken references)
+ */
+export function computeHealthScore(
+  summary: MemoryLintReport["summary"],
+  _total: number,
+): number {
+  const penalty =
+    summary.contradictions * 10 +
+    summary.duplicates * 5 +
+    Math.floor(summary.staleMemories * 0.5) +
+    summary.orphans * 3;
+
+  return Math.max(0, Math.min(100, 100 - penalty));
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+/**
+ * Run all memory lint checks and produce a report.
+ *
+ * @param deps.store - Memory store (only `list` is needed)
+ * @param deps.scope - Optional scope filter; undefined = all scopes
+ * @param deps.verbose - Reserved for future use
+ */
+export async function runMemoryLint(deps: LintDeps): Promise<MemoryLintReport> {
+  const scopeFilter = deps.scope ? [deps.scope] : undefined;
+  const allEntries = await deps.store.list(scopeFilter, undefined, 10000, 0);
+
+  // Filter to active entries only
+  const entries = allEntries.filter(e => isActiveMemory(e.metadata));
+
+  // Run all checks
+  const contradictions = findContradictions(entries);
+  const duplicates = findDuplicates(entries);
+  const stale = findStaleMemories(entries);
+  const orphans = findOrphans(entries);
+
+  const findings = [...contradictions, ...duplicates, ...stale, ...orphans];
+
+  const summary = {
+    contradictions: contradictions.length,
+    duplicates: duplicates.length,
+    staleMemories: stale.length,
+    orphans: orphans.length,
+  };
+
+  return {
+    findings,
+    healthScore: computeHealthScore(summary, entries.length),
+    totalScanned: entries.length,
+    timestamp: new Date().toISOString(),
+    summary,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Formatter
+// ---------------------------------------------------------------------------
+
+/** Format a lint report as human-readable text. */
+export function formatMemoryLintReport(report: MemoryLintReport): string {
+  const lines: string[] = [];
+
+  // All clear case
+  if (report.findings.length === 0) {
+    lines.push("Memory Lint: All Clear!");
+    lines.push(`Scanned: ${report.totalScanned} active memories`);
+    lines.push(`Health Score: ${report.healthScore}/100`);
+    return lines.join("\n");
+  }
+
+  // Header
+  lines.push(`Memory Lint Report (${report.timestamp})`);
+  lines.push("=".repeat(44));
+  lines.push(`Scanned: ${report.totalScanned} active memories`);
+  lines.push("");
+
+  // Contradictions
+  if (report.summary.contradictions > 0) {
+    const items = report.findings.filter(f => f.check === "contradiction");
+    lines.push(`Contradictions (${items.length}):`);
+    for (const f of items) {
+      lines.push(`  - ${f.detail} [${f.memoryIds.map(id => id.slice(0, 8)).join(", ")}]`);
+    }
+    lines.push("");
+  }
+
+  // Duplicates
+  if (report.summary.duplicates > 0) {
+    const items = report.findings.filter(f => f.check === "duplicate");
+    lines.push(`Duplicates (${items.length}):`);
+    for (const f of items) {
+      lines.push(`  - ${f.detail} [${f.memoryIds.map(id => id.slice(0, 8)).join(", ")}]`);
+    }
+    lines.push("");
+  }
+
+  // Stale
+  if (report.summary.staleMemories > 0) {
+    const items = report.findings.filter(f => f.check === "stale");
+    lines.push(`Stale (${items.length}):`);
+    if (items.length <= 5) {
+      for (const f of items) {
+        lines.push(`  - ${f.detail}`);
+      }
+    } else {
+      // Summarize when many
+      lines.push(`  - ${items.length} memories not accessed in ${STALE_DAYS}+ days`);
+    }
+    lines.push("");
+  }
+
+  // Orphans
+  if (report.summary.orphans > 0) {
+    const items = report.findings.filter(f => f.check === "orphan");
+    const missingScope = items.filter(f => f.severity === "info");
+    const brokenLinks = items.filter(f => f.severity === "warning");
+
+    lines.push(`Orphans (${items.length}):`);
+    if (missingScope.length > 0) {
+      lines.push(`  - ${missingScope.length} memor${missingScope.length === 1 ? "y" : "ies"} with missing scope`);
+    }
+    if (brokenLinks.length > 0) {
+      for (const f of brokenLinks) {
+        lines.push(`  - ${f.detail} [${f.memoryIds.map(id => id.slice(0, 8)).join(", ")}]`);
+      }
+    }
+    lines.push("");
+  }
+
+  // Health score
+  lines.push(`Health Score: ${report.healthScore}/100`);
+
+  return lines.join("\n");
+}

package/src/memory-output.ts

@@ -2,6 +2,9 @@ import type { MemoryEntry } from "./store.js";
 import type { RetrievalResult } from "./retriever.js";
 import type { RetrievalProfileName } from "./retrieval-profiles.js";
 import { extractMemoryProvenance } from "./memory-boundaries.js";
+import { extractMultiVectorText } from "./multi-vector.js";
+import { parseNarrative } from "./narrative-schema.js";
+import { getConfidence, getConfidenceMetadata } from "./confidence-tracker.js";
 
 interface MemoryMetadata {
   source?: string;
@@ -167,6 +170,7 @@ function getRetrievalPath(result: RetrievalResult): string {
   if (result.sources.vector) parts.push("vector");
   if (result.sources.bm25) parts.push("bm25");
   if (result.sources.reranked) parts.push("reranked");
+  if (result.sources.narrativeSibling) parts.push("narrative");
   return parts.join("+") || "direct";
 }
 
@@ -289,6 +293,71 @@ function buildSearchRow(index: number, query: string, result: RetrievalResult):
   ];
 }
 
+function extractBriefExcerpt(result: RetrievalResult): string {
+  const { l0 } = extractMultiVectorText(result.entry.metadata);
+  const raw = l0 || result.entry.text;
+  return cleanSnippet(raw, 80);
+}
+
+export function formatBriefResults(
+  results: RetrievalResult[],
+  context: { query: string },
+): string {
+  if (results.length === 0) return "No results found.";
+  const lines = [`Query: ${context.query}`, `Hits: ${results.length}`, ""];
+  for (let i = 0; i < results.length; i++) {
+    const r = results[i];
+    const excerpt = extractBriefExcerpt(r);
+    lines.push(`#${i + 1} ${r.entry.id.slice(0, 8)} ${Math.round(r.score * 100)}% — ${excerpt}`);
+  }
+  return lines.join("\n");
+}
+
+export function formatFullResults(
+  results: RetrievalResult[],
+  context: RenderContext,
+): string {
+  if (results.length === 0) return "No results found.";
+
+  const lines = [
+    `Query   : ${context.query}`,
+    `Profile : ${context.profile}`,
+    `Hits    : ${results.length}`,
+    "",
+    "#  ID       Score Category     Tier       Source  Date       Retrieval Path       File / Snippet",
+    "-- -------- ----- ------------ ---------- ------- ---------- -------------------- --------------",
+  ];
+
+  for (let i = 0; i < results.length; i++) {
+    const row = buildSearchRow(i, context.query, results[i]);
+    lines.push(`${row[0]} ${row[1]} ${row[2]} ${row[3]} ${row[4]} ${row[5]} ${row[6]} ${row[7]} ${row[8]} | ${row[9]}`);
+    lines.push(`   prov : ${getProvenanceSummary(results[i])}`);
+    // Full mode: append metadata details
+    const meta = parseMetadata(results[i].entry);
+    const evolution = typeof meta.evolutionStatus === "string" ? meta.evolutionStatus : "-";
+    const accessCount = typeof meta.accessCount === "number" ? String(meta.accessCount) : "-";
+    const importance = results[i].entry.importance.toFixed(2);
+    const tags = Array.isArray(meta.tags) ? (meta.tags as string[]).join(", ") : "-";
+    // Emotion metadata (from emotion-detector)
+    const emotionPart = meta.emotion && typeof meta.emotion === "object"
+      ? ` emotion=${(meta.emotion as Record<string, unknown>).label ?? "-"}(v=${(meta.emotion as Record<string, unknown>).valence ?? 0},a=${(meta.emotion as Record<string, unknown>).arousal ?? 0})`
+      : "";
+    // HP-narrative: Narrative metadata (from narrative-tagger)
+    const narrative = parseNarrative(results[i].entry.metadata);
+    const narrativePart = narrative
+      ? ` narrative=${narrative.lifePeriodLabel}/${narrative.generalEventLabel}`
+      : "";
+    // F1: Confidence metadata
+    const confMeta = getConfidenceMetadata(results[i].entry);
+    const confPart = confMeta
+      ? ` confidence=${confMeta.score.toFixed(2)}(${confMeta.reliability})`
+      : "";
+    lines.push(`   meta : evolution=${evolution} accessCount=${accessCount} importance=${importance} tags=[${tags}]${confPart}${emotionPart}${narrativePart}`);
+  }
+
+  return lines.join("\n");
+}
+
 export function formatSearchResults(
   results: RetrievalResult[],
   context: RenderContext,

package/src/memory-schema.ts

@@ -24,18 +24,80 @@ export const ALL_MEMORY_CATEGORIES = [
   ...LEGACY_MEMORY_CATEGORIES,
 ] as const;
 
-export const STORE_MEMORY_SOURCES = ["manual", "agent", "api"] as const;
+export const STORE_MEMORY_SOURCES = ["manual", "agent", "api", "session_distill", "conversation_import"] as const;
 
 export const DurableMemoryCategorySchema = z.enum(DURABLE_MEMORY_CATEGORIES);
 export const MemoryCategorySchema = z.enum(ALL_MEMORY_CATEGORIES);
 export const StoreMemorySourceSchema = z.enum(STORE_MEMORY_SOURCES);
-export const WriteDispositionSchema = z.enum(["stored", "updated", "deduped", "promoted", "conflict"]);
+export const WriteDispositionSchema = z.enum(["stored", "updated", "deduped", "promoted", "conflict", "rejected"]);
 
 export type DurableMemoryCategory = z.infer<typeof DurableMemoryCategorySchema>;
 export type MemoryCategoryValue = z.infer<typeof MemoryCategorySchema>;
 export type StoreMemorySource = z.infer<typeof StoreMemorySourceSchema>;
 export type WriteDisposition = z.infer<typeof WriteDispositionSchema>;
 
+// --- Emotional Valence (Philosophy of Memory: Affective Memory) ---
+export interface EmotionMetadata {
+  /** Negative (-1) to Positive (+1) */
+  valence: number;
+  /** Calm (0) to Excited (1) */
+  arousal: number;
+  /** Human-readable label */
+  label?: string;
+  /** Composite mnemonic significance: (|valence| + arousal) / 2, range 0-1 */
+  salience?: number;
+  /** Detection method */
+  source?: "keyword" | "llm" | "user";
+}
+
+export const EmotionMetadataSchema = z.object({
+  valence: z.number().min(-1).max(1),
+  arousal: z.number().min(0).max(1),
+  label: z.string().max(30).optional(),
+  salience: z.number().min(0).max(1).optional(),
+  source: z.enum(["keyword", "llm", "user"]).optional(),
+});
+
+/** Parse emotion from metadata JSON string, returns null if absent */
+export function parseEmotion(metadata: string | undefined): EmotionMetadata | null {
+  if (!metadata) return null;
+  try {
+    const parsed = JSON.parse(metadata);
+    if (parsed.emotion && typeof parsed.emotion.valence === "number") {
+      return EmotionMetadataSchema.parse(parsed.emotion);
+    }
+  } catch { /* malformed metadata - safe to ignore */ }
+  return null;
+}
+
+/** Default neutral emotion for memories without emotion data */
+export const NEUTRAL_EMOTION: EmotionMetadata = { valence: 0, arousal: 0, label: "neutral" };
+
+// --- Privacy Tier (Philosophy of Memory: Ethics Layer) ---
+/**
+ * Privacy tiers control how aggressively a memory is persisted, extracted, and shared.
+ *
+ * - ephemeral: auto-expire, no KG extraction, no sharing — session scratch
+ * - private:   persist but no KG extraction, no sharing — personal notes
+ * - durable:   full lifecycle, KG extraction, standard decay — default
+ * - shared:    marked safe for cross-scope retrieval — collaborative knowledge
+ */
+export const PRIVACY_TIERS = ["ephemeral", "private", "durable", "shared"] as const;
+export type PrivacyTier = (typeof PRIVACY_TIERS)[number];
+export const PrivacyTierSchema = z.enum(PRIVACY_TIERS);
+
+/** Parse privacyTier from metadata JSON string, returns "durable" (default) if absent */
+export function parsePrivacyTier(metadata: string | undefined): PrivacyTier {
+  if (!metadata) return "durable";
+  try {
+    const parsed = JSON.parse(metadata);
+    if (parsed.privacyTier && PRIVACY_TIERS.includes(parsed.privacyTier)) {
+      return parsed.privacyTier;
+    }
+  } catch { /* malformed metadata - safe default */ }
+  return "durable";
+}
+
 export const MemoryTextSchema = boundedStringSchema("text", 4000);
 export const MemoryScopeSchema = optionalBoundedStringSchema(160);
 export const RequiredMemoryScopeSchema = boundedStringSchema("scope", 160);
@@ -64,6 +126,8 @@ export const StoreMemoryInputSchema = z.object({
   source: StoreMemorySourceSchema.default("manual"),
   tags: MemoryTagsSchema,
   canonicalKey: CanonicalKeySchema,
+  topicTag: z.string().min(1).max(60).optional(),
+  privacyTier: PrivacyTierSchema.default("durable"),
 });
 
 export const StoredMemoryRecordSchema = StoreMemoryInputSchema.extend({
@@ -182,3 +246,11 @@ export type CaseMemoryInput = z.infer<typeof CaseMemoryInputSchema>;
 export type StoredCaseMemoryRecord = z.infer<typeof StoredCaseMemoryRecordSchema>;
 export type PromoteMemoryInput = z.infer<typeof PromoteMemoryInputSchema>;
 export type StoredPromotedMemoryRecord = z.infer<typeof StoredPromotedMemoryRecordSchema>;
+
+export function isEmotionScoringEnabled(): boolean {
+  return process.env.RECALLNEST_EMOTION_SCORING === "true";
+}
+
+export function isPredictiveMemoryEnabled(): boolean {
+  return process.env.RECALLNEST_PREDICTIVE_MEMORY === "true";
+}