recallnest 1.4.1

package/src/memory-boundaries.ts

@@ -0,0 +1,354 @@
+import type { DurableMemoryCategory } from "./memory-schema.js";
+import { inferPreferenceSlot } from "./preference-slots.js";
+
+export const MEMORY_LAYERS = [
+  "canonical",
+  "durable",
+  "session",
+  "evidence",
+] as const;
+
+export const MEMORY_AUTHORITIES = [
+  "manual-document",
+  "structured-memory",
+  "document-ingest",
+  "transcript-ingest",
+  "session-checkpoint",
+  "distillation",
+] as const;
+
+export const MEMORY_CONFLICT_POLICIES = [
+  "latest-wins",
+  "append-only",
+  "manual-review",
+] as const;
+
+export type MemoryLayer = (typeof MEMORY_LAYERS)[number];
+export type MemoryAuthority = (typeof MEMORY_AUTHORITIES)[number];
+export type MemoryConflictPolicy = (typeof MEMORY_CONFLICT_POLICIES)[number];
+
+export interface MemoryBoundaryMetadata {
+  layer: MemoryLayer;
+  authority: MemoryAuthority;
+  conflictPolicy: MemoryConflictPolicy;
+  originalCategory?: DurableMemoryCategory;
+  downgradedFrom?: DurableMemoryCategory;
+  note?: string;
+}
+
+export interface PromotedFromMetadata {
+  memoryId: string;
+  scope?: string;
+  category?: string;
+  source?: string;
+  boundary?: MemoryBoundaryMetadata | null;
+}
+
+export interface ProvenanceHistoryMetadata extends PromotedFromMetadata {
+  observedAt?: string;
+}
+
+export interface MemoryProvenance {
+  boundary: MemoryBoundaryMetadata | null;
+  canonicalKey: string | null;
+  promotedFrom: PromotedFromMetadata | null;
+  provenanceHistory: ProvenanceHistoryMetadata[];
+  provenanceHistoryCount: number;
+}
+
+export interface IngestBoundaryResolution {
+  category: DurableMemoryCategory;
+  boundary: MemoryBoundaryMetadata;
+}
+
+const TRANSCRIPT_SOURCES = new Set(["cc", "codex", "gemini"]);
+const TRANSCRIPT_SCOPE_PREFIXES = ["cc:", "codex:", "gemini:"];
+
+export function getConflictPolicyForCategory(category: DurableMemoryCategory): MemoryConflictPolicy {
+  return category === "events" || category === "cases"
+    ? "append-only"
+    : "latest-wins";
+}
+
+export function isTranscriptIngestSource(source: string): boolean {
+  return TRANSCRIPT_SOURCES.has(source);
+}
+
+export function isTranscriptScope(scope: string): boolean {
+  return TRANSCRIPT_SCOPE_PREFIXES.some((prefix) => scope.startsWith(prefix));
+}
+
+export function isDurableMemoryScope(scope: string): boolean {
+  return scope.startsWith("memory:") || scope.startsWith("asset:");
+}
+
+export function buildStructuredMemoryBoundary(
+  category: DurableMemoryCategory,
+): MemoryBoundaryMetadata {
+  return {
+    layer: "durable",
+    authority: "structured-memory",
+    conflictPolicy: getConflictPolicyForCategory(category),
+    originalCategory: category,
+    note: "Structured memory writes are the durable source inside RecallNest.",
+  };
+}
+
+export function buildDistillationBoundary(
+  category: DurableMemoryCategory,
+): MemoryBoundaryMetadata {
+  return {
+    layer: "durable",
+    authority: "distillation",
+    conflictPolicy: getConflictPolicyForCategory(category),
+    originalCategory: category,
+  };
+}
+
+export function normalizeCanonicalKey(value: string): string {
+  return value
+    .trim()
+    .toLowerCase()
+    .replace(/['"`]/g, "")
+    .replace(/[^a-z0-9\p{Script=Han}]+/gu, "-")
+    .replace(/^-+|-+$/g, "")
+    .replace(/-{2,}/g, "-")
+    .slice(0, 120);
+}
+
+function collapseKeyText(value: string): string {
+  return value
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+export function buildDefaultCanonicalKey(params: {
+  category: DurableMemoryCategory;
+  text?: string;
+  title?: string;
+}): string {
+  if (params.category === "preferences") {
+    const slot = inferPreferenceSlot(params.text || params.title || "");
+    if (slot) {
+      return truncateCanonicalKey(
+        slot.type === "brand-item"
+          ? [params.category, slot.type, slot.brand, slot.item]
+          : slot.type === "reply-style"
+            ? [params.category, slot.type, ...slot.traits]
+            : [params.category, slot.type, slot.preferredTool, "over", slot.avoidedTool],
+      );
+    }
+  }
+
+  const base = collapseKeyText(params.title || params.text || "");
+  const normalizedBase = normalizeCanonicalKey(base);
+  return normalizedBase
+    ? `${params.category}:${normalizedBase}`
+    : `${params.category}:memory`;
+}
+
+function truncateCanonicalKey(segments: string[]): string {
+  return segments
+    .filter(Boolean)
+    .join(":")
+    .slice(0, 120)
+    .replace(/[:\-]+$/u, "");
+}
+
+export function resolveIngestBoundary(params: {
+  source: string;
+  scope: string;
+  category: DurableMemoryCategory;
+}): IngestBoundaryResolution {
+  const { source, scope, category } = params;
+
+  if (isTranscriptIngestSource(source)) {
+    if (category === "profile" || category === "preferences") {
+      return {
+        category: "events",
+        boundary: {
+          layer: "evidence",
+          authority: "transcript-ingest",
+          conflictPolicy: "append-only",
+          originalCategory: category,
+          downgradedFrom: category,
+          note: "Transcript-derived stable facts stay as evidence until explicitly promoted.",
+        },
+      };
+    }
+
+    return {
+      category,
+      boundary: {
+        layer: "evidence",
+        authority: "transcript-ingest",
+        conflictPolicy: getConflictPolicyForCategory(category),
+        originalCategory: category,
+        note: "Transcript-derived memories are evidence and should not override curated memory.",
+      },
+    };
+  }
+
+  if (isDurableMemoryScope(scope) || scope === "memory") {
+    return {
+      category,
+      boundary: {
+        layer: "durable",
+        authority: "document-ingest",
+        conflictPolicy: getConflictPolicyForCategory(category),
+        originalCategory: category,
+        note: "Curated memory documents can mirror durable memory, but they are not session authority.",
+      },
+    };
+  }
+
+  return {
+    category,
+    boundary: {
+      layer: "evidence",
+      authority: "document-ingest",
+      conflictPolicy: getConflictPolicyForCategory(category),
+      originalCategory: category,
+      note: "Unstructured ingest stays evidence until explicitly promoted.",
+    },
+  };
+}
+
+export function parseMetadataObject(metadata?: string): Record<string, unknown> | null {
+  if (!metadata) return null;
+  try {
+    const parsed = JSON.parse(metadata);
+    return parsed && typeof parsed === "object" ? parsed as Record<string, unknown> : null;
+  } catch {
+    return null;
+  }
+}
+
+function coerceBoundaryMetadata(candidate: unknown): MemoryBoundaryMetadata | null {
+  if (!candidate || typeof candidate !== "object") return null;
+
+  const record = candidate as Record<string, unknown>;
+  const layer = typeof record.layer === "string" ? record.layer : "";
+  const authority = typeof record.authority === "string" ? record.authority : "";
+  const conflictPolicy = typeof record.conflictPolicy === "string" ? record.conflictPolicy : "";
+
+  if (
+    !MEMORY_LAYERS.includes(layer as MemoryLayer) ||
+    !MEMORY_AUTHORITIES.includes(authority as MemoryAuthority) ||
+    !MEMORY_CONFLICT_POLICIES.includes(conflictPolicy as MemoryConflictPolicy)
+  ) {
+    return null;
+  }
+
+  return {
+    layer: layer as MemoryLayer,
+    authority: authority as MemoryAuthority,
+    conflictPolicy: conflictPolicy as MemoryConflictPolicy,
+    ...(typeof record.originalCategory === "string"
+      ? { originalCategory: record.originalCategory as DurableMemoryCategory }
+      : {}),
+    ...(typeof record.downgradedFrom === "string"
+      ? { downgradedFrom: record.downgradedFrom as DurableMemoryCategory }
+      : {}),
+    ...(typeof record.note === "string" ? { note: record.note } : {}),
+  };
+}
+
+export function extractBoundaryMetadata(metadata?: string): MemoryBoundaryMetadata | null {
+  const parsed = parseMetadataObject(metadata);
+  return coerceBoundaryMetadata(parsed?.boundary);
+}
+
+function coercePromotedFromMetadata(candidate: unknown): PromotedFromMetadata | null {
+  if (!candidate || typeof candidate !== "object") return null;
+
+  const record = candidate as Record<string, unknown>;
+  const memoryId = typeof record.memoryId === "string" ? record.memoryId.trim() : "";
+  if (!memoryId) return null;
+
+  return {
+    memoryId,
+    ...(typeof record.scope === "string" ? { scope: record.scope } : {}),
+    ...(typeof record.category === "string" ? { category: record.category } : {}),
+    ...(typeof record.source === "string" ? { source: record.source } : {}),
+    boundary: coerceBoundaryMetadata(record.boundary),
+  };
+}
+
+function coerceProvenanceHistoryMetadata(candidate: unknown): ProvenanceHistoryMetadata | null {
+  const promotedFrom = coercePromotedFromMetadata(candidate);
+  if (!promotedFrom || typeof candidate !== "object") return null;
+
+  const record = candidate as Record<string, unknown>;
+  return {
+    ...promotedFrom,
+    ...(typeof record.observedAt === "string" ? { observedAt: record.observedAt } : {}),
+  };
+}
+
+export function extractPromotedFrom(metadata?: string): PromotedFromMetadata | null {
+  const parsed = parseMetadataObject(metadata);
+  return coercePromotedFromMetadata(parsed?.promotedFrom);
+}
+
+export function extractProvenanceHistory(metadata?: string): ProvenanceHistoryMetadata[] {
+  const parsed = parseMetadataObject(metadata);
+  const rawHistory = Array.isArray(parsed?.provenanceHistory) ? parsed.provenanceHistory : [];
+  const history = rawHistory
+    .map((candidate) => coerceProvenanceHistoryMetadata(candidate))
+    .filter((candidate): candidate is ProvenanceHistoryMetadata => Boolean(candidate));
+
+  if (history.length > 0) {
+    return history;
+  }
+
+  const promotedFrom = coerceProvenanceHistoryMetadata(parsed?.promotedFrom);
+  return promotedFrom ? [promotedFrom] : [];
+}
+
+export function extractProvenanceHistoryCount(metadata?: string): number {
+  const parsed = parseMetadataObject(metadata);
+  const candidate = parsed?.provenanceHistoryCount;
+  if (typeof candidate === "number" && Number.isFinite(candidate) && candidate >= 0) {
+    return Math.floor(candidate);
+  }
+  return extractProvenanceHistory(metadata).length;
+}
+
+export function extractMemoryProvenance(params: {
+  scope: string;
+  metadata?: string;
+}): MemoryProvenance {
+  return {
+    boundary: extractBoundaryMetadata(params.metadata),
+    canonicalKey: extractCanonicalKey(params.metadata),
+    promotedFrom: extractPromotedFrom(params.metadata),
+    provenanceHistory: extractProvenanceHistory(params.metadata),
+    provenanceHistoryCount: extractProvenanceHistoryCount(params.metadata),
+  };
+}
+
+export function extractCanonicalKey(metadata?: string): string | null {
+  const parsed = parseMetadataObject(metadata);
+  const canonicalKey = parsed?.canonicalKey;
+  return typeof canonicalKey === "string" && canonicalKey.trim().length > 0
+    ? canonicalKey
+    : null;
+}
+
+export function shouldUseStableMemoryResult(params: {
+  category: "profile" | "preferences" | "entities";
+  scope: string;
+  metadata?: string;
+}): boolean {
+  if (isTranscriptScope(params.scope)) {
+    return false;
+  }
+
+  const boundary = extractBoundaryMetadata(params.metadata);
+  if (!boundary) {
+    return true;
+  }
+
+  return boundary.layer !== "evidence";
+}

package/src/memory-health-rebalance.ts

@@ -0,0 +1,220 @@
+import type { MemoryTier } from "./decay-engine.js";
+
+export type StoredMemoryTier = MemoryTier | "unknown";
+
+export interface MemoryHealthSnapshot {
+  id: string;
+  importance: number;
+  timestamp: number;
+  metadata?: string;
+}
+
+export interface MemoryHealthRebalancePlan {
+  id: string;
+  accessCount: number;
+  currentImportance: number;
+  nextImportance: number;
+  currentTier: StoredMemoryTier;
+  targetTier: MemoryTier;
+  currentMetadata: Record<string, unknown>;
+  nextMetadata: Record<string, unknown>;
+  tierChanged: boolean;
+  tierBackfilled: boolean;
+  deadMemoryRow: boolean;
+  deadMemoryDemoted: boolean;
+  importanceChanged: boolean;
+  changed: boolean;
+}
+
+export interface MemoryHealthRebalanceSummary {
+  totalRows: number;
+  changedRows: number;
+  deadMemoryRows: number;
+  deadMemoryDemotions: number;
+  tierBackfills: number;
+  tierChanges: number;
+  importanceChanges: number;
+}
+
+export interface MemoryHealthDatasetStats {
+  maxAccessCount: number;
+  minTimestamp: number;
+  maxTimestamp: number;
+}
+
+interface ImportanceBand {
+  min: number;
+  max: number;
+}
+
+const IMPORTANCE_BANDS: Record<MemoryTier, ImportanceBand> = {
+  core: { min: 0.85, max: 0.95 },
+  working: { min: 0.6, max: 0.8 },
+  peripheral: { min: 0.3, max: 0.5 },
+};
+
+function clamp(value: number, min: number, max: number): number {
+  if (!Number.isFinite(value)) return min;
+  return Math.min(max, Math.max(min, value));
+}
+
+function roundImportance(value: number): number {
+  return Math.round(value * 1000) / 1000;
+}
+
+export function parseMemoryHealthMetadata(metadata?: string): Record<string, unknown> {
+  try {
+    const parsed = JSON.parse(metadata || "{}");
+    return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+      ? parsed as Record<string, unknown>
+      : {};
+  } catch {
+    return {};
+  }
+}
+
+export function getMemoryHealthAccessCount(metadata: Record<string, unknown>): number {
+  const camel = metadata.accessCount;
+  if (typeof camel === "number" && Number.isFinite(camel) && camel >= 0) {
+    return Math.floor(camel);
+  }
+
+  const snake = metadata.access_count;
+  if (typeof snake === "number" && Number.isFinite(snake) && snake >= 0) {
+    return Math.floor(snake);
+  }
+
+  return 0;
+}
+
+export function getStoredMemoryTier(metadata: Record<string, unknown>): StoredMemoryTier {
+  const tier = metadata.tier;
+  return tier === "core" || tier === "working" || tier === "peripheral"
+    ? tier
+    : "unknown";
+}
+
+export function resolveRebalancedTier(currentTier: StoredMemoryTier, accessCount: number): MemoryTier {
+  if (accessCount <= 0) return "peripheral";
+  if (currentTier !== "unknown") return currentTier;
+  if (accessCount > 5) return "core";
+  return "working";
+}
+
+function computeTierAccessSignal(
+  tier: MemoryTier,
+  accessCount: number,
+  maxAccessCount: number,
+): number {
+  if (accessCount <= 0) return 0;
+
+  switch (tier) {
+    case "peripheral": {
+      const ceiling = Math.max(1, Math.min(5, maxAccessCount));
+      return clamp(accessCount / ceiling, 0, 1);
+    }
+    case "working": {
+      if (accessCount <= 1) return 0;
+      return clamp((Math.min(accessCount, 5) - 1) / 4, 0, 1);
+    }
+    case "core": {
+      const ceiling = Math.max(6, maxAccessCount);
+      if (ceiling <= 6) return 1;
+      return clamp((Math.min(accessCount, ceiling) - 6) / (ceiling - 6), 0, 1);
+    }
+  }
+}
+
+export function computeRebalancedImportance(
+  currentImportance: number,
+  tier: MemoryTier,
+  accessCount: number,
+  maxAccessCount: number,
+  recencySignal: number,
+): number {
+  const band = IMPORTANCE_BANDS[tier];
+  const span = band.max - band.min;
+  const currentSignal = clamp(currentImportance, 0, 1);
+  const accessSignal = computeTierAccessSignal(tier, accessCount, maxAccessCount);
+  const blendedSignal = tier === "peripheral" && accessCount === 0
+    ? clamp(currentSignal * 0.15 + recencySignal * 0.85, 0, 1)
+    : clamp(currentSignal * 0.25 + accessSignal * 0.65 + recencySignal * 0.1, 0, 1);
+  return roundImportance(clamp(band.min + span * blendedSignal, band.min, band.max));
+}
+
+export function buildMemoryHealthRebalancePlan(
+  snapshot: MemoryHealthSnapshot,
+  stats: MemoryHealthDatasetStats,
+): MemoryHealthRebalancePlan {
+  const currentMetadata = parseMemoryHealthMetadata(snapshot.metadata);
+  const accessCount = getMemoryHealthAccessCount(currentMetadata);
+  const currentTier = getStoredMemoryTier(currentMetadata);
+  const targetTier = resolveRebalancedTier(currentTier, accessCount);
+  const currentImportance = clamp(snapshot.importance, 0, 1);
+  const recencySignal = stats.maxTimestamp <= stats.minTimestamp
+    ? 0.5
+    : clamp(
+        (snapshot.timestamp - stats.minTimestamp) / (stats.maxTimestamp - stats.minTimestamp),
+        0,
+        1,
+      );
+  const nextImportance = computeRebalancedImportance(
+    currentImportance,
+    targetTier,
+    accessCount,
+    stats.maxAccessCount,
+    recencySignal,
+  );
+
+  const nextMetadata: Record<string, unknown> = {
+    ...currentMetadata,
+    tier: targetTier,
+    importance: nextImportance,
+  };
+
+  const tierChanged = currentTier !== targetTier;
+  const tierBackfilled = currentTier === "unknown";
+  const deadMemoryRow = accessCount === 0;
+  const deadMemoryDemoted = deadMemoryRow && currentTier !== "peripheral";
+  const importanceChanged = roundImportance(currentImportance) !== nextImportance;
+
+  return {
+    id: snapshot.id,
+    accessCount,
+    currentImportance,
+    nextImportance,
+    currentTier,
+    targetTier,
+    currentMetadata,
+    nextMetadata,
+    tierChanged,
+    tierBackfilled,
+    deadMemoryRow,
+    deadMemoryDemoted,
+    importanceChanged,
+    changed: tierChanged || importanceChanged || !("importance" in currentMetadata),
+  };
+}
+
+export function summarizeMemoryHealthPlans(
+  plans: MemoryHealthRebalancePlan[],
+): MemoryHealthRebalanceSummary {
+  return plans.reduce<MemoryHealthRebalanceSummary>((summary, plan) => {
+    summary.totalRows += 1;
+    if (plan.changed) summary.changedRows += 1;
+    if (plan.deadMemoryRow) summary.deadMemoryRows += 1;
+    if (plan.deadMemoryDemoted) summary.deadMemoryDemotions += 1;
+    if (plan.tierBackfilled) summary.tierBackfills += 1;
+    if (plan.tierChanged) summary.tierChanges += 1;
+    if (plan.importanceChanged) summary.importanceChanges += 1;
+    return summary;
+  }, {
+    totalRows: 0,
+    changedRows: 0,
+    deadMemoryRows: 0,
+    deadMemoryDemotions: 0,
+    tierBackfills: 0,
+    tierChanges: 0,
+    importanceChanges: 0,
+  });
+}