recallnest 1.4.1 → 2.2.0

package/src/conversation-importer.ts

@@ -0,0 +1,441 @@
+import { readFileSync } from "node:fs";
+import type { PersistMemoryDeps } from "./capture-engine.js";
+import { persistMemory } from "./capture-engine.js";
+import type { StoredMemoryRecord } from "./memory-schema.js";
+
+export interface NormalizedMessage {
+  role: "user" | "assistant" | "system";
+  content: string;
+  timestamp?: string;
+}
+
+export type ConversationFormat =
+  | "claude-code"
+  | "claude-ai"
+  | "chatgpt"
+  | "slack"
+  | "plaintext";
+
+// ---------------------------------------------------------------------------
+// Format detection
+// ---------------------------------------------------------------------------
+
+export function detectFormat(content: string): ConversationFormat {
+  const trimmed = content.trimStart();
+
+  if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
+    try {
+      const parsed = JSON.parse(trimmed);
+
+      if (Array.isArray(parsed)) {
+        if (parsed.length > 0 && parsed[0].chat_messages !== undefined) {
+          return "claude-ai";
+        }
+        if (parsed.length > 0 && parsed[0].mapping !== undefined) {
+          return "chatgpt";
+        }
+        if (parsed.length > 0 && parsed[0].ts !== undefined && parsed[0].text !== undefined) {
+          return "slack";
+        }
+      }
+
+      if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+        if (parsed.chat_conversations !== undefined) {
+          return "claude-ai";
+        }
+        if (parsed.mapping !== undefined) {
+          return "chatgpt";
+        }
+        if (parsed.messages !== undefined && Array.isArray(parsed.messages)) {
+          const first = parsed.messages[0];
+          if (first && first.ts !== undefined) {
+            return "slack";
+          }
+        }
+      }
+    } catch {
+      // Not valid JSON — fall through
+    }
+  }
+
+  // JSONL: each line is a JSON object with type field
+  const firstLine = trimmed.split("\n")[0]?.trim();
+  if (firstLine?.startsWith("{")) {
+    try {
+      const obj = JSON.parse(firstLine);
+      if (obj.type === "human" || obj.type === "assistant" || obj.message !== undefined) {
+        return "claude-code";
+      }
+    } catch {
+      // Not JSONL
+    }
+  }
+
+  return "plaintext";
+}
+
+// ---------------------------------------------------------------------------
+// Claude Code JSONL normalizer
+// ---------------------------------------------------------------------------
+
+interface ClaudeCodeLine {
+  type?: string;
+  message?: {
+    content?: string | Array<{ type?: string; text?: string }>;
+    role?: string;
+  };
+  timestamp?: string;
+}
+
+export function normalizeClaudeCode(content: string): NormalizedMessage[] {
+  const messages: NormalizedMessage[] = [];
+
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    let parsed: ClaudeCodeLine;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch {
+      continue;
+    }
+
+    let role: NormalizedMessage["role"];
+    if (parsed.type === "human") {
+      role = "user";
+    } else if (parsed.type === "assistant") {
+      role = "assistant";
+    } else if (parsed.message?.role === "user" || parsed.type === "user") {
+      role = "user";
+    } else if (parsed.message?.role === "assistant") {
+      role = "assistant";
+    } else {
+      continue;
+    }
+
+    let text = "";
+    const rawContent = parsed.message?.content;
+    if (typeof rawContent === "string") {
+      text = rawContent;
+    } else if (Array.isArray(rawContent)) {
+      text = rawContent
+        .filter((part): part is { type?: string; text?: string } =>
+          typeof part === "object" && part !== null && typeof part.text === "string")
+        .map((part) => part.text!)
+        .join("\n");
+    }
+
+    if (!text.trim()) continue;
+
+    messages.push({
+      role,
+      content: text.trim(),
+      ...(parsed.timestamp ? { timestamp: parsed.timestamp } : {}),
+    });
+  }
+
+  return messages;
+}
+
+// ---------------------------------------------------------------------------
+// Claude.ai JSON export normalizer
+// ---------------------------------------------------------------------------
+
+interface ClaudeAiConversation {
+  chat_messages?: Array<{
+    sender?: string;
+    text?: string;
+    created_at?: string;
+  }>;
+}
+
+export function normalizeClaudeAi(content: string): NormalizedMessage[] {
+  const parsed = JSON.parse(content);
+  const messages: NormalizedMessage[] = [];
+
+  let conversations: ClaudeAiConversation[];
+  if (Array.isArray(parsed)) {
+    conversations = parsed;
+  } else if (parsed.chat_conversations) {
+    conversations = parsed.chat_conversations;
+  } else {
+    conversations = [parsed];
+  }
+
+  for (const conv of conversations) {
+    if (!conv.chat_messages) continue;
+    for (const msg of conv.chat_messages) {
+      let role: NormalizedMessage["role"];
+      if (msg.sender === "human") {
+        role = "user";
+      } else if (msg.sender === "assistant") {
+        role = "assistant";
+      } else {
+        continue;
+      }
+
+      const text = msg.text?.trim();
+      if (!text) continue;
+
+      messages.push({
+        role,
+        content: text,
+        ...(msg.created_at ? { timestamp: msg.created_at } : {}),
+      });
+    }
+  }
+
+  return messages;
+}
+
+// ---------------------------------------------------------------------------
+// ChatGPT conversations.json normalizer
+// ---------------------------------------------------------------------------
+
+interface ChatGPTMapping {
+  [id: string]: {
+    id: string;
+    parent?: string | null;
+    children?: string[];
+    message?: {
+      author?: { role?: string };
+      content?: { parts?: Array<string | Record<string, unknown>> };
+      create_time?: number;
+    } | null;
+  };
+}
+
+interface ChatGPTConversation {
+  mapping?: ChatGPTMapping;
+}
+
+export function normalizeChatGPT(content: string): NormalizedMessage[] {
+  const parsed = JSON.parse(content);
+  const messages: NormalizedMessage[] = [];
+
+  const conversations: ChatGPTConversation[] = Array.isArray(parsed) ? parsed : [parsed];
+
+  for (const conv of conversations) {
+    if (!conv.mapping) continue;
+    const mapping = conv.mapping;
+
+    // Find root: node whose parent is null or not in mapping
+    let rootId: string | null = null;
+    for (const [id, node] of Object.entries(mapping)) {
+      if (!node.parent || !mapping[node.parent]) {
+        rootId = id;
+        break;
+      }
+    }
+
+    if (!rootId) continue;
+
+    // BFS to rebuild order
+    const queue: string[] = [rootId];
+    const visited = new Set<string>();
+
+    while (queue.length > 0) {
+      const nodeId = queue.shift()!;
+      if (visited.has(nodeId)) continue;
+      visited.add(nodeId);
+
+      const node = mapping[nodeId];
+      if (!node) continue;
+
+      if (node.message?.author?.role && node.message.content?.parts) {
+        const authorRole = node.message.author.role;
+        let role: NormalizedMessage["role"] | null = null;
+        if (authorRole === "user") role = "user";
+        else if (authorRole === "assistant") role = "assistant";
+        else if (authorRole === "system") role = "system";
+
+        if (role) {
+          const text = node.message.content.parts
+            .filter((part): part is string => typeof part === "string")
+            .join("\n")
+            .trim();
+
+          if (text) {
+            const msg: NormalizedMessage = { role, content: text };
+            if (node.message.create_time) {
+              msg.timestamp = new Date(node.message.create_time * 1000).toISOString();
+            }
+            messages.push(msg);
+          }
+        }
+      }
+
+      if (node.children) {
+        for (const childId of node.children) {
+          queue.push(childId);
+        }
+      }
+    }
+  }
+
+  return messages;
+}
+
+// ---------------------------------------------------------------------------
+// Slack JSON export normalizer
+// ---------------------------------------------------------------------------
+
+interface SlackMessage {
+  user?: string;
+  bot_id?: string;
+  text?: string;
+  ts?: string;
+}
+
+export function normalizeSlack(content: string): NormalizedMessage[] {
+  const parsed = JSON.parse(content);
+  const messages: NormalizedMessage[] = [];
+
+  let rawMessages: SlackMessage[];
+  if (Array.isArray(parsed)) {
+    rawMessages = parsed;
+  } else if (parsed.messages && Array.isArray(parsed.messages)) {
+    rawMessages = parsed.messages;
+  } else {
+    return messages;
+  }
+
+  for (const msg of rawMessages) {
+    const text = msg.text?.trim();
+    if (!text) continue;
+
+    const role: NormalizedMessage["role"] = msg.bot_id ? "assistant" : "user";
+
+    const result: NormalizedMessage = { role, content: text };
+    if (msg.ts) {
+      result.timestamp = new Date(parseFloat(msg.ts) * 1000).toISOString();
+    }
+    messages.push(result);
+  }
+
+  return messages;
+}
+
+// ---------------------------------------------------------------------------
+// Plaintext normalizer
+// ---------------------------------------------------------------------------
+
+const ROLE_PREFIXES: Array<[RegExp, NormalizedMessage["role"]]> = [
+  [/^User:\s*/i, "user"],
+  [/^Human:\s*/i, "user"],
+  [/^Assistant:\s*/i, "assistant"],
+  [/^AI:\s*/i, "assistant"],
+  [/^System:\s*/i, "system"],
+];
+
+export function normalizePlaintext(content: string): NormalizedMessage[] {
+  const messages: NormalizedMessage[] = [];
+  let currentRole: NormalizedMessage["role"] | null = null;
+  let currentLines: string[] = [];
+
+  function flush() {
+    if (currentRole && currentLines.length > 0) {
+      const text = currentLines.join("\n").trim();
+      if (text) {
+        messages.push({ role: currentRole, content: text });
+      }
+    }
+    currentLines = [];
+  }
+
+  for (const line of content.split("\n")) {
+    let matched = false;
+    for (const [pattern, role] of ROLE_PREFIXES) {
+      const match = line.match(pattern);
+      if (match) {
+        flush();
+        currentRole = role;
+        const remainder = line.slice(match[0].length).trim();
+        if (remainder) currentLines.push(remainder);
+        matched = true;
+        break;
+      }
+    }
+    if (!matched && currentRole) {
+      currentLines.push(line);
+    }
+  }
+  flush();
+
+  return messages;
+}
+
+// ---------------------------------------------------------------------------
+// Unified normalize dispatcher
+// ---------------------------------------------------------------------------
+
+export function normalizeConversation(content: string, format: ConversationFormat): NormalizedMessage[] {
+  switch (format) {
+    case "claude-code":
+      return normalizeClaudeCode(content);
+    case "claude-ai":
+      return normalizeClaudeAi(content);
+    case "chatgpt":
+      return normalizeChatGPT(content);
+    case "slack":
+      return normalizeSlack(content);
+    case "plaintext":
+      return normalizePlaintext(content);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Batch ingest: normalized messages -> persistMemory
+// ---------------------------------------------------------------------------
+
+export interface ImportResult {
+  total: number;
+  stored: number;
+  rejected: number;
+  errors: string[];
+}
+
+export async function importConversation(
+  deps: PersistMemoryDeps,
+  filePath: string,
+  scope: string,
+  format: ConversationFormat | "auto",
+): Promise<ImportResult> {
+  const content = readFileSync(filePath, "utf-8");
+  const resolvedFormat = format === "auto" ? detectFormat(content) : format;
+  const messages = normalizeConversation(content, resolvedFormat);
+
+  return ingestNormalizedMessages(deps, messages, scope);
+}
+
+export async function ingestNormalizedMessages(
+  deps: PersistMemoryDeps,
+  messages: NormalizedMessage[],
+  scope: string,
+): Promise<ImportResult> {
+  const result: ImportResult = { total: messages.length, stored: 0, rejected: 0, errors: [] };
+
+  for (const msg of messages) {
+    try {
+      const stored: StoredMemoryRecord = await persistMemory(deps, {
+        text: msg.content,
+        category: "events" as const,
+        importance: 0.7,
+        scope,
+        source: "conversation_import" as const,
+        tags: [`role:${msg.role}`, ...(msg.timestamp ? [`ts:${msg.timestamp}`] : [])],
+      });
+
+      if (stored.disposition === "rejected") {
+        result.rejected++;
+      } else {
+        result.stored++;
+      }
+    } catch (err) {
+      result.errors.push(err instanceof Error ? err.message : String(err));
+    }
+  }
+
+  return result;
+}

package/src/data-checkup.ts

@@ -0,0 +1,232 @@
+/**
+ * LC-P4: Data Checkup Engine
+ *
+ * Health checks for the memory database, inspired by lossless-claw's
+ * IntegrityChecker. Pure read-only diagnostics — never modifies data.
+ *
+ * Checks:
+ * 1. Vector dimension consistency
+ * 2. Orphan memories (scope missing or empty)
+ * 3. Tier distribution health
+ * 4. Conflict backlog
+ * 5. Version group integrity
+ */
+
+import type { MemoryEntry, MemoryStore } from "./store.js";
+import { resolveTier, type MemoryTier } from "./decay-engine.js";
+import { measureInterferenceDensity } from "./interference-detector.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface CheckResult {
+  name: string;
+  status: "ok" | "warning" | "error";
+  detail: string;
+}
+
+export interface CheckupReport {
+  checks: CheckResult[];
+  totalEntries: number;
+  timestamp: string;
+}
+
+// ---------------------------------------------------------------------------
+// Individual checks
+// ---------------------------------------------------------------------------
+
+/** Check 1: All vectors should have the same dimension. */
+function checkVectorDimensions(entries: MemoryEntry[]): CheckResult {
+  const dims = new Map<number, number>(); // dim → count
+  for (const e of entries) {
+    const d = e.vector?.length ?? 0;
+    dims.set(d, (dims.get(d) ?? 0) + 1);
+  }
+
+  if (dims.size <= 1) {
+    const dim = dims.keys().next().value ?? 0;
+    return { name: "vector_dimensions", status: "ok", detail: `All ${entries.length} entries have dimension ${dim}` };
+  }
+
+  const sorted = [...dims.entries()].sort((a, b) => b[1] - a[1]);
+  const expected = sorted[0][0];
+  const mismatched = sorted.slice(1).reduce((sum, [, c]) => sum + c, 0);
+  return {
+    name: "vector_dimensions",
+    status: "error",
+    detail: `${mismatched} entries have wrong dimension (expected ${expected}): ${sorted.map(([d, c]) => `dim=${d}: ${c}`).join(", ")}`,
+  };
+}
+
+/** Check 2: Memories with missing/empty scope. */
+function checkOrphanMemories(entries: MemoryEntry[]): CheckResult {
+  const orphans = entries.filter(e => !e.scope || e.scope.trim() === "" || e.scope === "__schema__");
+  if (orphans.length === 0) {
+    return { name: "orphan_memories", status: "ok", detail: "No orphan memories found" };
+  }
+  return {
+    name: "orphan_memories",
+    status: orphans.length > 10 ? "error" : "warning",
+    detail: `${orphans.length} memories with missing/empty/schema scope (IDs: ${orphans.slice(0, 5).map(e => e.id.slice(0, 8)).join(", ")}${orphans.length > 5 ? "..." : ""})`,
+  };
+}
+
+/** Check 3: Tier distribution — core should not exceed 500, peripheral should not exceed 70%. */
+function checkTierDistribution(entries: MemoryEntry[]): CheckResult {
+  const tierCounts: Record<MemoryTier, number> = { core: 0, working: 0, peripheral: 0 };
+  for (const e of entries) {
+    const tier = resolveTier(e.metadata);
+    tierCounts[tier]++;
+  }
+
+  const total = entries.length;
+  const issues: string[] = [];
+
+  if (tierCounts.core > 500) {
+    issues.push(`core tier has ${tierCounts.core} entries (max recommended: 500)`);
+  }
+  if (total >= 20 && tierCounts.peripheral / total > 0.7) {
+    const pct = Math.round(tierCounts.peripheral / total * 100);
+    issues.push(`peripheral tier is ${pct}% of total (recommended: ≤70%)`);
+  }
+
+  const dist = `core=${tierCounts.core}, working=${tierCounts.working}, peripheral=${tierCounts.peripheral}`;
+  if (issues.length === 0) {
+    return { name: "tier_distribution", status: "ok", detail: `Healthy distribution: ${dist}` };
+  }
+  return { name: "tier_distribution", status: "warning", detail: `${dist} — ${issues.join("; ")}` };
+}
+
+/** Check 4: Open conflict backlog. */
+function checkConflictBacklog(openConflictCount: number): CheckResult {
+  if (openConflictCount === 0) {
+    return { name: "conflict_backlog", status: "ok", detail: "No open conflicts" };
+  }
+  const status = openConflictCount > 20 ? "error" : openConflictCount > 5 ? "warning" : "ok";
+  return { name: "conflict_backlog", status, detail: `${openConflictCount} open conflicts` };
+}
+
+/** Check 5: Version group integrity — members should reference the same group ID. */
+function checkVersionGroups(entries: MemoryEntry[]): CheckResult {
+  const groups = new Map<string, { ids: string[]; ranks: number[] }>();
+
+  for (const e of entries) {
+    if (!e.metadata) continue;
+    try {
+      const meta = JSON.parse(e.metadata);
+      const group = meta.version_group;
+      if (typeof group !== "string") continue;
+      if (!groups.has(group)) groups.set(group, { ids: [], ranks: [] });
+      const g = groups.get(group)!;
+      g.ids.push(e.id);
+      g.ranks.push(typeof meta.version_rank === "number" ? meta.version_rank : -1);
+    } catch { /* skip */ }
+  }
+
+  if (groups.size === 0) {
+    return { name: "version_groups", status: "ok", detail: "No version groups found" };
+  }
+
+  const issues: string[] = [];
+  for (const [groupId, { ids, ranks }] of groups) {
+    // Single-member group is suspicious
+    if (ids.length < 2) {
+      issues.push(`group ${groupId.slice(0, 8)} has only ${ids.length} member`);
+    }
+    // Check for missing ranks
+    const missingRanks = ranks.filter(r => r < 0).length;
+    if (missingRanks > 0) {
+      issues.push(`group ${groupId.slice(0, 8)} has ${missingRanks} members with missing rank`);
+    }
+  }
+
+  if (issues.length === 0) {
+    return { name: "version_groups", status: "ok", detail: `${groups.size} version groups, all healthy` };
+  }
+  return {
+    name: "version_groups",
+    status: issues.length > 3 ? "error" : "warning",
+    detail: `${groups.size} groups, ${issues.length} issue(s): ${issues.slice(0, 3).join("; ")}${issues.length > 3 ? "..." : ""}`,
+  };
+}
+
+/** Check 6 (F2): Interference density — detect semantic clustering pressure. */
+function checkInterferenceDensity(entries: MemoryEntry[]): CheckResult {
+  // Only check entries with vectors (skip schema/empty entries)
+  const withVectors = entries.filter(e => e.vector && e.vector.length > 0);
+  if (withVectors.length < 10) {
+    return { name: "interference_density", status: "ok", detail: "Too few entries for interference analysis" };
+  }
+
+  // Sample up to 200 entries for performance (interference detection is O(n²))
+  const sample = withVectors.length > 200
+    ? withVectors.sort(() => Math.random() - 0.5).slice(0, 200)
+    : withVectors;
+
+  const density = measureInterferenceDensity(sample);
+  const issues: string[] = [];
+
+  if (density.highRiskCount > sample.length * 0.2) {
+    issues.push(`${density.highRiskCount} high-risk entries (>${Math.round(sample.length * 0.2)} threshold)`);
+  }
+  if (density.avgClusterSize > 5) {
+    issues.push(`avg cluster size ${density.avgClusterSize.toFixed(1)} (recommended: ≤5)`);
+  }
+
+  const detail = `${density.clusterCount} clusters, avg size ${density.avgClusterSize.toFixed(1)}, ${density.highRiskCount} high-risk`;
+  if (issues.length === 0) {
+    return { name: "interference_density", status: "ok", detail: `Healthy: ${detail}` };
+  }
+  return {
+    name: "interference_density",
+    status: issues.length > 1 ? "error" : "warning",
+    detail: `${detail} — ${issues.join("; ")}`,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+export interface CheckupDeps {
+  store: Pick<MemoryStore, "list" | "stats">;
+  openConflictCount: number;
+}
+
+export async function runDataCheckup(deps: CheckupDeps): Promise<CheckupReport> {
+  const entries = await deps.store.list(undefined, undefined, 10000, 0);
+
+  return {
+    checks: [
+      checkVectorDimensions(entries),
+      checkOrphanMemories(entries),
+      checkTierDistribution(entries),
+      checkConflictBacklog(deps.openConflictCount),
+      checkVersionGroups(entries),
+      checkInterferenceDensity(entries),
+    ],
+    totalEntries: entries.length,
+    timestamp: new Date().toISOString(),
+  };
+}
+
+export function formatCheckupReport(report: CheckupReport): string {
+  const lines = [
+    `Data Checkup Report (${report.timestamp})`,
+    `Total entries scanned: ${report.totalEntries}`,
+    "",
+  ];
+
+  for (const check of report.checks) {
+    const icon = check.status === "ok" ? "[OK]" : check.status === "warning" ? "[WARN]" : "[ERR]";
+    lines.push(`${icon} ${check.name}: ${check.detail}`);
+  }
+
+  const errorCount = report.checks.filter(c => c.status === "error").length;
+  const warnCount = report.checks.filter(c => c.status === "warning").length;
+  lines.push("");
+  lines.push(`Summary: ${errorCount} error(s), ${warnCount} warning(s), ${report.checks.length - errorCount - warnCount} ok`);
+
+  return lines.join("\n");
+}