the-campfire 0.1.0

package/server/semantic-memory.ts

@@ -0,0 +1,389 @@
+/**
+ * Layer 1: Semantic Memory
+ *
+ * Persistent, shared knowledge base anchored to a git repository.
+ * Backed by LanceDB (embedded, TypeScript-native vector database — same stack as Continue.dev).
+ *
+ * Two tables:
+ *   - fragments:    episodic/semantic MemoryFragments with vector embeddings
+ *   - consolidated: distilled ConsolidatedKnowledge per semantic tag
+ *
+ * Data is stored at ~/.campfire/memory/lancedb/ (one DB directory, two tables).
+ *
+ * Retrieval is vector similarity search via LanceDB's built-in ANN/flat-scan.
+ * Embeddings are generated by the configured provider (OpenAI or Ollama) at write time.
+ */
+
+import { mkdirSync, appendFileSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { randomUUID } from "node:crypto";
+import type { BackendType } from "./session-types.js";
+import { embed, getEmbeddingDim } from "./embedding.js";
+
+// ─── Types ───────────────────────────────────────────────────────────────────
+
+export type MemoryType = "observation" | "hypothesis" | "decision" | "pattern";
+
+export interface GitContext {
+  commitHash?: string;
+  branch: string;
+  files: string[];
+  repoRoot: string;
+}
+
+export interface MemoryFragment {
+  id: string;
+  sessionId: string;
+  agentId: string;
+  backendType: BackendType;
+  timestamp: number;
+  type: MemoryType;
+  content: string;
+  gitContext: GitContext;
+  references: string[];
+  confidence: number;
+  tags: string[];
+  consolidatedInto?: string;
+  isConsolidated: boolean;
+  // Stored in LanceDB as a Float32 vector column — null when provider = "none"
+  embedding?: number[];
+}
+
+export interface ConsolidatedKnowledge {
+  id: string;
+  tag: string;
+  summary: string;
+  sourceFragments: string[];
+  lastUpdated: number;
+  confidence: number;
+  repoRoot: string;
+}
+
+export interface MemoryQueryOptions {
+  limit?: number;
+  repoRoot?: string;
+  tags?: string[];
+  type?: MemoryType;
+  sessionId?: string;
+}
+
+// ─── LanceDB integration ─────────────────────────────────────────────────────
+
+// Lazily import to avoid startup cost when memory is not used
+type LanceDB = typeof import("@lancedb/lancedb");
+type LanceTable = Awaited<ReturnType<Awaited<ReturnType<LanceDB["connect"]>>["openTable"]>>;
+
+let _db: Awaited<ReturnType<LanceDB["connect"]>> | null = null;
+let _fragmentsTable: LanceTable | null = null;
+let _consolidatedTable: LanceTable | null = null;
+let _memoryDir = join(homedir(), ".campfire", "memory", "lancedb");
+
+async function getDB() {
+  if (_db) return _db;
+  const lancedb = await import("@lancedb/lancedb");
+  mkdirSync(_memoryDir, { recursive: true });
+  _db = await lancedb.connect(_memoryDir);
+  return _db;
+}
+
+async function getFragmentsTable(): Promise<LanceTable> {
+  if (_fragmentsTable) return _fragmentsTable;
+  const db = await getDB();
+  const tableNames = await db.tableNames();
+  if (tableNames.includes("fragments")) {
+    _fragmentsTable = await db.openTable("fragments");
+  } else {
+    // Create with a seed row so the schema is established, then delete seed
+    const dim = getEmbeddingDim();
+    const seed = [{
+      id: "__seed__",
+      sessionId: "",
+      agentId: "",
+      backendType: "claude" as BackendType,
+      timestamp: 0,
+      type: "observation" as MemoryType,
+      content: "",
+      gitContextJson: "{}",
+      referencesJson: "[]",
+      tagsJson: "[]",
+      confidence: 0,
+      consolidatedInto: "",
+      isConsolidated: false,
+      vector: Array(dim).fill(0) as number[],
+    }];
+    _fragmentsTable = await db.createTable("fragments", seed);
+    await _fragmentsTable.delete('id = "__seed__"');
+  }
+  return _fragmentsTable;
+}
+
+async function getConsolidatedTable(): Promise<LanceTable> {
+  if (_consolidatedTable) return _consolidatedTable;
+  const db = await getDB();
+  const tableNames = await db.tableNames();
+  if (tableNames.includes("consolidated")) {
+    _consolidatedTable = await db.openTable("consolidated");
+  } else {
+    const seed = [{
+      id: "__seed__",
+      tag: "",
+      summary: "",
+      sourceFragmentsJson: "[]",
+      lastUpdated: 0,
+      confidence: 0,
+      repoRoot: "",
+    }];
+    _consolidatedTable = await db.createTable("consolidated", seed);
+    await _consolidatedTable.delete('id = "__seed__"');
+  }
+  return _consolidatedTable;
+}
+
+// ─── Internal row <-> domain object mapping ──────────────────────────────────
+
+function fragmentToRow(fragment: MemoryFragment, embedding: number[] | null) {
+  const dim = getEmbeddingDim();
+  return {
+    id: fragment.id,
+    sessionId: fragment.sessionId,
+    agentId: fragment.agentId,
+    backendType: fragment.backendType,
+    timestamp: fragment.timestamp,
+    type: fragment.type,
+    content: fragment.content,
+    gitContextJson: JSON.stringify(fragment.gitContext),
+    referencesJson: JSON.stringify(fragment.references),
+    tagsJson: JSON.stringify(fragment.tags),
+    confidence: fragment.confidence,
+    consolidatedInto: fragment.consolidatedInto ?? "",
+    isConsolidated: fragment.isConsolidated,
+    vector: embedding ?? Array(dim).fill(0),
+  };
+}
+
+function rowToFragment(row: Record<string, unknown>): MemoryFragment {
+  return {
+    id: row.id as string,
+    sessionId: row.sessionId as string,
+    agentId: row.agentId as string,
+    backendType: row.backendType as BackendType,
+    timestamp: row.timestamp as number,
+    type: row.type as MemoryType,
+    content: row.content as string,
+    gitContext: JSON.parse(row.gitContextJson as string || "{}") as GitContext,
+    references: JSON.parse(row.referencesJson as string || "[]") as string[],
+    tags: JSON.parse(row.tagsJson as string || "[]") as string[],
+    confidence: row.confidence as number,
+    consolidatedInto: (row.consolidatedInto as string) || undefined,
+    isConsolidated: row.isConsolidated as boolean,
+  };
+}
+
+function rowToConsolidated(row: Record<string, unknown>): ConsolidatedKnowledge {
+  return {
+    id: row.id as string,
+    tag: row.tag as string,
+    summary: row.summary as string,
+    sourceFragments: JSON.parse(row.sourceFragmentsJson as string || "[]") as string[],
+    lastUpdated: row.lastUpdated as number,
+    confidence: row.confidence as number,
+    repoRoot: row.repoRoot as string,
+  };
+}
+
+// ─── Public API ──────────────────────────────────────────────────────────────
+
+export interface StoreFragmentOptions {
+  sessionId: string;
+  agentId: string;
+  backendType: BackendType;
+  type: MemoryType;
+  content: string;
+  gitContext: GitContext;
+  tags?: string[];
+  confidence?: number;
+  references?: string[];
+}
+
+/**
+ * Store a new memory fragment. Generates an embedding at write time.
+ * Non-blocking: embedding generation failures are logged and the fragment
+ * is stored with a zero vector (still retrievable by metadata filters).
+ */
+export async function storeFragment(opts: StoreFragmentOptions): Promise<MemoryFragment> {
+  const fragment: MemoryFragment = {
+    id: randomUUID(),
+    sessionId: opts.sessionId,
+    agentId: opts.agentId,
+    backendType: opts.backendType,
+    timestamp: Date.now(),
+    type: opts.type,
+    content: opts.content,
+    gitContext: opts.gitContext,
+    references: opts.references ?? [],
+    confidence: opts.confidence ?? 0.7,
+    tags: opts.tags ?? [],
+    isConsolidated: false,
+  };
+
+  const embedding = await embed(opts.content);
+  const table = await getFragmentsTable();
+  await table.add([fragmentToRow(fragment, embedding)]);
+
+  return fragment;
+}
+
+/**
+ * Semantic similarity search over stored fragments.
+ * If an embedding is available, uses vector search.
+ * Falls back to a full table scan filtered by metadata (tag/session/repo).
+ */
+export async function queryFragments(
+  query: string,
+  options: MemoryQueryOptions = {},
+): Promise<MemoryFragment[]> {
+  const { limit = 10, repoRoot, tags, type, sessionId } = options;
+  const table = await getFragmentsTable();
+  const queryVector = await embed(query);
+
+  let rows: Record<string, unknown>[];
+
+  if (queryVector) {
+    // Vector similarity search — LanceDB returns closest by cosine distance
+    let search = table.search(queryVector).limit(limit * 3); // over-fetch for post-filter
+    const result = await search.toArray();
+    rows = result as unknown as Record<string, unknown>[];
+  } else {
+    // No embedding configured — full scan
+    const result = await table.query().limit(500).toArray();
+    rows = result as unknown as Record<string, unknown>[];
+  }
+
+  // Post-filter by metadata
+  let fragments = rows.map(rowToFragment).filter((f) => {
+    if (repoRoot && f.gitContext.repoRoot !== repoRoot) return false;
+    if (sessionId && f.sessionId !== sessionId) return false;
+    if (type && f.type !== type) return false;
+    if (tags && tags.length > 0) {
+      const fragTags = new Set(f.tags);
+      if (!tags.some((t) => fragTags.has(t))) return false;
+    }
+    return true;
+  });
+
+  // If no vector search, sort by recency
+  if (!queryVector) {
+    fragments.sort((a, b) => b.timestamp - a.timestamp);
+  }
+
+  return fragments.slice(0, limit);
+}
+
+/**
+ * Retrieve all fragments for a session (used during consolidation).
+ */
+export async function getSessionFragments(sessionId: string): Promise<MemoryFragment[]> {
+  const table = await getFragmentsTable();
+  const result = await table.query().limit(10000).toArray();
+  return (result as unknown as Record<string, unknown>[])
+    .map(rowToFragment)
+    .filter((f) => f.sessionId === sessionId);
+}
+
+/**
+ * Consolidate episodic fragments from a session into semantic knowledge.
+ * Groups fragments by tag, synthesizes a summary per tag, and stores in
+ * the consolidated table.
+ *
+ * Returns the list of consolidated knowledge items created/updated.
+ *
+ * Note: Summary synthesis uses simple concatenation by default.
+ * In production, replace the `synthesize` function with an LLM call.
+ */
+export async function consolidateSession(
+  sessionId: string,
+  repoRoot: string,
+): Promise<ConsolidatedKnowledge[]> {
+  const fragments = await getSessionFragments(sessionId);
+  if (fragments.length === 0) return [];
+
+  // Group by tag
+  const byTag = new Map<string, MemoryFragment[]>();
+  for (const f of fragments) {
+    for (const tag of f.tags.length > 0 ? f.tags : ["general"]) {
+      if (!byTag.has(tag)) byTag.set(tag, []);
+      byTag.get(tag)!.push(f);
+    }
+  }
+
+  const table = await getConsolidatedTable();
+  const results: ConsolidatedKnowledge[] = [];
+
+  for (const [tag, tagFragments] of byTag) {
+    const avgConfidence = tagFragments.reduce((s, f) => s + f.confidence, 0) / tagFragments.length;
+    const summary = synthesize(tag, tagFragments);
+    const knowledge: ConsolidatedKnowledge = {
+      id: randomUUID(),
+      tag,
+      summary,
+      sourceFragments: tagFragments.map((f) => f.id),
+      lastUpdated: Date.now(),
+      confidence: avgConfidence,
+      repoRoot,
+    };
+
+    await table.add([{
+      id: knowledge.id,
+      tag: knowledge.tag,
+      summary: knowledge.summary,
+      sourceFragmentsJson: JSON.stringify(knowledge.sourceFragments),
+      lastUpdated: knowledge.lastUpdated,
+      confidence: knowledge.confidence,
+      repoRoot: knowledge.repoRoot,
+    }]);
+
+    results.push(knowledge);
+  }
+
+  return results;
+}
+
+/**
+ * Retrieve consolidated knowledge for a repo, optionally filtered by tag.
+ */
+export async function getConsolidatedKnowledge(
+  repoRoot: string,
+  tag?: string,
+): Promise<ConsolidatedKnowledge[]> {
+  const table = await getConsolidatedTable();
+  const result = await table.query().limit(1000).toArray();
+  return (result as unknown as Record<string, unknown>[])
+    .map(rowToConsolidated)
+    .filter((k) => k.repoRoot === repoRoot && (!tag || k.tag === tag));
+}
+
+// ─── Internal helpers ─────────────────────────────────────────────────────────
+
+/**
+ * Simple synthesis: concatenate fragment contents into a bullet list.
+ * Replace with an LLM call for production-quality summaries.
+ */
+function synthesize(tag: string, fragments: MemoryFragment[]): string {
+  const sorted = [...fragments].sort((a, b) => b.confidence - a.confidence);
+  const bullets = sorted
+    .slice(0, 10)
+    .map((f) => `- [${f.type}] ${f.content}`)
+    .join("\n");
+  return `Knowledge about "${tag}":\n${bullets}`;
+}
+
+// ─── Test helpers ─────────────────────────────────────────────────────────────
+
+/** Override memory dir for tests — must be called before any DB access. */
+export function _resetForTest(dir: string): void {
+  _db = null;
+  _fragmentsTable = null;
+  _consolidatedTable = null;
+  _memoryDir = dir;
+}