recallnest 1.4.1

package/src/kg-extractor.ts

@@ -0,0 +1,217 @@
+/**
+ * Knowledge Graph Triple Extractor
+ * Uses LLM few-shot OpenIE to extract (subject, predicate, object) triples from memory text.
+ * Gated by RECALLNEST_KG_MODE env var.
+ */
+
+import type { LLMClient } from "./llm-client.js";
+import type { KGStore } from "./kg-store.js";
+import { logInfo, logWarn } from "./stderr-log.js";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface RawTriple {
+  subject: string;
+  predicate: string;
+  object: string;
+  confidence: number;
+}
+
+export interface KGExtractorConfig {
+  /** Minimum confidence to keep a triple (0-1). Default: 0.6 */
+  minConfidence?: number;
+  /** LLM client for extraction */
+  llmClient: LLMClient;
+  /** KG store for persisting triples */
+  kgStore: KGStore;
+}
+
+interface LlmExtractionResponse {
+  triples: RawTriple[];
+}
+
+// ============================================================================
+// Feature Gate
+// ============================================================================
+
+/** Check if KG mode is enabled (env var) */
+export function isKGModeEnabled(): boolean {
+  return process.env.RECALLNEST_KG_MODE === "true";
+}
+
+// ============================================================================
+// Entity Normalization
+// ============================================================================
+
+/**
+ * Normalize entity names for consistent matching:
+ * - Trim whitespace, collapse internal whitespace
+ * - Title case for Latin text, CJK unchanged
+ * - Strip surrounding quotes/brackets
+ */
+export function normalizeEntity(raw: string): string {
+  let entity = raw.trim();
+
+  // Strip surrounding quotes and brackets
+  entity = entity.replace(/^[\s"'""''「」『』【】\[\]()（）]+/, "");
+  entity = entity.replace(/[\s"'""''「」『』【】\[\]()（）]+$/, "");
+
+  // Collapse whitespace
+  entity = entity.replace(/\s+/g, " ").trim();
+
+  if (!entity) return "";
+
+  // For CJK text, return as-is
+  if (/[\u4e00-\u9fff\u3400-\u4dbf]/.test(entity)) {
+    return entity;
+  }
+
+  // Title Case for English entities
+  return entity
+    .toLowerCase()
+    .replace(/\b\w/g, (ch) => ch.toUpperCase());
+}
+
+/**
+ * Normalize predicate to snake_case for consistency.
+ */
+export function normalizePredicate(raw: string): string {
+  return raw
+    .trim()
+    .toLowerCase()
+    .replace(/[\s-]+/g, "_")
+    .replace(/[^a-z0-9_\u4e00-\u9fff]/g, "");
+}
+
+// ============================================================================
+// Extraction Prompt
+// ============================================================================
+
+const KG_EXTRACT_SYSTEM = `You are a knowledge graph extraction assistant. Extract (subject, predicate, object) triples from text. Respond with valid JSON only.`;
+
+function buildKGExtractionPrompt(text: string): string {
+  return `Extract knowledge graph triples from the text below.
+
+Rules:
+- Extract relationships between named entities, concepts, tools, people, projects
+- Use simple predicates: uses, created_by, works_with, friend_of, depends_on, located_in, member_of, prefers, knows, discussed, mentioned_in, related_to, is_a, has, manages, built, wrote, lives_in
+- Assign confidence (0-1) based on how explicit the relationship is
+- Skip vague or speculative relationships (confidence < 0.5)
+- 3-8 triples per text typically
+
+Examples:
+
+Text: "Alice uses Python for her data science projects at Google"
+{"triples":[{"subject":"Alice","predicate":"uses","object":"Python","confidence":0.95},{"subject":"Alice","predicate":"works_at","object":"Google","confidence":0.90}]}
+
+Text: "RecallNest depends on LanceDB for vector storage and uses OpenAI for embeddings"
+{"triples":[{"subject":"RecallNest","predicate":"depends_on","object":"LanceDB","confidence":0.95},{"subject":"RecallNest","predicate":"uses","object":"OpenAI","confidence":0.90}]}
+
+Now extract triples from:
+
+${text}`;
+}
+
+// ============================================================================
+// Extractor
+// ============================================================================
+
+export class KGExtractor {
+  private readonly minConfidence: number;
+  private readonly llm: LLMClient;
+  private readonly kgStore: KGStore;
+
+  constructor(config: KGExtractorConfig) {
+    this.minConfidence = config.minConfidence ?? 0.6;
+    this.llm = config.llmClient;
+    this.kgStore = config.kgStore;
+  }
+
+  /**
+   * Extract triples from text and persist to KG store.
+   * Returns the number of triples stored.
+   */
+  async extractAndStore(
+    text: string,
+    sourceMemoryId: string,
+    scope: string,
+  ): Promise<number> {
+    const rawTriples = await this.extract(text);
+    if (rawTriples.length === 0) return 0;
+
+    const triples = await this.kgStore.createTriples(
+      rawTriples.map((t) => ({
+        scope,
+        subject: t.subject,
+        predicate: t.predicate,
+        object: t.object,
+        confidence: t.confidence,
+        source_memory_id: sourceMemoryId,
+        source_text: text.slice(0, 500), // Cap source text
+      })),
+    );
+
+    logInfo(`[KG] stored ${triples.length} triples from memory ${sourceMemoryId}`);
+    return triples.length;
+  }
+
+  /**
+   * Extract raw triples from text via LLM. Does not persist.
+   */
+  async extract(text: string): Promise<RawTriple[]> {
+    if (!text || text.length < 10) return [];
+
+    const prompt = buildKGExtractionPrompt(text);
+
+    let response: LlmExtractionResponse | null = null;
+    try {
+      response = await this.llm.chatJson<LlmExtractionResponse>(
+        KG_EXTRACT_SYSTEM,
+        prompt,
+      );
+    } catch (err) {
+      logWarn(`[KG] LLM extraction failed: ${String(err)}`);
+      return [];
+    }
+
+    if (!response?.triples || !Array.isArray(response.triples)) {
+      return [];
+    }
+
+    // Validate, normalize, filter
+    const valid: RawTriple[] = [];
+    for (const t of response.triples) {
+      if (!t.subject || !t.predicate || !t.object) continue;
+
+      const subject = normalizeEntity(t.subject);
+      const predicate = normalizePredicate(t.predicate);
+      const object = normalizeEntity(t.object);
+      const confidence = Number(t.confidence) || 0;
+
+      if (!subject || !predicate || !object) continue;
+      if (confidence < this.minConfidence) continue;
+      if (subject === object) continue; // Skip self-referencing
+
+      valid.push({ subject, predicate, object, confidence });
+    }
+
+    // Deduplicate within batch
+    const seen = new Set<string>();
+    return valid.filter((t) => {
+      const key = `${t.subject}\x00${t.predicate}\x00${t.object}`;
+      if (seen.has(key)) return false;
+      seen.add(key);
+      return true;
+    });
+  }
+}
+
+// ============================================================================
+// Factory
+// ============================================================================
+
+export function createKGExtractor(config: KGExtractorConfig): KGExtractor {
+  return new KGExtractor(config);
+}

package/src/kg-store.ts

@@ -0,0 +1,353 @@
+/**
+ * Knowledge Graph Triple Store
+ * Stores (subject, predicate, object) triples in a dedicated LanceDB table.
+ * Pure relational — no vector columns.
+ */
+
+import type * as LanceDB from "@lancedb/lancedb";
+import { createHash } from "node:crypto";
+import { loadLanceDB } from "./store.js";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface KGTriple {
+  /** Deterministic ID: sha256(scope + subject + predicate + object) */
+  id: string;
+  scope: string;
+  subject: string;
+  predicate: string;
+  object: string;
+  /** Extraction confidence 0-1 */
+  confidence: number;
+  /** ID of the source memory entry */
+  source_memory_id: string;
+  /** Original text snippet used for extraction */
+  source_text: string;
+  timestamp: number;
+}
+
+export interface KGStoreConfig {
+  /** Reuse the same LanceDB connection path as MemoryStore */
+  dbPath: string;
+}
+
+export interface NeighborhoodResult {
+  entity: string;
+  triples: KGTriple[];
+  /** How many hops from the seed entity */
+  hops: number;
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+const KG_TABLE_NAME = "kg_triples";
+
+/** Deterministic triple ID for dedup */
+export function tripleId(scope: string, subject: string, predicate: string, object: string): string {
+  const raw = `${scope}\x00${subject}\x00${predicate}\x00${object}`;
+  return createHash("sha256").update(raw).digest("hex").slice(0, 32);
+}
+
+function escapeSql(value: string): string {
+  return value.replace(/'/g, "''");
+}
+
+// ============================================================================
+// KG Store
+// ============================================================================
+
+export class KGStore {
+  private db: LanceDB.Connection | null = null;
+  private table: LanceDB.Table | null = null;
+  private initPromise: Promise<void> | null = null;
+
+  constructor(private readonly config: KGStoreConfig) {}
+
+  // --------------------------------------------------------------------------
+  // Initialization
+  // --------------------------------------------------------------------------
+
+  private async ensureInitialized(): Promise<void> {
+    if (this.table) return;
+    if (this.initPromise) return this.initPromise;
+
+    this.initPromise = this.doInitialize().catch((err) => {
+      this.initPromise = null;
+      throw err;
+    });
+    return this.initPromise;
+  }
+
+  private async doInitialize(): Promise<void> {
+    const lancedb = await loadLanceDB();
+    const db = await lancedb.connect(this.config.dbPath);
+
+    let table: LanceDB.Table;
+    try {
+      table = await db.openTable(KG_TABLE_NAME);
+    } catch {
+      // Table doesn't exist — create with schema entry then remove it
+      const schemaEntry: KGTriple = {
+        id: "__schema__",
+        scope: "global",
+        subject: "",
+        predicate: "",
+        object: "",
+        confidence: 0,
+        source_memory_id: "",
+        source_text: "",
+        timestamp: 0,
+      };
+
+      try {
+        table = await db.createTable(KG_TABLE_NAME, [schemaEntry]);
+        await table.delete('id = "__schema__"');
+      } catch (createErr) {
+        if (String(createErr).includes("already exists")) {
+          table = await db.openTable(KG_TABLE_NAME);
+        } else {
+          throw createErr;
+        }
+      }
+    }
+
+    // Create FTS index on subject for entity lookup (optional)
+    try {
+      const indices = await table.listIndices();
+      const hasSubjectFts = indices?.some(
+        (idx: any) => idx.indexType === "FTS" && idx.columns?.includes("subject"),
+      );
+      if (!hasSubjectFts) {
+        const lance = await loadLanceDB();
+        await table.createIndex("subject", { config: (lance as any).Index.fts() });
+      }
+    } catch {
+      // FTS on subject is optional — BFS still works without it
+    }
+
+    this.db = db;
+    this.table = table;
+  }
+
+  // --------------------------------------------------------------------------
+  // Write
+  // --------------------------------------------------------------------------
+
+  async createTriple(triple: Omit<KGTriple, "id" | "timestamp">): Promise<KGTriple> {
+    await this.ensureInitialized();
+
+    const full: KGTriple = {
+      ...triple,
+      id: tripleId(triple.scope, triple.subject, triple.predicate, triple.object),
+      timestamp: Date.now(),
+    };
+
+    // Upsert: delete old then add new (LanceDB has no native upsert)
+    try {
+      await this.table!.delete(`id = '${escapeSql(full.id)}'`);
+    } catch {
+      // OK if not found
+    }
+    await this.table!.add([full]);
+    return full;
+  }
+
+  async createTriples(triples: Array<Omit<KGTriple, "id" | "timestamp">>): Promise<KGTriple[]> {
+    if (triples.length === 0) return [];
+    await this.ensureInitialized();
+
+    const now = Date.now();
+    const fullTriples: KGTriple[] = triples.map((t) => ({
+      ...t,
+      id: tripleId(t.scope, t.subject, t.predicate, t.object),
+      timestamp: now,
+    }));
+
+    // Deduplicate by id (same triple in batch)
+    const seen = new Set<string>();
+    const deduped = fullTriples.filter((t) => {
+      if (seen.has(t.id)) return false;
+      seen.add(t.id);
+      return true;
+    });
+
+    // Batch delete existing then add
+    const ids = deduped.map((t) => `'${escapeSql(t.id)}'`).join(", ");
+    try {
+      await this.table!.delete(`id IN (${ids})`);
+    } catch {
+      // OK if none found
+    }
+    await this.table!.add(deduped);
+    return deduped;
+  }
+
+  // --------------------------------------------------------------------------
+  // Read — Edge queries
+  // --------------------------------------------------------------------------
+
+  /** Get all triples where entity is the subject */
+  async getOutgoingEdges(entity: string, scope?: string): Promise<KGTriple[]> {
+    await this.ensureInitialized();
+
+    let filter = `subject = '${escapeSql(entity)}'`;
+    if (scope) filter += ` AND scope = '${escapeSql(scope)}'`;
+
+    const rows = await this.table!.query().where(filter).limit(500).toArray();
+    return rows.map(rowToTriple);
+  }
+
+  /** Get all triples where entity is the object */
+  async getIncomingEdges(entity: string, scope?: string): Promise<KGTriple[]> {
+    await this.ensureInitialized();
+
+    let filter = `object = '${escapeSql(entity)}'`;
+    if (scope) filter += ` AND scope = '${escapeSql(scope)}'`;
+
+    const rows = await this.table!.query().where(filter).limit(500).toArray();
+    return rows.map(rowToTriple);
+  }
+
+  /**
+   * BFS neighborhood traversal up to `maxHops` hops.
+   * Returns all triples reachable from the seed entity.
+   */
+  async getNeighborhood(
+    seedEntities: string[],
+    maxHops = 2,
+    scope?: string,
+  ): Promise<NeighborhoodResult[]> {
+    await this.ensureInitialized();
+
+    const visited = new Map<string, number>(); // entity -> min hops
+    const allTriples: KGTriple[] = [];
+    let frontier = [...seedEntities];
+
+    for (const seed of seedEntities) {
+      visited.set(seed, 0);
+    }
+
+    for (let hop = 1; hop <= maxHops; hop++) {
+      if (frontier.length === 0) break;
+
+      const nextFrontier: string[] = [];
+
+      for (const entity of frontier) {
+        const [outgoing, incoming] = await Promise.all([
+          this.getOutgoingEdges(entity, scope),
+          this.getIncomingEdges(entity, scope),
+        ]);
+
+        for (const t of [...outgoing, ...incoming]) {
+          allTriples.push(t);
+
+          for (const neighbor of [t.subject, t.object]) {
+            if (!visited.has(neighbor)) {
+              visited.set(neighbor, hop);
+              nextFrontier.push(neighbor);
+            }
+          }
+        }
+      }
+
+      frontier = nextFrontier;
+    }
+
+    // Group by entity
+    const results: NeighborhoodResult[] = [];
+    for (const [entity, hops] of visited) {
+      const related = allTriples.filter(
+        (t) => t.subject === entity || t.object === entity,
+      );
+      if (related.length > 0 || seedEntities.includes(entity)) {
+        results.push({ entity, triples: related, hops });
+      }
+    }
+
+    return results;
+  }
+
+  // --------------------------------------------------------------------------
+  // Read — Entity listing
+  // --------------------------------------------------------------------------
+
+  /** Get all unique entities in scope */
+  async getAllEntities(scope?: string): Promise<string[]> {
+    await this.ensureInitialized();
+
+    const filter = scope ? `scope = '${escapeSql(scope)}'` : "1=1";
+    const rows = await this.table!.query()
+      .select(["subject", "object"])
+      .where(filter)
+      .limit(10000)
+      .toArray();
+
+    const entities = new Set<string>();
+    for (const row of rows) {
+      if (row.subject) entities.add(row.subject as string);
+      if (row.object) entities.add(row.object as string);
+    }
+    return [...entities];
+  }
+
+  /** Check if an entity exists in the KG */
+  async hasEntity(entity: string, scope?: string): Promise<boolean> {
+    await this.ensureInitialized();
+
+    const safeEntity = escapeSql(entity);
+    let filter = `subject = '${safeEntity}' OR object = '${safeEntity}'`;
+    if (scope) filter = `(${filter}) AND scope = '${escapeSql(scope)}'`;
+
+    const rows = await this.table!.query().where(filter).limit(1).toArray();
+    return rows.length > 0;
+  }
+
+  // --------------------------------------------------------------------------
+  // Delete
+  // --------------------------------------------------------------------------
+
+  /** Delete all triples extracted from a given source memory */
+  async deleteBySource(sourceMemoryId: string): Promise<void> {
+    await this.ensureInitialized();
+    await this.table!.delete(`source_memory_id = '${escapeSql(sourceMemoryId)}'`);
+  }
+
+  /** Delete all triples in scope */
+  async deleteByScope(scope: string): Promise<void> {
+    await this.ensureInitialized();
+    await this.table!.delete(`scope = '${escapeSql(scope)}'`);
+  }
+
+  // --------------------------------------------------------------------------
+  // Stats
+  // --------------------------------------------------------------------------
+
+  async countTriples(scope?: string): Promise<number> {
+    await this.ensureInitialized();
+    const filter = scope ? `scope = '${escapeSql(scope)}'` : "1=1";
+    const rows = await this.table!.query().where(filter).select(["id"]).limit(100000).toArray();
+    return rows.length;
+  }
+}
+
+// ============================================================================
+// Row mapping
+// ============================================================================
+
+function rowToTriple(row: Record<string, unknown>): KGTriple {
+  return {
+    id: row.id as string,
+    scope: (row.scope as string) ?? "global",
+    subject: row.subject as string,
+    predicate: row.predicate as string,
+    object: row.object as string,
+    confidence: Number(row.confidence),
+    source_memory_id: row.source_memory_id as string,
+    source_text: row.source_text as string,
+    timestamp: Number(row.timestamp),
+  };
+}

package/src/legacy-scope-review.ts

@@ -0,0 +1,53 @@
+import type { LegacyScopeIssueKind } from "./store.js";
+
+export interface LegacyScopeReviewRecord {
+  decision: "keep";
+  kind: LegacyScopeIssueKind;
+  reason: string;
+  reviewedAt: string;
+}
+
+function asObject(value: unknown): Record<string, unknown> | undefined {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
+  return value as Record<string, unknown>;
+}
+
+export function parseLegacyScopeReview(raw: unknown): LegacyScopeReviewRecord | undefined {
+  const record = asObject(raw);
+  if (!record) return undefined;
+  if (record.decision !== "keep") return undefined;
+  if (record.kind !== "missing" && record.kind !== "empty" && record.kind !== "global") return undefined;
+  if (typeof record.reason !== "string" || !record.reason.trim()) return undefined;
+  if (typeof record.reviewedAt !== "string" || !Number.isFinite(Date.parse(record.reviewedAt))) return undefined;
+  return {
+    decision: "keep",
+    kind: record.kind,
+    reason: record.reason.trim(),
+    reviewedAt: record.reviewedAt,
+  };
+}
+
+export function buildLegacyScopeKeepReview(
+  kind: LegacyScopeIssueKind,
+  reason: string,
+  reviewedAt = new Date().toISOString(),
+): LegacyScopeReviewRecord {
+  const normalizedReason = reason.trim();
+  if (!normalizedReason) {
+    throw new Error("legacy scope keep review reason must not be empty.");
+  }
+  return {
+    decision: "keep",
+    kind,
+    reason: normalizedReason,
+    reviewedAt,
+  };
+}
+
+export function suppressesLegacyScopeIssue(
+  raw: unknown,
+  kind: LegacyScopeIssueKind,
+): boolean {
+  const review = parseLegacyScopeReview(raw);
+  return review?.decision === "keep" && review.kind === kind;
+}