recallnest 2.3.0 → 2.6.0

package/src/kg-store.ts

@@ -19,15 +19,25 @@ export interface KGTriple {
   subject: string;
   predicate: string;
   object: string;
-  /** Extraction confidence 0-1 */
+  /** Extraction confidence 0-1 (max across all extractions of this triple) */
   confidence: number;
-  /** ID of the source memory entry */
+  /** First source memory that contributed this triple (read-path compat) */
   source_memory_id: string;
   /** Original text snippet used for extraction */
   source_text: string;
+  /** Last time this triple was (re-)extracted */
   timestamp: number;
+  /** Number of distinct source memories that mentioned this fact (frequency = importance signal) */
+  mention_count: number;
+  /** Timestamp of the first extraction of this triple */
+  first_seen: number;
+  /** JSON array of contributing source memory ids, capped at SOURCE_IDS_CAP */
+  source_memory_ids: string;
 }
 
+/** Caller-facing input: counting fields are managed internally by the store. */
+export type KGTripleInput = Omit<KGTriple, "id" | "timestamp" | "mention_count" | "first_seen" | "source_memory_ids">;
+
 export interface KGStoreConfig {
   /** Reuse the same LanceDB connection path as MemoryStore */
   dbPath: string;
@@ -56,6 +66,25 @@ function escapeSql(value: string): string {
   return value.replace(/'/g, "''");
 }
 
+/**
+ * Cap on the source_memory_ids list (row-bloat guard). mention_count itself is
+ * NOT capped: once the list is full, a returning 201st+ source can no longer be
+ * recognized and would double-count on re-extraction. Accepted error — a fact
+ * mentioned by 200+ distinct memories is extreme head-of-distribution either way.
+ */
+const SOURCE_IDS_CAP = 200;
+
+/** Parse the JSON source id list, falling back to the legacy single-id column. */
+export function parseSourceIds(raw: string | undefined | null, fallbackId?: string): string[] {
+  if (raw) {
+    try {
+      const arr = JSON.parse(raw);
+      if (Array.isArray(arr)) return arr.filter((x): x is string => typeof x === "string");
+    } catch { /* malformed — fall back below */ }
+  }
+  return fallbackId ? [fallbackId] : [];
+}
+
 // ============================================================================
 // KG Store
 // ============================================================================
@@ -64,9 +93,19 @@ export class KGStore {
   private db: LanceDB.Connection | null = null;
   private table: LanceDB.Table | null = null;
   private initPromise: Promise<void> | null = null;
+  /** In-process write serialization: concurrent workers (e.g. backfill
+   * --concurrency) would otherwise race read-modify-write AND collide on
+   * LanceDB optimistic commits. Cross-process races stay best-effort. */
+  private writeChain: Promise<unknown> = Promise.resolve();
 
   constructor(private readonly config: KGStoreConfig) {}
 
+  private withWriteLock<T>(fn: () => Promise<T>): Promise<T> {
+    const run = this.writeChain.then(fn, fn);
+    this.writeChain = run.catch(() => {});
+    return run;
+  }
+
   // --------------------------------------------------------------------------
   // Initialization
   // --------------------------------------------------------------------------
@@ -101,6 +140,9 @@ export class KGStore {
         source_memory_id: "",
         source_text: "",
         timestamp: 0,
+        mention_count: 0,
+        first_seen: 0,
+        source_memory_ids: "[]",
       };
 
       try {
@@ -115,6 +157,31 @@ export class KGStore {
       }
     }
 
+    // Defensive migration for pre-mention-count tables. Production has never
+    // created this table (KG mode was never enabled), so this only guards
+    // stale dev/test databases. Best-effort: on failure we log and continue —
+    // rowToTriple() still reads legacy rows via fallbacks.
+    try {
+      const schema = await table.schema();
+      const fields = new Set(schema.fields.map((f) => f.name));
+      const missing: Array<{ name: string; valueSql: string }> = [];
+      if (!fields.has("mention_count")) missing.push({ name: "mention_count", valueSql: "1" });
+      // Constant 0 (not a column reference — SQL-dialect-proof); rowToTriple treats
+      // first_seen<=0 as "unknown" and falls back to the row's timestamp.
+      if (!fields.has("first_seen")) missing.push({ name: "first_seen", valueSql: "0" });
+      if (!fields.has("source_memory_ids")) {
+        missing.push({ name: "source_memory_ids", valueSql: `concat('["', source_memory_id, '"]')` });
+      }
+      if (missing.length > 0) {
+        await table.addColumns(missing);
+      }
+    } catch (err) {
+      console.error(
+        "[recallnest] KG schema migration check failed (legacy rows still readable via fallbacks):",
+        err instanceof Error ? err.message : String(err),
+      );
+    }
+
     // Create FTS index on subject for entity lookup (optional)
     try {
       const indices = await table.listIndices();
@@ -138,53 +205,93 @@ export class KGStore {
   // Write
   // --------------------------------------------------------------------------
 
-  async createTriple(triple: Omit<KGTriple, "id" | "timestamp">): Promise<KGTriple> {
-    await this.ensureInitialized();
-
-    const full: KGTriple = {
-      ...triple,
-      id: tripleId(triple.scope, triple.subject, triple.predicate, triple.object),
-      timestamp: Date.now(),
-    };
-
-    // Upsert: delete old then add new (LanceDB has no native upsert)
-    try {
-      await this.table!.delete(`id = '${escapeSql(full.id)}'`);
-    } catch {
-      // OK if not found
-    }
-    await this.table!.add([full]);
-    return full;
+  async createTriple(triple: KGTripleInput): Promise<KGTriple> {
+    const [result] = await this.createTriples([triple]);
+    return result;
   }
 
-  async createTriples(triples: Array<Omit<KGTriple, "id" | "timestamp">>): Promise<KGTriple[]> {
+  /**
+   * Upsert triples with mention counting (Memori-style: a repeated fact does
+   * not create a new row — its mention_count grows and timestamp refreshes).
+   *
+   * Counting semantics:
+   * - mention_count = number of DISTINCT source memories that contributed the triple
+   * - the same source memory re-contributing (e.g. memory re-edit) refreshes
+   *   timestamp and confidence (max) but does NOT increment the count
+   * - first_seen is preserved from the first extraction
+   *
+   * Concurrency: read-modify-write — two processes racing on the same triple id
+   * may lose one count increment (accepted: the count is a best-effort signal).
+   * Row loss is NOT possible: the final write is an atomic mergeInsert, the same
+   * primitive the main memories table adopted in the 2026-07-02 surgery.
+   */
+  async createTriples(triples: KGTripleInput[]): Promise<KGTriple[]> {
     if (triples.length === 0) return [];
     await this.ensureInitialized();
+    return this.withWriteLock(() => this.createTriplesLocked(triples));
+  }
 
+  private async createTriplesLocked(triples: KGTripleInput[]): Promise<KGTriple[]> {
     const now = Date.now();
-    const fullTriples: KGTriple[] = triples.map((t) => ({
-      ...t,
-      id: tripleId(t.scope, t.subject, t.predicate, t.object),
-      timestamp: now,
-    }));
-
-    // Deduplicate by id (same triple in batch)
-    const seen = new Set<string>();
-    const deduped = fullTriples.filter((t) => {
-      if (seen.has(t.id)) return false;
-      seen.add(t.id);
-      return true;
-    });
 
-    // Batch delete existing then add
-    const ids = deduped.map((t) => `'${escapeSql(t.id)}'`).join(", ");
-    try {
-      await this.table!.delete(`id IN (${ids})`);
-    } catch {
-      // OK if none found
+    // 1. Aggregate within batch: same triple id → union of contributing sources
+    const byId = new Map<string, { input: KGTripleInput; sources: string[]; confidence: number }>();
+    for (const t of triples) {
+      const id = tripleId(t.scope, t.subject, t.predicate, t.object);
+      const agg = byId.get(id);
+      if (agg) {
+        if (!agg.sources.includes(t.source_memory_id)) agg.sources.push(t.source_memory_id);
+        agg.confidence = Math.max(agg.confidence, t.confidence);
+      } else {
+        byId.set(id, { input: t, sources: [t.source_memory_id], confidence: t.confidence });
+      }
     }
-    await this.table!.add(deduped);
-    return deduped;
+
+    // 2. Fetch existing rows for these ids
+    const idList = [...byId.keys()].map((id) => `'${escapeSql(id)}'`).join(", ");
+    const existingRows = await this.table!.query().where(`id IN (${idList})`).limit(byId.size).toArray();
+    const existingById = new Map<string, KGTriple>(
+      existingRows.map((r) => [r.id as string, rowToTriple(r)]),
+    );
+
+    // 3. Merge counts in memory
+    const toWrite: KGTriple[] = [];
+    for (const [id, agg] of byId) {
+      const prev = existingById.get(id);
+      if (prev) {
+        const knownIds = parseSourceIds(prev.source_memory_ids, prev.source_memory_id);
+        const knownSet = new Set(knownIds);
+        let count = prev.mention_count;
+        for (const s of agg.sources) {
+          if (knownSet.has(s)) continue;
+          count++;
+          knownSet.add(s);
+          if (knownIds.length < SOURCE_IDS_CAP) knownIds.push(s);
+        }
+        toWrite.push({
+          ...prev,
+          confidence: Math.max(prev.confidence, agg.confidence),
+          timestamp: now,
+          mention_count: count,
+          source_memory_ids: JSON.stringify(knownIds),
+        });
+      } else {
+        toWrite.push({
+          ...agg.input,
+          id,
+          confidence: agg.confidence,
+          source_memory_id: agg.sources[0],
+          timestamp: now,
+          mention_count: agg.sources.length,
+          first_seen: now,
+          source_memory_ids: JSON.stringify(agg.sources.slice(0, SOURCE_IDS_CAP)),
+        });
+      }
+    }
+
+    // 4. Atomic idempotent write
+    await this.table!.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(toWrite);
+    return toWrite;
   }
 
   // --------------------------------------------------------------------------
@@ -272,6 +379,35 @@ export class KGStore {
     return results;
   }
 
+  /**
+   * Batch reverse lookup: which triples did each of these memories contribute?
+   * Used by the consolidation engine as a second merge-evidence source.
+   * A multi-source triple appears under every contributing memory id.
+   */
+  async getTriplesBySourceMemories(memoryIds: string[]): Promise<Map<string, KGTriple[]>> {
+    const result = new Map<string, KGTriple[]>();
+    if (memoryIds.length === 0) return result;
+    await this.ensureInitialized();
+
+    const conds = memoryIds.map((id) => {
+      const safe = escapeSql(id);
+      return `source_memory_id = '${safe}' OR source_memory_ids LIKE '%"${safe}"%'`;
+    });
+    const rows = await this.table!.query().where(conds.join(" OR ")).limit(100000).toArray();
+
+    for (const row of rows) {
+      const t = rowToTriple(row);
+      const contributors = parseSourceIds(t.source_memory_ids, t.source_memory_id);
+      for (const mid of memoryIds) {
+        if (!contributors.includes(mid)) continue; // LIKE prefilter false positive guard
+        const list = result.get(mid);
+        if (list) list.push(t);
+        else result.set(mid, [t]);
+      }
+    }
+    return result;
+  }
+
   // --------------------------------------------------------------------------
   // Read — Entity listing
   // --------------------------------------------------------------------------
@@ -311,10 +447,57 @@ export class KGStore {
   // Delete
   // --------------------------------------------------------------------------
 
-  /** Delete all triples extracted from a given source memory */
+  /**
+   * Remove a source memory's contribution from the KG (forget path).
+   *
+   * Multi-source semantics: a triple evidenced by OTHER memories survives with
+   * its count decremented — forgetting one memory must not erase a fact that
+   * other memories still prove. Only when the last evidence is removed does the
+   * row get deleted. If the forgotten memory supplied the stored source_text
+   * snippet, the snippet is cleared (no forgotten-content residue).
+   */
   async deleteBySource(sourceMemoryId: string): Promise<void> {
     await this.ensureInitialized();
-    await this.table!.delete(`source_memory_id = '${escapeSql(sourceMemoryId)}'`);
+    const safe = escapeSql(sourceMemoryId);
+
+    // LIKE on the JSON array is an exact match thanks to the quotes:
+    // '%"mem-1"%' never matches "mem-12". source_memory_id covers legacy rows.
+    const rows = await this.table!.query()
+      .where(`source_memory_id = '${safe}' OR source_memory_ids LIKE '%"${safe}"%'`)
+      .limit(100000)
+      .toArray();
+    if (rows.length === 0) return;
+
+    const toDelete: string[] = [];
+    const toUpdate: KGTriple[] = [];
+    for (const row of rows) {
+      const t = rowToTriple(row);
+      const ids = parseSourceIds(t.source_memory_ids, t.source_memory_id);
+      if (!ids.includes(sourceMemoryId)) continue; // LIKE prefilter false positive — exact check in JS
+      const remaining = ids.filter((s) => s !== sourceMemoryId);
+      if (remaining.length === 0) {
+        toDelete.push(t.id);
+      } else {
+        toUpdate.push({
+          ...t,
+          // count never drops below the remaining evidence count (cap-era divergence guard)
+          mention_count: Math.max(t.mention_count - 1, remaining.length),
+          source_memory_id: remaining[0],
+          source_memory_ids: JSON.stringify(remaining),
+          source_text: t.source_memory_id === sourceMemoryId ? "" : t.source_text,
+        });
+      }
+    }
+
+    await this.withWriteLock(async () => {
+      if (toUpdate.length > 0) {
+        await this.table!.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(toUpdate);
+      }
+      if (toDelete.length > 0) {
+        const idList = toDelete.map((id) => `'${escapeSql(id)}'`).join(", ");
+        await this.table!.delete(`id IN (${idList})`);
+      }
+    });
   }
 
   /** Delete all triples in scope */
@@ -340,6 +523,7 @@ export class KGStore {
 // ============================================================================
 
 function rowToTriple(row: Record<string, unknown>): KGTriple {
+  const sourceMemoryId = row.source_memory_id as string;
   return {
     id: row.id as string,
     scope: (row.scope as string) ?? "global",
@@ -347,8 +531,16 @@ function rowToTriple(row: Record<string, unknown>): KGTriple {
     predicate: row.predicate as string,
     object: row.object as string,
     confidence: Number(row.confidence),
-    source_memory_id: row.source_memory_id as string,
+    source_memory_id: sourceMemoryId,
     source_text: row.source_text as string,
     timestamp: Number(row.timestamp),
+    // Legacy-row fallbacks (pre-mention-count schema; first_seen<=0 = migrated placeholder)
+    mention_count: row.mention_count != null ? Number(row.mention_count) : 1,
+    first_seen:
+      row.first_seen != null && Number(row.first_seen) > 0 ? Number(row.first_seen) : Number(row.timestamp),
+    source_memory_ids:
+      typeof row.source_memory_ids === "string" && row.source_memory_ids.length > 0
+        ? row.source_memory_ids
+        : JSON.stringify(sourceMemoryId ? [sourceMemoryId] : []),
   };
 }

package/src/llm-client.ts

@@ -106,12 +106,57 @@ export interface LLMConfig {
   apiKey: string;
   model: string;
   baseURL: string;
+  /** SDK-level retries. Undefined preserves the OpenAI SDK default. */
+  maxRetries?: number;
   /** Request timeout in ms (default: 15000) */
   timeoutMs?: number;
   /** Temperature (default: 0.1 for consistency) */
   temperature?: number;
 }
 
+export type DetailedChatTokenLimit =
+  | { parameter: "max_tokens"; value: number }
+  | { parameter: "max_completion_tokens"; value: number };
+
+export interface DetailedChatOptions {
+  /** Per-request temperature override. Defaults to the client configuration. */
+  temperature?: number;
+  /** OpenAI-compatible JSON object response mode. */
+  responseFormat?: { type: "json_object" };
+  /** Provider extension used by Qwen-compatible endpoints. */
+  enableThinking?: boolean;
+  /** Explicitly records both the token-limit field name and its value. */
+  tokenLimit?: DetailedChatTokenLimit;
+}
+
+export interface DetailedChatUsage {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  cachedPromptTokens?: number;
+  reasoningTokens?: number;
+}
+
+export interface DetailedChatResult {
+  content: string | null;
+  finishReason: string | null;
+  responseModel: string;
+  usage: DetailedChatUsage | null;
+}
+
+interface OpenAICompatibleLongChatRequest {
+  model: string;
+  messages: [
+    { role: "system"; content: string },
+    { role: "user"; content: string },
+  ];
+  temperature: number;
+  max_tokens?: number;
+  max_completion_tokens?: number;
+  response_format?: { type: "json_object" };
+  enable_thinking?: boolean;
+}
+
 export interface DedupDecision {
   action: "CREATE" | "MERGE" | "SKIP";
   reason: string;
@@ -190,9 +235,13 @@ export class LLMClient {
 
   constructor(config: LLMConfig, breakerConfig?: Partial<CircuitBreakerConfig>) {
     const apiKey = resolveEnvVars(config.apiKey);
+    if (config.maxRetries !== undefined && (!Number.isInteger(config.maxRetries) || config.maxRetries < 0)) {
+      throw new Error("LLM maxRetries must be a non-negative integer");
+    }
     this.client = new OpenAI({
       apiKey,
       baseURL: config.baseURL,
+      ...(config.maxRetries !== undefined ? { maxRetries: config.maxRetries } : {}),
     });
     this.model = config.model;
     this.temperature = config.temperature ?? 0.1;
@@ -696,29 +745,68 @@ export class LLMClient {
     }
   }
 
-  async chatLong(system: string, user: string, maxTokens = 2000): Promise<string | null> {
+  async chatLongDetailed(
+    system: string,
+    user: string,
+    options: DetailedChatOptions = {},
+  ): Promise<DetailedChatResult | null> {
     if (!this.breaker.canAttempt()) {
       return null;
     }
 
+    const tokenLimit = options.tokenLimit ?? { parameter: "max_tokens", value: 2000 };
+    if (!Number.isInteger(tokenLimit.value) || tokenLimit.value <= 0) {
+      throw new Error("Detailed chat token limit must be a positive integer");
+    }
+    const temperature = options.temperature ?? this.temperature;
+    if (!Number.isFinite(temperature)) {
+      throw new Error("Detailed chat temperature must be finite");
+    }
+    const request: OpenAICompatibleLongChatRequest = {
+      model: this.model,
+      messages: [
+        { role: "system", content: system },
+        { role: "user", content: user },
+      ],
+      temperature,
+    };
+    if (tokenLimit.parameter === "max_tokens") {
+      request.max_tokens = tokenLimit.value;
+    } else {
+      request.max_completion_tokens = tokenLimit.value;
+    }
+    if (options.responseFormat) request.response_format = options.responseFormat;
+    if (options.enableThinking !== undefined) request.enable_thinking = options.enableThinking;
+
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), this.timeoutMs * 2);
 
     try {
       const response = await this.client.chat.completions.create(
-        {
-          model: this.model,
-          messages: [
-            { role: "system", content: system },
-            { role: "user", content: user },
-          ],
-          temperature: this.temperature,
-          max_tokens: maxTokens,
-        },
+        request,
         { signal: controller.signal },
       );
 
-      const result = response.choices[0]?.message?.content?.trim() || null;
+      const choice = response.choices[0];
+      const usage = response.usage
+        ? {
+            promptTokens: response.usage.prompt_tokens,
+            completionTokens: response.usage.completion_tokens,
+            totalTokens: response.usage.total_tokens,
+            ...(response.usage.prompt_tokens_details?.cached_tokens !== undefined
+              ? { cachedPromptTokens: response.usage.prompt_tokens_details.cached_tokens }
+              : {}),
+            ...(response.usage.completion_tokens_details?.reasoning_tokens !== undefined
+              ? { reasoningTokens: response.usage.completion_tokens_details.reasoning_tokens }
+              : {}),
+          }
+        : null;
+      const result: DetailedChatResult = {
+        content: choice?.message?.content?.trim() || null,
+        finishReason: choice?.finish_reason ?? null,
+        responseModel: response.model,
+        usage,
+      };
       this.breaker.recordSuccess();
       return result;
     } catch (err) {
@@ -729,6 +817,13 @@ export class LLMClient {
     }
   }
 
+  async chatLong(system: string, user: string, maxTokens = 2000): Promise<string | null> {
+    const result = await this.chatLongDetailed(system, user, {
+      tokenLimit: { parameter: "max_tokens", value: maxTokens },
+    });
+    return result?.content ?? null;
+  }
+
   // --------------------------------------------------------------------------
   // Constructive Retrieval — public wrappers for reconstruction pipeline
   // --------------------------------------------------------------------------

package/src/llm-consolidation.ts

@@ -16,6 +16,7 @@ import type { LLMClient } from "./llm-client.js";
 import type { MemoryEntry } from "./store.js";
 import { createVersionGroup } from "./version-manager.js";
 import { logInfo, logWarn } from "./stderr-log.js";
+import * as envConfig from "./env-config.js";
 
 // ============================================================================
 // Configuration
@@ -25,7 +26,7 @@ import { logInfo, logWarn } from "./stderr-log.js";
  * Check if LLM-driven consolidation is enabled.
  */
 export function isLLMConsolidationEnabled(): boolean {
-  return process.env.RECALLNEST_LLM_CONSOLIDATION === "true";
+  return envConfig.llmConsolidation();
 }
 
 // ============================================================================