sqlite-local-ai-memory 0.0.0-stage → 1.4.3

package/dist/MemoryEngine.js

@@ -0,0 +1,365 @@
+/**
+ * ============================================================================
+ * FILE: src/memory/MemoryEngine.ts
+ * RESPONSIBILITY: Primary Exportable Facade Class (Library + MCP).
+ * ============================================================================
+ */
+import * as fs from "fs";
+import * as path from "path";
+import os from "os";
+import { EMBEDDING_DIM } from "./types.js";
+import { SqliteStore } from "./sqlite_store.js";
+import { CognitiveAgents } from "./cognitive_agents.js";
+/** Maximum number of recent turns retained per user in RAM. */
+const MAX_RING_BUFFER_TURNS = 10;
+const defaultTelemetryPath = path.join(os.homedir(), ".local-ai-memory", "telemetry.jsonl");
+/**
+ * KeyedMutex: Guarantees deterministic serialization of asynchronous write operations
+ * per user ID, eliminating Lost Updates and TOCTOU race conditions.
+ */
+class KeyedMutex {
+    queues = new Map();
+    async run(key, fn) {
+        const prev = this.queues.get(key) ?? Promise.resolve();
+        let release;
+        // FIX R3-1.1: store `tail` directly (not a derived .then() promise) so the
+        // identity comparison below correctly detects whether we are the last waiter.
+        const tail = new Promise((resolve) => (release = resolve));
+        this.queues.set(key, tail);
+        await prev;
+        try {
+            return await fn();
+        }
+        finally {
+            release();
+            // Only delete the entry if nobody else enqueued behind us.
+            if (this.queues.get(key) === tail)
+                this.queues.delete(key);
+        }
+    }
+}
+export class MemoryEngine {
+    store;
+    agents;
+    userId;
+    debug;
+    telemetry;
+    telemetryPath;
+    telemetryStream;
+    MAX_TRACKED_USERS = 500;
+    writeMutex = new KeyedMutex();
+    backfillInFlight = new Set();
+    /** Ephemeral per-user ring buffer of recent conversation turns (RAM only, never persisted). */
+    turnBuffers = new Map();
+    constructor(config = {}) {
+        this.userId = config.userId || "user_default";
+        this.debug = config.debug !== undefined
+            ? config.debug
+            : (process.env.MEMORY_DEBUG === "true" || process.env.DEBUG === "true");
+        this.telemetry = config.telemetry !== undefined
+            ? config.telemetry
+            : (process.env.MEMORY_TELEMETRY === "true" || process.env.TELEMETRY === "true");
+        const rawTelemetryPath = (config.telemetryPath || process.env.MEMORY_TELEMETRY_PATH || defaultTelemetryPath).trim();
+        this.telemetryPath = rawTelemetryPath === "~"
+            ? os.homedir()
+            : (rawTelemetryPath.startsWith("~/") || rawTelemetryPath.startsWith("~\\"))
+                ? path.join(os.homedir(), rawTelemetryPath.slice(2))
+                : rawTelemetryPath;
+        this.store = new SqliteStore(config.dbPath);
+        this.agents = new CognitiveAgents({ ...config, debug: this.debug }, this.store);
+        if (this.telemetry) {
+            try {
+                const targetDir = path.dirname(this.telemetryPath);
+                if (!fs.existsSync(targetDir)) {
+                    fs.mkdirSync(targetDir, { recursive: true });
+                }
+                this.telemetryStream = fs.createWriteStream(this.telemetryPath, { flags: "a" });
+                this.telemetryStream.on("error", (err) => {
+                    console.error(`⚠️ [MEMORY:TELEMETRY] Stream error, disabling telemetry: ${err.message}`);
+                    this.telemetry = false;
+                    this.telemetryStream = undefined;
+                });
+            }
+            catch (err) {
+                if (this.debug) {
+                    console.error(`⚠️ [MEMORY:TELEMETRY] Failed to initialize telemetry stream: ${err.message}`);
+                }
+            }
+        }
+    }
+    /**
+     * Appends an operational telemetry record to the JSONL log file asynchronously.
+     * Fail-safe: Any file system or serialization error is logged and swallowed so it
+     * NEVER interferes with the core user query flow.
+     */
+    recordTelemetry(payload) {
+        if (!this.telemetry || !this.telemetryStream)
+            return;
+        try {
+            const entry = JSON.stringify({
+                timestamp: new Date().toISOString(),
+                ...payload,
+            }) + "\n";
+            this.telemetryStream.write(entry);
+        }
+        catch (err) {
+            if (this.debug) {
+                console.error(`⚠️ [MEMORY:TELEMETRY] Error writing telemetry record: ${err.message}`);
+            }
+        }
+    }
+    /**
+     * Saves text or conversation input, automatically filtering noise with Gatekeeper
+     * and extracting atomic facts using the Notary compiler.
+     *
+     * @param text - Input text or conversational dialogue.
+     * @param userId - Target user identifier (defaults to configured userId).
+     * @param opts - Execution options including optional AbortSignal and subproject `source` tag.
+     */
+    async save(text, userId = this.userId, opts) {
+        return this.writeMutex.run(userId, async () => {
+            await this.agents.saveMemory(text, userId, opts?.signal, opts?.source);
+            if (this.telemetry) {
+                this.recordTelemetry({
+                    event: "SAVE",
+                    userId,
+                    source: opts?.source || "system",
+                    input_text: text,
+                });
+            }
+        });
+    }
+    /**
+     * Saves a pre-formatted fact directly into memory without running through the Notary compiler.
+     * Runs through the Semantic Arbiter to detect collisions or updates against existing state.
+     *
+     * @param fact - Direct fact string (e.g. '[TECHNICAL] Database upgraded to v2').
+     * @param userId - Target user identifier.
+     * @param opts - Execution options including optional AbortSignal and subproject `source` tag.
+     */
+    async saveFact(fact, userId = this.userId, opts) {
+        return this.writeMutex.run(userId, async () => {
+            await this.agents.injectUnifiedFact(fact, userId, [], opts?.signal, opts?.source);
+            if (this.telemetry) {
+                this.recordTelemetry({
+                    event: "SAVE_FACT",
+                    userId,
+                    source: opts?.source || "system",
+                    fact,
+                });
+            }
+        });
+    }
+    /**
+     * Dual Parallel Hybrid RAG Search (FTS5 BM25 + Int8 Scalar Quantized Cosine Similarity).
+     * Automatically invokes QUERY_EXPANDER via Smart Gate when conversational context is present.
+     */
+    async search(query, limit = 5, userId = this.userId, opts) {
+        const searchStartTime = Date.now();
+        // --- SMART GATE: Activate QUERY_EXPANDER only when conversational context exists ---
+        let effectiveQuery = query;
+        const ringTurns = this.turnBuffers.get(userId) || [];
+        const ringHistory = this.getFormattedHistory(userId);
+        const externalHistory = opts?.history || "";
+        const availableHistory = ringHistory || externalHistory;
+        let expanderInvoked = false;
+        let expanderLatencyMs = 0;
+        let expanderError = null;
+        if (availableHistory.length > 0) {
+            expanderInvoked = true;
+            const expanderStart = Date.now();
+            try {
+                effectiveQuery = await this.agents.contextualizeQuery(query, availableHistory);
+            }
+            catch (err) {
+                expanderError = err.message || String(err);
+                if (this.debug) {
+                    console.error(`⚠️ [MemoryEngine] contextualizeQuery failed, using raw query: ${err.message}`);
+                }
+                effectiveQuery = query;
+            }
+            finally {
+                expanderLatencyMs = Date.now() - expanderStart;
+            }
+        }
+        const ftsHits = this.store.searchFTS5(effectiveQuery, limit * 2, userId);
+        let queryVec = [];
+        try {
+            queryVec = await this.agents.getEmbedding(effectiveQuery, opts?.signal);
+        }
+        catch (err) {
+            if (err.name === "AbortError" || opts?.signal?.aborted)
+                throw err;
+            console.error("⚠️ [MemoryEngine] Embedding generation failed, degrading gracefully to FTS5 exact search:", err.message);
+        }
+        let vecHits = [];
+        if (queryVec.length === EMBEDDING_DIM) {
+            vecHits = this.store.searchVectorInt8(queryVec, limit * 2, userId);
+            // FIX R2-2: pass `undefined` (not opts?.signal) so backfill survives
+            // even if the MCP client cancels the originating search request.
+            if (!this.backfillInFlight.has(userId)) {
+                this.backfillInFlight.add(userId);
+                this.store.backfillMissingVectors((txt) => this.agents.getEmbedding(txt, undefined), userId)
+                    .catch((err) => console.error("⚠️ [MemoryEngine] Lazy backfill failed in background:", err.message))
+                    .finally(() => this.backfillInFlight.delete(userId));
+            }
+        }
+        const maxRankAbs = Math.max(...ftsHits.map((h) => Math.abs(h.rank ?? 0)), 1);
+        const combinedMap = new Map();
+        for (const h of ftsHits) {
+            const normalizedFtsScore = h.rank != null ? Math.abs(h.rank) / maxRankAbs : 0.5;
+            combinedMap.set(h.id, { ...h, score: normalizedFtsScore * 0.8 });
+        }
+        for (const h of vecHits) {
+            if (combinedMap.has(h.id)) {
+                const prev = combinedMap.get(h.id);
+                combinedMap.set(h.id, {
+                    ...prev,
+                    score: Math.max(prev.score || 0, (h.score || 0) * 0.9) + 0.1,
+                });
+            }
+            else {
+                combinedMap.set(h.id, { ...h, score: (h.score || 0) * 0.9 });
+            }
+        }
+        const unified = Array.from(combinedMap.values());
+        unified.sort((a, b) => (b.score || 0) - (a.score || 0));
+        const results = unified.slice(0, limit);
+        // Deep Telemetry logging
+        if (this.telemetry) {
+            this.recordTelemetry({
+                event: "SEARCH",
+                userId,
+                raw_query: query,
+                effective_query: effectiveQuery,
+                smart_gate: {
+                    expander_invoked: expanderInvoked,
+                    ring_turns_count: ringTurns.length,
+                    ring_history_snapshot: ringTurns.map(t => ({ role: t.role, text: t.text })),
+                    external_history_provided: Boolean(externalHistory),
+                },
+                expander: {
+                    latency_ms: expanderLatencyMs,
+                    error: expanderError,
+                    disambiguated: effectiveQuery !== query,
+                },
+                retrieval: {
+                    total_duration_ms: Date.now() - searchStartTime,
+                    raw_fts_matches: ftsHits.length,
+                    raw_vec_matches: vecHits.length,
+                    returned_hits: results.map(r => ({
+                        id: r.id,
+                        score: Number((r.score ?? 0).toFixed(3)),
+                        data: r.data.length > 120 ? `${r.data.slice(0, 120)}...` : r.data,
+                        source: r.source,
+                    })),
+                },
+            });
+        }
+        if (this.debug) {
+            console.error(`\n🔍 [MEMORY:SEARCH]`);
+            console.error(`├─ Query: "${query}" → Effective: "${effectiveQuery}" (User: ${userId})`);
+            console.error(`├─ Raw Matches: FTS5 BM25 = ${ftsHits.length} | Vector Cosine = ${vecHits.length}`);
+            console.error(`└─ Ranked Results (Top ${results.length}):`);
+            if (results.length === 0) {
+                console.error(`   (No matching memories found)`);
+            }
+            else {
+                results.forEach((r, i) => {
+                    const scoreStr = (r.score ?? 0).toFixed(3);
+                    const snippet = r.data.length > 90 ? `${r.data.slice(0, 90)}...` : r.data;
+                    console.error(`   [${i + 1}] [Score: ${scoreStr}] ${snippet}`);
+                });
+            }
+            console.error(`────────────────────────────────────────────────────\n`);
+        }
+        return results;
+    }
+    /**
+     * Retrieves the current consolidated active working state summary (Dashboard).
+     */
+    getDashboard(userId = this.userId) {
+        const dash = this.store.getDashboardFact(userId);
+        if (this.debug) {
+            console.error(`\n📋 [MEMORY:DASHBOARD]`);
+            console.error(`├─ User: ${userId}`);
+            console.error(`├─ Updated: ${dash?.updated_at || "Never"}`);
+            console.error(`└─ Content:\n${dash?.data || "(Empty dashboard)"}`);
+            console.error(`────────────────────────────────────────────────────\n`);
+        }
+        return dash;
+    }
+    /**
+     * Executes the AutoDream cognitive pruning and state consolidation cycle.
+     * Analyzes recent facts, prunes TTL-expired working notes, graduates resolved
+     * incubator cases into long-term memory, and produces a unified executive narrative.
+     *
+     * @param userId - Target user identifier.
+     * @param source - Optional subproject or source being consolidated (e.g. 'TOTALCONNECT').
+     *                 When provided, guarantees deterministic cross-topic preservation of other projects.
+     * @returns AutoDreamResult containing executive narrative, active dashboard, and triage facts.
+     */
+    async consolidate(userId = this.userId, source, opts) {
+        return this.writeMutex.run(userId, async () => {
+            return await this.agents.runAutoDream(userId, source, opts?.signal);
+        });
+    }
+    /**
+     * Registers a conversation turn in the ephemeral ring buffer.
+     * This provides the QUERY_EXPANDER with recent conversational context
+     * to resolve ambiguous queries (pronouns, references, ellipsis).
+     * Turns are stored in RAM only and never written to SQLite.
+     *
+     * @param role - "user" or "assistant"
+     * @param text - The message content
+     * @param userId - Target user identifier (defaults to configured userId)
+     */
+    addTurn(role, text, userId = this.userId) {
+        let buffer = this.turnBuffers.get(userId);
+        if (buffer) {
+            this.turnBuffers.delete(userId);
+        }
+        else {
+            if (this.turnBuffers.size >= this.MAX_TRACKED_USERS) {
+                const oldestKey = this.turnBuffers.keys().next().value;
+                if (oldestKey !== undefined)
+                    this.turnBuffers.delete(oldestKey);
+            }
+            buffer = [];
+        }
+        buffer.push({ role, text });
+        if (buffer.length > MAX_RING_BUFFER_TURNS) {
+            buffer.splice(0, buffer.length - MAX_RING_BUFFER_TURNS);
+        }
+        this.turnBuffers.set(userId, buffer);
+    }
+    /**
+     * Returns the last N turns formatted as "U: ...\nA: ..." for the QUERY_EXPANDER.
+     * Returns empty string if no turns are available.
+     */
+    getFormattedHistory(userId, maxTurns = 5) {
+        const buffer = this.turnBuffers.get(userId);
+        if (!buffer || buffer.length === 0)
+            return "";
+        const recent = buffer.slice(-maxTurns);
+        return recent
+            .map((t) => `${t.role === "user" ? "U" : "A"}: ${t.text}`)
+            .join("\n");
+    }
+    /**
+     * @deprecated Use addTurn() + search() instead. The smart gate inside search()
+     * now calls contextualizeQuery automatically when turns are available.
+     */
+    async contextualizeQuery(query, shortHistory) {
+        return await this.agents.contextualizeQuery(query, shortHistory);
+    }
+    /**
+     * Gracefully closes the SQLite database connection.
+     */
+    close() {
+        this.store.close();
+        if (this.telemetryStream) {
+            this.telemetryStream.end();
+        }
+    }
+}

package/dist/cognitive_agents.d.ts

@@ -0,0 +1,84 @@
+import { type MemoryConfig, type AgentModelConfig, type AutoDreamResult, type ModelProvider } from "./types.js";
+import { SqliteStore } from "./sqlite_store.js";
+export interface ParsedModelTarget {
+    provider: ModelProvider;
+    model: string;
+}
+export declare function parseModelTarget(raw?: string, defaultProvider?: ModelProvider, defaultModel?: string): ParsedModelTarget | null;
+export declare function extractJsonPayload(text: string): string;
+export interface DeterministicAnchorResult {
+    paths: string[];
+    ipsAndEndpoints: string[];
+    nodesAndEnvironments: string[];
+    dates: string[];
+    formattedText: string;
+}
+/**
+ * Extracts hard, deterministic entities (filesystem paths, IPv4/IPv6 addresses,
+ * network ports, URLs, cluster nodes/containers, and timestamps) from raw working facts
+ * to guarantee zero compression loss during AutoDream state consolidation.
+ */
+export declare function extractDeterministicAnchor(sources: string[]): DeterministicAnchorResult;
+export declare class CognitiveAgents {
+    private config;
+    private store;
+    private genAI;
+    constructor(config: MemoryConfig, store: SqliteStore);
+    private sanitizeTag;
+    private stripLeadingTag;
+    private normalizeTriageFact;
+    private getGenAIClient;
+    private getOllamaApiUrl;
+    getAgentsMatrix(): {
+        GATEKEEPER: AgentModelConfig;
+        QUERY_EXPANDER: AgentModelConfig;
+        NOTARY: AgentModelConfig;
+        STATE_ORCHESTRATOR: AgentModelConfig;
+        SEMANTIC_ARBITER: AgentModelConfig;
+        EMBEDDER: AgentModelConfig;
+    };
+    getEmbedding(text: string, signal?: AbortSignal): Promise<number[]>;
+    executeAgent(agentConfig: AgentModelConfig, systemPrompt: string, userPrompt: string, isJson?: boolean, signal?: AbortSignal): Promise<string>;
+    private getResponseSchema;
+    contextualizeQuery(originalQuery: string, shortHistory: string): Promise<string>;
+    /**
+     * Directly ingests a unified memory fact into SQLite (FTS5 + Int8 Vector).
+     * Automatically performs consistency auditing via Semantic Arbiter if enabled,
+     * deactivating superseded facts and tagging the record with its origin source.
+     *
+     * @param fact - Complete fact string to ingest.
+     * @param userId - Target user identifier.
+     * @param protectedIds - Array of newly generated UUIDs that should not be superseded in the same batch.
+     * @param signal - Optional AbortSignal.
+     * @param source - Project or origin identifier (e.g. 'TOTALCONNECT', 'system').
+     */
+    injectUnifiedFact(fact: string, userId?: string, protectedIds?: string[], signal?: AbortSignal, source?: string, skipArbiter?: boolean): Promise<void>;
+    /**
+     * Processes raw text or conversational turns through the cognitive pipeline:
+     * Layer 1: Regex chatter filtering.
+     * Layer 2: Gatekeeper noise classifier.
+     * Layer 3: Notary atomic fact compilation.
+     * Layer 4: Semantic Arbiter collision auditing and SQLite persistence with source tagging.
+     *
+     * @param text - Raw message or conversational turn.
+     * @param userId - Target user identifier.
+     * @param signal - Optional AbortSignal.
+     * @param source - Subproject or client origin label.
+     */
+    saveMemory(text: string, userId?: string, signal?: AbortSignal, source?: string): Promise<void>;
+    /**
+     * Orchestrates the AutoDream consolidation cycle:
+     * 1. Applies deterministic TTL pruning on working notes.
+     * 2. Auto-archives expired unresolved incubator cases into long-term memory.
+     * 3. Executes the State Orchestrator agent to generate executive narrative and triage cards.
+     * 4. Enforces the Deterministic Cross-Topic Preservation Guardrail: items from other
+     *    projects are guaranteed never to be dropped or overwritten.
+     * 5. Saves the updated unified dashboard singleton in SQLite under DASHBOARD_UUID.
+     *
+     * @param userId - Target user identifier.
+     * @param currentSource - Subproject or client source triggering consolidation (e.g. 'TOTALCONNECT').
+     * @returns AutoDreamResult with executive summary, active dashboard items, and triage cards.
+     */
+    runAutoDream(userId?: string, currentSource?: string, signal?: AbortSignal): Promise<AutoDreamResult>;
+}
+//# sourceMappingURL=cognitive_agents.d.ts.map

package/dist/cognitive_agents.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cognitive_agents.d.ts","sourceRoot":"","sources":["../src/cognitive_agents.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,gBAAgB,EAAE,KAAK,eAAe,EAA0D,KAAK,aAAa,EAAsC,MAAM,YAAY,CAAC;AAC5M,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIhD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,CAAC,EAAE,MAAM,EACZ,eAAe,GAAE,aAAwB,EACzC,YAAY,GAAE,MAAgC,GAC7C,iBAAiB,GAAG,IAAI,CAyB1B;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAYvD;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,oBAAoB,EAAE,MAAM,EAAE,CAAC;IAC/B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,yBAAyB,CAkDvF;AAgBD,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAyB;IACvC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,KAAK,CAAa;gBAEd,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,WAAW;IAmBpD,OAAO,CAAC,WAAW;IAMnB,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,mBAAmB;YAoCb,cAAc;IAc5B,OAAO,CAAC,eAAe;IAQhB,eAAe;;;;;;;;IA2IT,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA0EnE,YAAY,CACvB,WAAW,EAAE,gBAAgB,EAC7B,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,EAClB,MAAM,GAAE,OAAe,EACvB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,MAAM,CAAC;IAsIlB,OAAO,CAAC,iBAAiB;IAoFZ,kBAAkB,CAAC,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAsD7F;;;;;;;;;;OAUG;IACU,iBAAiB,CAC5B,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,MAA2B,EACnC,YAAY,GAAE,MAAM,EAAO,EAC3B,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,GAAE,MAA8C,EACtD,WAAW,GAAE,OAAe,GAC3B,OAAO,CAAC,IAAI,CAAC;IAoFhB;;;;;;;;;;;OAWG;IACU,UAAU,CACrB,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,MAA2B,EACnC,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,GAAE,MAA8C,GACrD,OAAO,CAAC,IAAI,CAAC;IA0EhB;;;;;;;;;;;;OAYG;IACU,YAAY,CACvB,MAAM,GAAE,MAA2B,EACnC,aAAa,CAAC,EAAE,MAAM,EACtB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,eAAe,CAAC;CA0T5B"}