sigma-memory 0.1.2 → 0.2.0

package/dist/vector-store.js

@@ -0,0 +1,322 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VectorStore = void 0;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const sql_js_1 = __importDefault(require("sql.js"));
+/**
+ * Helper for loading ESM-only modules from a CJS context.
+ * Uses native import() via Function constructor to bypass
+ * TypeScript's CJS transformation of dynamic imports.
+ */
+const esmImport = new Function('specifier', 'return import(specifier)');
+/**
+ * Self-contained vector store using sql.js (SQLite via WebAssembly) and
+ * @huggingface/transformers for local embeddings.
+ *
+ * Zero configuration — works out of the box on any platform.
+ * No native compilation required.
+ */
+class VectorStore {
+    db = null;
+    pipeline = null;
+    dbPath;
+    initialized = false;
+    initPromise = null;
+    modelPromise = null;
+    constructor(dbPath) {
+        this.dbPath = dbPath;
+    }
+    /**
+     * Initialize the vector store: sets up the SQLite database.
+     * The embedding model is loaded lazily on first embed operation.
+     * Safe to call multiple times — only initializes once.
+     */
+    async init() {
+        if (this.initialized)
+            return;
+        if (this.initPromise)
+            return this.initPromise;
+        this.initPromise = this._init();
+        await this.initPromise;
+    }
+    async _init() {
+        // Ensure directory exists
+        const dir = (0, path_1.dirname)(this.dbPath);
+        if (!(0, fs_1.existsSync)(dir)) {
+            (0, fs_1.mkdirSync)(dir, { recursive: true });
+        }
+        // Initialize sql.js (loads WASM automatically)
+        const SQL = await (0, sql_js_1.default)();
+        // Load existing DB or create a new one
+        if ((0, fs_1.existsSync)(this.dbPath)) {
+            const fileBuffer = (0, fs_1.readFileSync)(this.dbPath);
+            this.db = new SQL.Database(fileBuffer);
+        }
+        else {
+            this.db = new SQL.Database();
+        }
+        // Create schema
+        this.db.run(`
+      CREATE TABLE IF NOT EXISTS documents (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        file TEXT NOT NULL,
+        chunk_index INTEGER NOT NULL,
+        content TEXT NOT NULL,
+        embedding BLOB NOT NULL,
+        updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
+        UNIQUE(file, chunk_index)
+      )
+    `);
+        // Index for fast file lookups
+        this.db.run('CREATE INDEX IF NOT EXISTS idx_documents_file ON documents(file)');
+        this.persist();
+        this.initialized = true;
+    }
+    /**
+     * Load the embedding model. Called lazily on first embed operation.
+     * Downloads the model on first use (~23MB for all-MiniLM-L6-v2).
+     */
+    async loadEmbeddingModel() {
+        if (this.pipeline)
+            return;
+        if (this.modelPromise)
+            return this.modelPromise;
+        this.modelPromise = (async () => {
+            console.log('[VectorStore] Loading embedding model (Xenova/all-MiniLM-L6-v2)...');
+            console.log('[VectorStore] First run may download the model (~23MB).');
+            // Dynamic ESM import for @huggingface/transformers (ESM-only package)
+            const { pipeline: createPipeline } = await esmImport('@huggingface/transformers');
+            this.pipeline = await createPipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+            console.log('[VectorStore] Embedding model loaded.');
+        })();
+        await this.modelPromise;
+    }
+    /**
+     * Ensure the embedding model is loaded before use.
+     */
+    async ensurePipeline() {
+        if (!this.pipeline) {
+            await this.loadEmbeddingModel();
+        }
+    }
+    /**
+     * Persist the in-memory SQLite database to disk.
+     */
+    persist() {
+        if (!this.db)
+            return;
+        const data = this.db.export();
+        const buffer = Buffer.from(data);
+        (0, fs_1.writeFileSync)(this.dbPath, buffer);
+    }
+    /**
+     * Embed text into a Float32Array vector (384 dimensions).
+     */
+    async embed(text) {
+        await this.ensurePipeline();
+        const output = await this.pipeline(text, { pooling: 'mean', normalize: true });
+        return new Float32Array(output.data);
+    }
+    /**
+     * Chunk text into overlapping segments.
+     *
+     * Strategy:
+     * 1. Split by paragraphs (double newline)
+     * 2. If a paragraph exceeds 500 chars, split by sentences
+     * 3. Each chunk gets a 100-char overlap with the previous chunk
+     */
+    chunkText(text) {
+        const MAX_CHUNK_SIZE = 500;
+        const OVERLAP = 100;
+        // Split by paragraphs (double newline)
+        const paragraphs = text.split(/\n\s*\n/).filter(p => p.trim().length > 0);
+        const rawChunks = [];
+        for (const paragraph of paragraphs) {
+            const trimmed = paragraph.trim();
+            if (trimmed.length <= MAX_CHUNK_SIZE) {
+                rawChunks.push(trimmed);
+            }
+            else {
+                // Split long paragraphs by sentences
+                const sentences = trimmed.split(/(?<=[.!?])\s+/);
+                let current = '';
+                for (const sentence of sentences) {
+                    if (current.length + sentence.length + 1 > MAX_CHUNK_SIZE && current.length > 0) {
+                        rawChunks.push(current.trim());
+                        current = sentence;
+                    }
+                    else {
+                        current = current ? current + ' ' + sentence : sentence;
+                    }
+                }
+                if (current.trim().length > 0) {
+                    rawChunks.push(current.trim());
+                }
+            }
+        }
+        if (rawChunks.length === 0)
+            return [];
+        // Apply overlap: each chunk (except the first) gets the last 100 chars
+        // of the previous chunk prepended
+        const chunks = [rawChunks[0]];
+        for (let i = 1; i < rawChunks.length; i++) {
+            const prevChunk = rawChunks[i - 1];
+            const overlap = prevChunk.slice(-OVERLAP);
+            chunks.push(overlap + ' ' + rawChunks[i]);
+        }
+        return chunks;
+    }
+    /**
+     * Serialize a Float32Array to a Buffer for BLOB storage.
+     */
+    serializeEmbedding(embedding) {
+        return new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength);
+    }
+    /**
+     * Deserialize a BLOB (Uint8Array) back to Float32Array.
+     */
+    deserializeEmbedding(blob) {
+        // Create a proper copy to ensure alignment
+        const buffer = new ArrayBuffer(blob.byteLength);
+        new Uint8Array(buffer).set(blob);
+        return new Float32Array(buffer);
+    }
+    /**
+     * Compute cosine similarity between two vectors.
+     * Both vectors should be normalized (which they are from the model),
+     * so this is equivalent to the dot product.
+     */
+    cosineSimilarity(a, b) {
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+        if (denominator === 0)
+            return 0;
+        return dotProduct / denominator;
+    }
+    /**
+     * Add a document to the vector store.
+     * Chunks the content, embeds each chunk, and stores in the DB.
+     * Replaces any existing chunks for this file.
+     */
+    async addDocument(file, content) {
+        if (!this.db)
+            throw new Error('VectorStore not initialized. Call init() first.');
+        const chunks = this.chunkText(content);
+        if (chunks.length === 0)
+            return;
+        // Remove existing chunks for this file
+        this.db.run('DELETE FROM documents WHERE file = ?', [file]);
+        // Embed and insert each chunk
+        for (let i = 0; i < chunks.length; i++) {
+            const embedding = await this.embed(chunks[i]);
+            const embeddingBlob = this.serializeEmbedding(embedding);
+            const now = new Date().toISOString();
+            this.db.run('INSERT INTO documents (file, chunk_index, content, embedding, updated_at) VALUES (?, ?, ?, ?, ?)', [file, i, chunks[i], embeddingBlob, now]);
+        }
+        this.persist();
+    }
+    /**
+     * Search the vector store for content similar to the query.
+     * Returns top-k results sorted by cosine similarity (descending).
+     *
+     * For performance with large DBs (>10K chunks), embeddings are loaded
+     * as Float32Array and compared using optimized JS computation.
+     */
+    async search(query, limit = 10) {
+        if (!this.db)
+            throw new Error('VectorStore not initialized. Call init() first.');
+        // Embed the query
+        const queryEmbedding = await this.embed(query);
+        // Load all documents with embeddings
+        const results = this.db.exec('SELECT file, chunk_index, content, embedding FROM documents');
+        if (results.length === 0 || results[0].values.length === 0) {
+            return [];
+        }
+        // Compute cosine similarity for each document
+        const scored = [];
+        for (const row of results[0].values) {
+            const [file, chunkIndex, content, embeddingBlob] = row;
+            const docEmbedding = this.deserializeEmbedding(embeddingBlob instanceof Uint8Array ? embeddingBlob : new Uint8Array(embeddingBlob));
+            const score = this.cosineSimilarity(queryEmbedding, docEmbedding);
+            scored.push({
+                file: file,
+                chunkIndex: chunkIndex,
+                content: content,
+                score
+            });
+        }
+        // Sort by score descending and return top-k
+        scored.sort((a, b) => b.score - a.score);
+        return scored.slice(0, limit);
+    }
+    /**
+     * Remove all chunks for a given file.
+     */
+    async removeDocument(file) {
+        if (!this.db)
+            throw new Error('VectorStore not initialized. Call init() first.');
+        this.db.run('DELETE FROM documents WHERE file = ?', [file]);
+        this.persist();
+    }
+    /**
+     * Full reindex from a file map (filename → content).
+     * Clears all existing data and re-indexes everything.
+     */
+    async reindex(files) {
+        if (!this.db)
+            throw new Error('VectorStore not initialized. Call init() first.');
+        // Clear all existing documents
+        this.db.run('DELETE FROM documents');
+        // Re-add all files
+        for (const [file, content] of files) {
+            await this.addDocument(file, content);
+        }
+        this.persist();
+    }
+    /**
+     * Get statistics about the vector store.
+     */
+    getStats() {
+        if (!this.db) {
+            return { documentCount: 0, chunkCount: 0, lastUpdate: '' };
+        }
+        const countResult = this.db.exec('SELECT COUNT(DISTINCT file) as docs, COUNT(*) as chunks FROM documents');
+        const updateResult = this.db.exec('SELECT MAX(updated_at) as last_update FROM documents');
+        const docs = countResult.length > 0 && countResult[0].values.length > 0
+            ? countResult[0].values[0][0]
+            : 0;
+        const chunks = countResult.length > 0 && countResult[0].values.length > 0
+            ? countResult[0].values[0][1]
+            : 0;
+        const lastUpdate = updateResult.length > 0 && updateResult[0].values.length > 0
+            ? (updateResult[0].values[0][0] || '')
+            : '';
+        return { documentCount: docs, chunkCount: chunks, lastUpdate };
+    }
+    /**
+     * Close the database connection and persist to disk.
+     */
+    close() {
+        if (this.db) {
+            this.persist();
+            this.db.close();
+            this.db = null;
+        }
+        this.initialized = false;
+        this.initPromise = null;
+        this.modelPromise = null;
+    }
+}
+exports.VectorStore = VectorStore;
+//# sourceMappingURL=vector-store.js.map

package/dist/vector-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-store.js","sourceRoot":"","sources":["../src/vector-store.ts"],"names":[],"mappings":";;;;;;AAAA,2BAAwE;AACxE,+BAA+B;AAC/B,oDAAkD;AAGlD;;;;GAIG;AACH,MAAM,SAAS,GAAG,IAAI,QAAQ,CAC5B,WAAW,EACX,0BAA0B,CACY,CAAC;AAEzC;;;;;;GAMG;AACH,MAAa,WAAW;IACd,EAAE,GAAoB,IAAI,CAAC;IAC3B,QAAQ,GAAQ,IAAI,CAAC;IACrB,MAAM,CAAS;IACf,WAAW,GAAG,KAAK,CAAC;IACpB,WAAW,GAAyB,IAAI,CAAC;IACzC,YAAY,GAAyB,IAAI,CAAC;IAElD,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC;QAE9C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,IAAI,CAAC,WAAW,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,0BAA0B;QAC1B,MAAM,GAAG,GAAG,IAAA,cAAO,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,IAAA,eAAU,EAAC,GAAG,CAAC,EAAE,CAAC;YACrB,IAAA,cAAS,EAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,CAAC;QAED,+CAA+C;QAC/C,MAAM,GAAG,GAAG,MAAM,IAAA,gBAAS,GAAE,CAAC;QAE9B,uCAAuC;QACvC,IAAI,IAAA,eAAU,EAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,IAAA,iBAAY,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC/B,CAAC;QAED,gBAAgB;QAChB,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC;;;;;;;;;;KAUX,CAAC,CAAC;QAEH,8BAA8B;QAC9B,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;QAEhF,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB;QAC9B,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC;QAEhD,IAAI,CAAC,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC9B,OAAO,CAAC,GAAG,CAAC,oEAAoE,CAAC,CAAC;YAClF,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;YAEvE,sEAAsE;YACtE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,MAAM,SAAS,CAAC,2BAA2B,CAAC,CAAC;YAElF,IAAI,CAAC,QAAQ,GAAG,MAAM,cAAc,CAClC,oBAAoB,EACpB,yBAAyB,CAC1B,CAAC;YAEF,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACvD,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,IAAI,CAAC,YAAY,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QAC1B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,OAAO;QACb,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QACrB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAA,kBAAa,EAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,KAAK,CAAC,IAAY;QAC9B,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/E,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAED;;;;;;;OAOG;IACK,SAAS,CAAC,IAAY;QAC5B,MAAM,cAAc,GAAG,GAAG,CAAC;QAC3B,MAAM,OAAO,GAAG,GAAG,CAAC;QAEpB,uCAAuC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE1E,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YAEjC,IAAI,OAAO,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;gBACrC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,qCAAqC;gBACrC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBACjD,IAAI,OAAO,GAAG,EAAE,CAAC;gBAEjB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,cAAc,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChF,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;wBAC/B,OAAO,GAAG,QAAQ,CAAC;oBACrB,CAAC;yBAAM,CAAC;wBACN,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;oBAC1D,CAAC;gBACH,CAAC;gBAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,uEAAuE;QACvE,kCAAkC;QAClC,MAAM,MAAM,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACnC,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,SAAuB;QAChD,OAAO,IAAI,UAAU,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;IACtF,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,IAAgB;QAC3C,2CAA2C;QAC3C,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACK,gBAAgB,CAAC,CAAe,EAAE,CAAe;QACvD,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,WAAW,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEhC,OAAO,UAAU,GAAG,WAAW,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,IAAY,EAAE,OAAe;QAC7C,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAEjF,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,uCAAuC;QACvC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,sCAAsC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAE5D,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAErC,IAAI,CAAC,EAAE,CAAC,GAAG,CACT,kGAAkG,EAClG,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,aAAoB,EAAE,GAAG,CAAC,CAChD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,QAAgB,EAAE;QAC5C,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAEjF,kBAAkB;QAClB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE/C,qCAAqC;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,IAAI,CAC1B,6DAA6D,CAC9D,CAAC;QAEF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3D,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,8CAA8C;QAC9C,MAAM,MAAM,GAAyB,EAAE,CAAC;QAExC,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,CAAC,GAAG,GAA2C,CAAC;YAE/F,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAC5C,aAAa,YAAY,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,aAAoB,CAAC,CAC3F,CAAC;YAEF,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;YAElE,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,IAAc;gBACpB,UAAU,EAAE,UAAoB;gBAChC,OAAO,EAAE,OAAiB;gBAC1B,KAAK;aACN,CAAC,CAAC;QACL,CAAC;QAED,4CAA4C;QAC5C,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAAY;QAC/B,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAEjF,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,sCAAsC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,KAA0B;QACtC,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QAErC,mBAAmB;QACnB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC;YACpC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;QAC7D,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,IAAI,CAC9B,wEAAwE,CACzE,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,IAAI,CAC/B,sDAAsD,CACvD,CAAC;QAEF,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;YACrE,CAAC,CAAE,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAY;YACzC,CAAC,CAAC,CAAC,CAAC;QAEN,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;YACvE,CAAC,CAAE,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAY;YACzC,CAAC,CAAC,CAAC,CAAC;QAEN,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;YAC7E,CAAC,CAAC,CAAE,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAY,IAAI,EAAE,CAAC;YAClD,CAAC,CAAC,EAAE,CAAC;QAEP,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;CACF;AAzWD,kCAyWC"}

package/package.json

@@ -1,17 +1,20 @@
 {
 	"name": "sigma-memory",
-	"version": "0.1.2",
-	"description": "Persistent memory for Phi Code \u2014 QMD vector search + Ontology graph + Markdown notes",
+	"version": "0.2.0",
+	"description": "Persistent memory for Phi Code — embedded vector search + Ontology graph + Markdown notes",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
 	"scripts": {
 		"build": "tsc",
 		"clean": "rm -rf dist"
 	},
-	"dependencies": {},
+	"dependencies": {
+		"@huggingface/transformers": "^3.8.1",
+		"sql.js": "^1.14.1"
+	},
 	"devDependencies": {
-		"typescript": "^5.4.0",
-		"@types/node": "^20.0.0"
+		"@types/node": "^20.0.0",
+		"typescript": "^5.4.0"
 	},
 	"repository": {
 		"type": "git",
@@ -21,6 +24,7 @@
 	"keywords": [
 		"memory",
 		"vector-search",
+		"embeddings",
 		"ontology",
 		"phi-code"
 	],
@@ -30,4 +34,4 @@
 		"src",
 		"README.md"
 	]
-}
+}

package/src/index.ts

@@ -3,13 +3,13 @@ import { homedir } from 'os';
 import { existsSync, mkdirSync } from 'fs';
 import { NotesManager } from './notes.js';
 import { OntologyManager } from './ontology.js';
-import { QMDManager } from './qmd.js';
+import { VectorStore } from './vector-store.js';
 import type { MemoryConfig, UnifiedSearchResult, MemoryStatus } from './types.js';
 
 export class SigmaMemory {
   public readonly notes: NotesManager;
   public readonly ontology: OntologyManager;
-  public readonly qmd: QMDManager;
+  public readonly vectors: VectorStore;
   private readonly config: MemoryConfig;
 
   constructor(config?: Partial<MemoryConfig>) {
@@ -17,9 +17,7 @@ export class SigmaMemory {
     const defaultConfig: MemoryConfig = {
       memoryDir: join(homedir(), '.phi', 'memory'),
       projectMemoryDir: join(process.cwd(), '.phi', 'memory'),
-      ontologyPath: join(homedir(), '.phi', 'memory', 'ontology', 'graph.jsonl'),
-      qmdEnabled: true,
-      qmdCommand: 'qmd'
+      ontologyPath: join(homedir(), '.phi', 'memory', 'ontology', 'graph.jsonl')
     };
 
     this.config = { ...defaultConfig, ...config };
@@ -27,23 +25,23 @@ export class SigmaMemory {
     // Initialize managers
     this.notes = new NotesManager(this.config);
     this.ontology = new OntologyManager(this.config);
-    this.qmd = new QMDManager(this.config);
+    this.vectors = new VectorStore(join(this.config.memoryDir, 'vectors.db'));
   }
 
   /**
-   * Unified search: searches notes + ontology + QMD, combines results
+   * Unified search: searches notes + ontology + vectors, combines results
    */
   async search(query: string): Promise<UnifiedSearchResult[]> {
     const results: UnifiedSearchResult[] = [];
 
-    // Search in notes
+    // Search in notes (full-text grep)
     try {
       const notesResults = this.notes.search(query);
       for (const result of notesResults) {
         results.push({
           source: 'notes',
           type: 'note',
-          score: 0.8, // Default score for notes
+          score: 0.8, // Default score for text-match notes
           data: result
         });
       }
@@ -77,19 +75,19 @@ export class SigmaMemory {
       // Ontology search failed silently
     }
 
-    // QMD vector search
+    // Vector similarity search
     try {
-      const qmdResults = await this.qmd.search(query, 5);
-      for (const result of qmdResults) {
+      const vectorResults = await this.vectors.search(query, 5);
+      for (const result of vectorResults) {
         results.push({
-          source: 'qmd',
+          source: 'vectors',
           type: 'file',
           score: result.score,
           data: result
         });
       }
     } catch (error) {
-      // QMD search failed silently
+      // Vector search failed silently
     }
 
     // Sort by score descending
@@ -99,7 +97,8 @@ export class SigmaMemory {
   }
 
   /**
-   * Initialize all required directories
+   * Initialize all required directories and the vector store.
+   * On first run, this will download the embedding model and index notes.
    */
   async init(): Promise<void> {
     // Create base directories
@@ -111,12 +110,26 @@ export class SigmaMemory {
       mkdirSync(this.config.projectMemoryDir, { recursive: true });
     }
 
-    // Initialize QMD if enabled
-    if (this.config.qmdEnabled && this.qmd.isAvailable()) {
+    // Initialize vector store (DB setup only — fast)
+    await this.vectors.init();
+
+    // Auto-index existing notes into the vector store
+    await this.indexNotes();
+  }
+
+  /**
+   * Index all markdown notes into the vector store.
+   * Reads every .md file from the notes directory and adds it.
+   */
+  async indexNotes(): Promise<void> {
+    const notesList = this.notes.list();
+
+    for (const note of notesList) {
       try {
-        await this.qmd.update();
-      } catch (error) {
-        // QMD initialization failed silently
+        const content = this.notes.read(note.name);
+        await this.vectors.addDocument(note.name, content);
+      } catch {
+        // Skip files that can't be read
       }
     }
   }
@@ -143,20 +156,13 @@ export class SigmaMemory {
       relationsByType: ontologyStats.relationsByType
     };
 
-    // QMD status
-    let qmdStatus: MemoryStatus['qmd'] = { available: false };
-    if (this.config.qmdEnabled && this.qmd.isAvailable()) {
-      const status = await this.qmd.status();
-      qmdStatus = {
-        available: true,
-        status: status || { files: 0, chunks: 0, lastUpdate: null }
-      };
-    }
+    // Vector store status
+    const vectorStats = this.vectors.getStats();
 
     return {
       notes: notesStatus,
       ontology: ontologyStatus,
-      qmd: qmdStatus
+      vectors: vectorStats
     };
   }
 
@@ -171,8 +177,8 @@ export class SigmaMemory {
 // Convenient exports
 export { NotesManager } from './notes.js';
 export { OntologyManager } from './ontology.js';
-export { QMDManager } from './qmd.js';
+export { VectorStore } from './vector-store.js';
 export * from './types.js';
 
 // Default export
-export default SigmaMemory;
+export default SigmaMemory;

package/src/types.ts

@@ -1,9 +1,7 @@
 export interface MemoryConfig {
   memoryDir: string;        // ~/.phi/memory/
-  projectMemoryDir: string; // .phi/memory/ (dans le projet courant)
+  projectMemoryDir: string; // .phi/memory/ (in the current project)
   ontologyPath: string;     // ~/.phi/memory/ontology/graph.jsonl
-  qmdEnabled: boolean;
-  qmdCommand?: string;      // Chemin vers le binaire QMD
 }
 
 export interface SearchResult {
@@ -13,6 +11,13 @@ export interface SearchResult {
   score: number;
 }
 
+export interface VectorSearchResult {
+  file: string;
+  chunkIndex: number;
+  content: string;
+  score: number; // cosine similarity 0-1
+}
+
 export interface OntologyEntity {
   id: string;
   type: 'Person' | 'Project' | 'Device' | 'Account' | 'Document' | 'Service' | 'Concept';
@@ -38,7 +43,7 @@ export interface Note {
 }
 
 export interface UnifiedSearchResult {
-  source: 'notes' | 'ontology' | 'qmd';
+  source: 'notes' | 'ontology' | 'vectors';
   type?: 'entity' | 'relation' | 'note' | 'file';
   score: number;
   data: any;
@@ -56,13 +61,10 @@ export interface MemoryStatus {
     entitiesByType: Record<string, number>;
     relationsByType: Record<string, number>;
   };
-  qmd: {
-    available: boolean;
-    status?: {
-      files: number;
-      chunks: number;
-      lastUpdate: string | null;
-    };
+  vectors: {
+    documentCount: number;
+    chunkCount: number;
+    lastUpdate: string;
   };
 }
 
@@ -93,4 +95,4 @@ export interface OntologyDeleteEntry {
   deletedAt: string;
 }
 
-export type OntologyJSONLEntry = OntologyEntityEntry | OntologyRelationEntry | OntologyDeleteEntry;
+export type OntologyJSONLEntry = OntologyEntityEntry | OntologyRelationEntry | OntologyDeleteEntry;