rag-lite-ts 1.0.1

package/dist/db.js

@@ -0,0 +1,340 @@
+import sqlite3 from 'sqlite3';
+import { promisify } from 'util';
+import { handleError, ErrorSeverity, createError } from './error-handler.js';
+/**
+ * Opens a SQLite database connection with promisified methods
+ * @param dbPath - Path to the SQLite database file
+ * @returns Promise that resolves to a database connection object
+ */
+export function openDatabase(dbPath) {
+    return new Promise((resolve, reject) => {
+        const db = new sqlite3.Database(dbPath, (err) => {
+            if (err) {
+                const errorMsg = `Failed to open database at ${dbPath}: ${err.message}`;
+                // Categorize database errors for better handling
+                if (err.message.includes('ENOENT')) {
+                    handleError(createError.fileSystem(`Database file not found: ${dbPath}. It will be created automatically.`), 'Database Connection', { severity: ErrorSeverity.INFO });
+                }
+                else if (err.message.includes('EACCES') || err.message.includes('permission')) {
+                    reject(createError.database(`Permission denied accessing database: ${dbPath}. Check file permissions.`));
+                    return;
+                }
+                else if (err.message.includes('SQLITE_CORRUPT')) {
+                    reject(createError.database(`Database file is corrupted: ${dbPath}. Try running 'raglite rebuild'.`));
+                    return;
+                }
+                else {
+                    reject(createError.database(errorMsg));
+                    return;
+                }
+            }
+            // Enable foreign key constraints
+            db.run('PRAGMA foreign_keys = ON', (err) => {
+                if (err) {
+                    reject(createError.database(`Failed to enable foreign keys: ${err.message}`));
+                    return;
+                }
+                // Create promisified methods with proper context binding and error handling
+                const connection = {
+                    db,
+                    run: (sql, params) => {
+                        return new Promise((resolve, reject) => {
+                            db.run(sql, params || [], function (err) {
+                                if (err) {
+                                    // Enhance SQLite error messages
+                                    const enhancedError = enhanceSQLiteError(err, sql);
+                                    reject(enhancedError);
+                                }
+                                else {
+                                    resolve(this);
+                                }
+                            });
+                        });
+                    },
+                    get: promisify(db.get.bind(db)),
+                    all: promisify(db.all.bind(db)),
+                    close: promisify(db.close.bind(db))
+                };
+                resolve(connection);
+            });
+        });
+    });
+}
+/**
+ * Enhance SQLite error messages with more context
+ */
+function enhanceSQLiteError(error, sql) {
+    let enhancedMessage = error.message;
+    if (error.message.includes('SQLITE_BUSY')) {
+        enhancedMessage = 'Database is locked by another process. Ensure no other RAG-lite instances are running.';
+    }
+    else if (error.message.includes('SQLITE_FULL')) {
+        enhancedMessage = 'Database disk is full. Free up disk space and try again.';
+    }
+    else if (error.message.includes('SQLITE_CORRUPT')) {
+        enhancedMessage = 'Database file is corrupted. Try running "raglite rebuild" to recreate it.';
+    }
+    else if (error.message.includes('UNIQUE constraint failed')) {
+        enhancedMessage = `Duplicate entry detected: ${error.message}. This item may already exist.`;
+    }
+    else if (error.message.includes('FOREIGN KEY constraint failed')) {
+        enhancedMessage = `Foreign key constraint violation: ${error.message}. Referenced record may not exist.`;
+    }
+    if (sql && sql.length < 200) {
+        enhancedMessage += `\nSQL: ${sql}`;
+    }
+    return new Error(enhancedMessage);
+}
+/**
+ * Initializes the database schema with all required tables and indexes
+ * @param connection - Database connection object
+ */
+export async function initializeSchema(connection) {
+    try {
+        // Create documents table
+        await connection.run(`
+      CREATE TABLE IF NOT EXISTS documents (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        source TEXT NOT NULL UNIQUE,
+        title TEXT NOT NULL,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      )
+    `);
+        // Create chunks table with foreign key relationship
+        await connection.run(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        embedding_id TEXT NOT NULL UNIQUE,
+        document_id INTEGER NOT NULL,
+        text TEXT NOT NULL,
+        chunk_index INTEGER NOT NULL,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE
+      )
+    `);
+        // Create system_info table for model version tracking
+        await connection.run(`
+      CREATE TABLE IF NOT EXISTS system_info (
+        id INTEGER PRIMARY KEY CHECK (id = 1),
+        model_version TEXT NOT NULL,
+        model_name TEXT,
+        model_dimensions INTEGER,
+        updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      )
+    `);
+        // Add model tracking columns if they don't exist (migration)
+        try {
+            await connection.run(`ALTER TABLE system_info ADD COLUMN model_name TEXT`);
+        }
+        catch (error) {
+            // Column already exists, ignore error
+            if (error instanceof Error && !error.message.includes('duplicate column name')) {
+                throw error;
+            }
+        }
+        try {
+            await connection.run(`ALTER TABLE system_info ADD COLUMN model_dimensions INTEGER`);
+        }
+        catch (error) {
+            // Column already exists, ignore error
+            if (error instanceof Error && !error.message.includes('duplicate column name')) {
+                throw error;
+            }
+        }
+        // Create indexes for performance
+        await connection.run(`
+      CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)
+    `);
+        await connection.run(`
+      CREATE INDEX IF NOT EXISTS idx_chunks_embedding_id ON chunks(embedding_id)
+    `);
+        await connection.run(`
+      CREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source)
+    `);
+        console.log('Database schema initialized successfully');
+    }
+    catch (error) {
+        throw new Error(`Failed to initialize database schema: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Inserts a new document into the database
+ * @param connection - Database connection object
+ * @param source - Source path of the document
+ * @param title - Title of the document
+ * @returns Promise that resolves to the document ID
+ */
+export async function insertDocument(connection, source, title) {
+    try {
+        const result = await connection.run('INSERT INTO documents (source, title) VALUES (?, ?)', [source, title]);
+        if (typeof result.lastID !== 'number' || result.lastID <= 0) {
+            throw new Error('Failed to get document ID after insertion');
+        }
+        return result.lastID;
+    }
+    catch (error) {
+        if (error instanceof Error && error.message.includes('UNIQUE constraint failed')) {
+            throw new Error(`Document with source '${source}' already exists`);
+        }
+        throw new Error(`Failed to insert document: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Inserts or updates a chunk in the database (upsert operation)
+ * @param connection - Database connection object
+ * @param embeddingId - Unique embedding ID for the chunk
+ * @param documentId - ID of the parent document
+ * @param text - Text content of the chunk
+ * @param chunkIndex - Index of the chunk within the document
+ */
+export async function insertChunk(connection, embeddingId, documentId, text, chunkIndex) {
+    try {
+        // Use INSERT OR REPLACE to handle duplicates gracefully
+        await connection.run('INSERT OR REPLACE INTO chunks (embedding_id, document_id, text, chunk_index) VALUES (?, ?, ?, ?)', [embeddingId, documentId, text, chunkIndex]);
+    }
+    catch (error) {
+        if (error instanceof Error && error.message.includes('FOREIGN KEY constraint failed')) {
+            throw new Error(`Document with ID ${documentId} does not exist`);
+        }
+        throw new Error(`Failed to insert/update chunk: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Inserts a new document or returns existing document ID if it already exists
+ * @param connection - Database connection object
+ * @param source - Source path of the document
+ * @param title - Title of the document
+ * @returns Promise that resolves to the document ID
+ */
+export async function upsertDocument(connection, source, title) {
+    try {
+        // First try to get existing document
+        const existing = await connection.get('SELECT id FROM documents WHERE source = ?', [source]);
+        if (existing) {
+            return existing.id;
+        }
+        // Insert new document if it doesn't exist
+        const result = await connection.run('INSERT INTO documents (source, title) VALUES (?, ?)', [source, title]);
+        if (typeof result.lastID !== 'number' || result.lastID <= 0) {
+            throw new Error('Failed to get document ID after insertion');
+        }
+        return result.lastID;
+    }
+    catch (error) {
+        throw new Error(`Failed to upsert document: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Retrieves chunks by their embedding IDs with document metadata
+ * @param connection - Database connection object
+ * @param embeddingIds - Array of embedding IDs to retrieve
+ * @returns Promise that resolves to an array of chunk results with document metadata
+ */
+export async function getChunksByEmbeddingIds(connection, embeddingIds) {
+    if (embeddingIds.length === 0) {
+        return [];
+    }
+    try {
+        const placeholders = embeddingIds.map(() => '?').join(',');
+        const sql = `
+      SELECT 
+        c.id,
+        c.embedding_id,
+        c.document_id,
+        c.text,
+        c.chunk_index,
+        c.created_at,
+        d.source as document_source,
+        d.title as document_title
+      FROM chunks c
+      JOIN documents d ON c.document_id = d.id
+      WHERE c.embedding_id IN (${placeholders})
+      ORDER BY c.chunk_index
+    `;
+        const results = await connection.all(sql, embeddingIds);
+        return results;
+    }
+    catch (error) {
+        throw new Error(`Failed to retrieve chunks: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Gets the current model version from system_info table
+ * @param connection - Database connection object
+ * @returns Promise that resolves to the model version string or null if not set
+ */
+export async function getModelVersion(connection) {
+    try {
+        const result = await connection.get('SELECT model_version FROM system_info WHERE id = 1');
+        return result ? result.model_version : null;
+    }
+    catch (error) {
+        throw new Error(`Failed to get model version: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Sets the model version in system_info table
+ * @param connection - Database connection object
+ * @param modelVersion - Model version string to store
+ */
+export async function setModelVersion(connection, modelVersion) {
+    try {
+        // Check if there's already a row
+        const existing = await connection.get('SELECT model_name, model_dimensions FROM system_info WHERE id = 1');
+        if (existing) {
+            // Update only the model_version field, preserve existing model info
+            await connection.run('UPDATE system_info SET model_version = ?, updated_at = CURRENT_TIMESTAMP WHERE id = 1', [modelVersion]);
+        }
+        else {
+            // Insert new row with just model_version
+            await connection.run('INSERT INTO system_info (id, model_version, updated_at) VALUES (1, ?, CURRENT_TIMESTAMP)', [modelVersion]);
+        }
+    }
+    catch (error) {
+        throw new Error(`Failed to set model version: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Gets the stored model information from system_info table
+ * @param connection - Database connection object
+ * @returns Promise that resolves to model info object or null if not set
+ */
+export async function getStoredModelInfo(connection) {
+    try {
+        const result = await connection.get('SELECT model_name, model_dimensions FROM system_info WHERE id = 1');
+        if (!result || !result.model_name || !result.model_dimensions) {
+            return null;
+        }
+        return {
+            modelName: result.model_name,
+            dimensions: result.model_dimensions
+        };
+    }
+    catch (error) {
+        throw new Error(`Failed to get stored model info: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+/**
+ * Sets the model information in system_info table
+ * @param connection - Database connection object
+ * @param modelName - Name of the embedding model
+ * @param dimensions - Number of dimensions for the model
+ */
+export async function setStoredModelInfo(connection, modelName, dimensions) {
+    try {
+        // Check if there's already a row
+        const existing = await connection.get('SELECT model_version FROM system_info WHERE id = 1');
+        if (existing) {
+            // Update only the model info fields, preserve existing model_version
+            await connection.run('UPDATE system_info SET model_name = ?, model_dimensions = ?, updated_at = CURRENT_TIMESTAMP WHERE id = 1', [modelName, dimensions]);
+        }
+        else {
+            // Insert new row with placeholder model_version (will be updated by setModelVersion)
+            await connection.run('INSERT INTO system_info (id, model_version, model_name, model_dimensions, updated_at) VALUES (1, "", ?, ?, CURRENT_TIMESTAMP)', [modelName, dimensions]);
+        }
+    }
+    catch (error) {
+        throw new Error(`Failed to set stored model info: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+}
+//# sourceMappingURL=db.js.map

package/dist/db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,EAAE,WAAW,EAAiB,aAAa,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAsB5F;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,EAAE,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;YAC9C,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,QAAQ,GAAG,8BAA8B,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;gBAExE,iDAAiD;gBACjD,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACnC,WAAW,CACT,WAAW,CAAC,UAAU,CAAC,4BAA4B,MAAM,qCAAqC,CAAC,EAC/F,qBAAqB,EACrB,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CACjC,CAAC;gBACJ,CAAC;qBAAM,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBAChF,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,yCAAyC,MAAM,2BAA2B,CAAC,CAAC,CAAC;oBACzG,OAAO;gBACT,CAAC;qBAAM,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;oBAClD,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,+BAA+B,MAAM,kCAAkC,CAAC,CAAC,CAAC;oBACtG,OAAO;gBACT,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;oBACvC,OAAO;gBACT,CAAC;YACH,CAAC;YAED,iCAAiC;YACjC,EAAE,CAAC,GAAG,CAAC,0BAA0B,EAAE,CAAC,GAAG,EAAE,EAAE;gBACzC,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,kCAAkC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBAC9E,OAAO;gBACT,CAAC;gBAED,4EAA4E;gBAC5E,MAAM,UAAU,GAAuB;oBACrC,EAAE;oBACF,GAAG,EAAE,CAAC,GAAW,EAAE,MAAc,EAAE,EAAE;wBACnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;4BACrC,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE,EAAE,UAAS,GAAG;gCACpC,IAAI,GAAG,EAAE,CAAC;oCACR,gCAAgC;oCAChC,MAAM,aAAa,GAAG,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;oCACnD,MAAM,CAAC,aAAa,CAAC,CAAC;gCACxB,CAAC;qCAAM,CAAC;oCACN,OAAO,CAAC,IAAI,CAAC,CAAC;gCAChB,CAAC;4BACH,CAAC,CAAC,CAAC;wBACL,CAAC,CAAC,CAAC;oBACL,CAAC;oBACD,GAAG,EAAE,SAAS,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC/B,GAAG,EAAE,SAAS,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC/B,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;iBACpC,CAAC;gBAEF,OAAO,CAAC,UAAU,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,KAAY,EAAE,GAAY;IACpD,IAAI,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC;IAEpC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QAC1C,eAAe,GAAG,wFAAwF,CAAC;IAC7G,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACjD,eAAe,GAAG,0DAA0D,CAAC;IAC/E,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACpD,eAAe,GAAG,2EAA2E,CAAC;IAChG,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,CAAC;QAC9D,eAAe,GAAG,6BAA6B,KAAK,CAAC,OAAO,gCAAgC,CAAC;IAC/F,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,+BAA+B,CAAC,EAAE,CAAC;QACnE,eAAe,GAAG,qCAAqC,KAAK,CAAC,OAAO,oCAAoC,CAAC;IAC3G,CAAC;IAED,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC5B,eAAe,IAAI,UAAU,GAAG,EAAE,CAAC;IACrC,CAAC;IAED,OAAO,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;AACpC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,UAA8B;IACnE,IAAI,CAAC;QACH,yBAAyB;QACzB,MAAM,UAAU,CAAC,GAAG,CAAC;;;;;;;KAOpB,CAAC,CAAC;QAEH,oDAAoD;QACpD,MAAM,UAAU,CAAC,GAAG,CAAC;;;;;;;;;;KAUpB,CAAC,CAAC;QAEH,sDAAsD;QACtD,MAAM,UAAU,CAAC,GAAG,CAAC;;;;;;;;KAQpB,CAAC,CAAC;QAEH,6DAA6D;QAC7D,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAC7E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sCAAsC;YACtC,IAAI,KAAK,YAAY,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBAC/E,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;QACtF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sCAAsC;YACtC,IAAI,KAAK,YAAY,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBAC/E,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,MAAM,UAAU,CAAC,GAAG,CAAC;;KAEpB,CAAC,CAAC;QAEH,MAAM,UAAU,CAAC,GAAG,CAAC;;KAEpB,CAAC,CAAC;QAEH,MAAM,UAAU,CAAC,GAAG,CAAC;;KAEpB,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,yCAAyC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IACvH,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,UAA8B,EAC9B,MAAc,EACd,KAAa;IAEb,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CACjC,qDAAqD,EACrD,CAAC,MAAM,EAAE,KAAK,CAAC,CAChB,CAAC;QAEF,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,CAAC;YACjF,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,kBAAkB,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC5G,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,UAA8B,EAC9B,WAAmB,EACnB,UAAkB,EAClB,IAAY,EACZ,UAAkB;IAElB,IAAI,CAAC;QACH,wDAAwD;QACxD,MAAM,UAAU,CAAC,GAAG,CAClB,kGAAkG,EAClG,CAAC,WAAW,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,CAAC,CAC5C,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,+BAA+B,CAAC,EAAE,CAAC;YACtF,MAAM,IAAI,KAAK,CAAC,oBAAoB,UAAU,iBAAiB,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kCAAkC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAChH,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,UAA8B,EAC9B,MAAc,EACd,KAAa;IAEb,IAAI,CAAC;QACH,qCAAqC;QACrC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,GAAG,CACnC,2CAA2C,EAC3C,CAAC,MAAM,CAAC,CACT,CAAC;QAEF,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,QAAQ,CAAC,EAAE,CAAC;QACrB,CAAC;QAED,0CAA0C;QAC1C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CACjC,qDAAqD,EACrD,CAAC,MAAM,EAAE,KAAK,CAAC,CAChB,CAAC;QAEF,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC5G,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,UAA8B,EAC9B,YAAsB;IAEtB,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,GAAG,GAAG;;;;;;;;;;;;iCAYiB,YAAY;;KAExC,CAAC;QAEF,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,OAAO,OAAwB,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC5G,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,UAA8B;IAClE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAC1F,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,gCAAgC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC9G,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,UAA8B,EAAE,YAAoB;IACxF,IAAI,CAAC;QACH,iCAAiC;QACjC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;QAE3G,IAAI,QAAQ,EAAE,CAAC;YACb,oEAAoE;YACpE,MAAM,UAAU,CAAC,GAAG,CAClB,uFAAuF,EACvF,CAAC,YAAY,CAAC,CACf,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,yCAAyC;YACzC,MAAM,UAAU,CAAC,GAAG,CAClB,0FAA0F,EAC1F,CAAC,YAAY,CAAC,CACf,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,gCAAgC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAC9G,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,UAA8B;IAIrE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CACjC,mEAAmE,CACpE,CAAC;QAEF,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC9D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO;YACL,SAAS,EAAE,MAAM,CAAC,UAAU;YAC5B,UAAU,EAAE,MAAM,CAAC,gBAAgB;SACpC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,oCAAoC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAClH,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,UAA8B,EAC9B,SAAiB,EACjB,UAAkB;IAElB,IAAI,CAAC;QACH,iCAAiC;QACjC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAE5F,IAAI,QAAQ,EAAE,CAAC;YACb,qEAAqE;YACrE,MAAM,UAAU,CAAC,GAAG,CAClB,0GAA0G,EAC1G,CAAC,SAAS,EAAE,UAAU,CAAC,CACxB,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,qFAAqF;YACrF,MAAM,UAAU,CAAC,GAAG,CAClB,+HAA+H,EAC/H,CAAC,SAAS,EAAE,UAAU,CAAC,CACxB,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,oCAAoC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;IAClH,CAAC;AACH,CAAC"}

package/dist/embedder.d.ts

@@ -0,0 +1,101 @@
+import type { EmbeddingResult } from './types.js';
+/**
+ * Embedding engine using transformers.js for generating embeddings
+ */
+export declare class EmbeddingEngine {
+    private model;
+    private modelVersion;
+    private readonly modelName;
+    private readonly batchSize;
+    constructor(modelName?: string, batchSize?: number);
+    /**
+     * Load the embedding model
+     * @throws {Error} If model loading fails
+     */
+    loadModel(): Promise<void>;
+    /**
+     * Generate embeddings for a batch of texts
+     * @param texts - Array of text strings to embed
+     * @returns Promise resolving to array of embedding results
+     */
+    embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
+    /**
+     * Process a single batch with error handling for individual chunks
+     * @param batch - Array of text strings in this batch
+     * @param startIndex - Starting index for this batch in the original array
+     * @returns Promise resolving to array of embedding results
+     */
+    private processBatchWithErrorHandling;
+    /**
+     * Fallback to individual chunk processing when batch fails
+     */
+    private fallbackToIndividualProcessing;
+    /**
+     * Process a single chunk with error handling
+     * @param text - Text to embed
+     * @param index - Index of this chunk
+     * @returns Promise resolving to embedding result or null if failed
+     */
+    private processSingleChunk;
+    /**
+     * Generate embedding for a single text
+     * @param text - Text string to embed
+     * @returns Promise resolving to embedding result
+     */
+    embedSingle(text: string): Promise<EmbeddingResult>;
+    /**
+     * Generate embeddings for document chunks with progress logging
+     * Optimized for large document ingestion with batch processing
+     * @param chunks - Array of text chunks from documents
+     * @returns Promise resolving to array of embedding results
+     */
+    embedDocumentBatch(chunks: string[]): Promise<EmbeddingResult[]>;
+    /**
+     * Get the current model version identifier
+     * @returns Model version string
+     */
+    getModelVersion(): string;
+    /**
+     * Check if the model is loaded
+     * @returns True if model is loaded
+     */
+    isLoaded(): boolean;
+    /**
+     * Get the model name
+     * @returns Model name string
+     */
+    getModelName(): string;
+    /**
+     * Get the batch size
+     * @returns Batch size number
+     */
+    getBatchSize(): number;
+    /**
+     * Generate a deterministic model version identifier
+     * Uses model name and configuration for consistent versioning
+     * @returns Model version string
+     */
+    private generateModelVersion;
+    /**
+     * Generate a deterministic embedding ID for a text chunk
+     * @param text - The text content
+     * @param index - Index in the batch
+     * @returns Deterministic embedding ID
+     */
+    private generateEmbeddingId;
+}
+/**
+ * Get the singleton embedding engine instance
+ * @param modelName - Optional model name override
+ * @param batchSize - Optional batch size override
+ * @returns EmbeddingEngine instance
+ */
+export declare function getEmbeddingEngine(modelName?: string, batchSize?: number): EmbeddingEngine;
+/**
+ * Initialize the embedding engine and load the model
+ * @param modelName - Optional model name override
+ * @param batchSize - Optional batch size override
+ * @returns Promise resolving to the loaded embedding engine
+ */
+export declare function initializeEmbeddingEngine(modelName?: string, batchSize?: number): Promise<EmbeddingEngine>;
+//# sourceMappingURL=embedder.d.ts.map

package/dist/embedder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAUlD;;GAEG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAelD;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA6ChC;;;;OAIG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAqB7D;;;;;OAKG;YACW,6BAA6B;IAmC3C;;OAEG;YACW,8BAA8B;IAgC5C;;;;;OAKG;YACW,kBAAkB;IAqBhC;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAQzD;;;;;OAKG;IACG,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA+CtE;;;OAGG;IACH,eAAe,IAAI,MAAM;IAOzB;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,YAAY,IAAI,MAAM;IAItB;;;OAGG;IACH,YAAY,IAAI,MAAM;IAItB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAa5B;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;CAM5B;AAQD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,eAAe,CAS1F;AAED;;;;;GAKG;AACH,wBAAsB,yBAAyB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAQhH"}