rag-lite-ts 1.0.1

package/dist/types.d.ts

@@ -0,0 +1,44 @@
+export interface SearchResult {
+    text: string;
+    score: number;
+    document: {
+        id: number;
+        source: string;
+        title: string;
+    };
+}
+export interface SearchOptions {
+    top_k?: number;
+    rerank?: boolean;
+}
+export interface Chunk {
+    text: string;
+    chunk_index: number;
+}
+export interface Document {
+    source: string;
+    title: string;
+    content: string;
+}
+export interface EmbeddingResult {
+    embedding_id: string;
+    vector: Float32Array;
+}
+export interface Preprocessor {
+    appliesTo(language: string): boolean;
+    process(content: string, options: PreprocessorOptions): string;
+}
+export interface PreprocessorOptions {
+    mode: 'strip' | 'keep' | 'placeholder' | 'extract';
+    [key: string]: any;
+}
+export interface PreprocessingConfig {
+    mode: 'strict' | 'balanced' | 'rich';
+    overrides?: {
+        mdx?: 'strip' | 'keep' | 'placeholder';
+        mermaid?: 'strip' | 'extract' | 'placeholder';
+        code?: 'strip' | 'keep' | 'placeholder';
+    };
+}
+export type { DatabaseConnection, ChunkResult } from './db.js';
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE;QACR,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,YAAY,CAAC;CACtB;AAGD,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACrC,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,GAAG,MAAM,CAAC;CAChE;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,SAAS,CAAC;IACnD,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,QAAQ,GAAG,UAAU,GAAG,MAAM,CAAC;IACrC,SAAS,CAAC,EAAE;QACV,GAAG,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,aAAa,CAAC;QACvC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,GAAG,aAAa,CAAC;QAC9C,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,aAAa,CAAC;KACzC,CAAC;CACH;AAGD,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC"}

package/dist/types.js

@@ -0,0 +1,3 @@
+// Type definitions for rag-lite-ts
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,mCAAmC"}

package/dist/vector-index.d.ts

@@ -0,0 +1,64 @@
+export interface VectorIndexOptions {
+    dimensions: number;
+    maxElements: number;
+    efConstruction?: number;
+    M?: number;
+    seed?: number;
+}
+export interface SearchResult {
+    neighbors: number[];
+    distances: number[];
+}
+export declare class VectorIndex {
+    private index;
+    private hnswlib;
+    private indexPath;
+    private options;
+    private currentSize;
+    private vectorStorage;
+    constructor(indexPath: string, options: VectorIndexOptions);
+    /**
+     * Initialize the HNSW index with cosine similarity using hnswlib-wasm
+     */
+    initialize(): Promise<void>;
+    /**
+     * Load existing index from file using hnswlib-wasm
+     */
+    loadIndex(): Promise<void>;
+    /**
+     * Save index to file using JSON format (since IDBFS doesn't work in Node.js)
+     */
+    saveIndex(): Promise<void>;
+    /**
+     * Add a single vector to the HNSW index
+     */
+    addVector(embeddingId: number, vector: Float32Array): void;
+    /**
+     * Add multiple vectors to the index in batch
+     */
+    addVectors(vectors: Array<{
+        id: number;
+        vector: Float32Array;
+    }>): void;
+    /**
+     * Search for k nearest neighbors using hnswlib-wasm
+     */
+    search(queryVector: Float32Array, k?: number): SearchResult;
+    /**
+     * Get current number of vectors in the index
+     */
+    getCurrentCount(): number;
+    /**
+     * Check if index exists on disk
+     */
+    indexExists(): boolean;
+    /**
+     * Set search parameters for query time
+     */
+    setEf(ef: number): void;
+    /**
+     * Resize index to accommodate more vectors
+     */
+    resizeIndex(newMaxElements: number): void;
+}
+//# sourceMappingURL=vector-index.d.ts.map

package/dist/vector-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-index.d.ts","sourceRoot":"","sources":["../src/vector-index.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AA0BD,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,OAAO,CAAa;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,aAAa,CAAwC;gBAEjD,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB;IAU1D;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAmEjC;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAwFhC;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA6BhC;;OAEG;IACH,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,IAAI;IAmB1D;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,YAAY,CAAA;KAAE,CAAC,GAAG,IAAI;IAMtE;;OAEG;IACH,MAAM,CAAC,WAAW,EAAE,YAAY,EAAE,CAAC,GAAE,MAAU,GAAG,YAAY;IAwB9D;;OAEG;IACH,eAAe,IAAI,MAAM;IAIzB;;OAEG;IACH,WAAW,IAAI,OAAO;IAItB;;OAEG;IACH,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAkBvB;;OAEG;IACH,WAAW,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI;CAiB1C"}

package/dist/vector-index.js

@@ -0,0 +1,308 @@
+import { readFileSync, writeFileSync, existsSync } from 'fs';
+import { JSDOM } from 'jsdom';
+import { ErrorCategory, ErrorSeverity, safeExecute } from './error-handler.js';
+// Set up browser-like environment for hnswlib-wasm
+if (typeof window === 'undefined') {
+    const dom = new JSDOM('<!DOCTYPE html><html><body></body></html>', {
+        url: 'http://localhost',
+        pretendToBeVisual: true,
+        resources: 'usable'
+    });
+    // Type assertion to avoid TypeScript issues with global polyfills
+    global.window = dom.window;
+    global.document = dom.window.document;
+    global.XMLHttpRequest = dom.window.XMLHttpRequest;
+    // Disable IndexedDB to prevent hnswlib-wasm from trying to use it
+    global.indexedDB = undefined;
+    // Override indexedDB on the window object to return undefined
+    Object.defineProperty(dom.window, 'indexedDB', {
+        value: undefined,
+        writable: false,
+        configurable: true
+    });
+}
+export class VectorIndex {
+    index = null;
+    hnswlib = null;
+    indexPath;
+    options;
+    currentSize = 0;
+    vectorStorage = new Map(); // For persistence
+    constructor(indexPath, options) {
+        this.indexPath = indexPath;
+        this.options = {
+            efConstruction: 200,
+            M: 16,
+            seed: 100,
+            ...options
+        };
+    }
+    /**
+     * Initialize the HNSW index with cosine similarity using hnswlib-wasm
+     */
+    async initialize() {
+        await safeExecute(async () => {
+            // Load the hnswlib module
+            if (!this.hnswlib) {
+                // Temporarily suppress stderr output during hnswlib loading to avoid IndexedDB warnings
+                const originalStderrWrite = process.stderr.write;
+                const originalConsoleError = console.error;
+                process.stderr.write = function (chunk, encoding, callback) {
+                    const message = chunk.toString();
+                    // Suppress specific IndexedDB/IDBFS related errors and WebAssembly errors
+                    if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
+                        message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
+                        message.includes('jsFS Error') || message.includes('syncing FS') ||
+                        message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
+                        message.includes('abort') || message.includes('assert') ||
+                        message.includes('hnswlib-wasm/dist/hnswlib')) {
+                        if (callback)
+                            callback();
+                        return true;
+                    }
+                    return originalStderrWrite.call(this, chunk, encoding, callback);
+                };
+                console.error = (...args) => {
+                    const message = args.join(' ');
+                    if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
+                        message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
+                        message.includes('jsFS Error') || message.includes('syncing FS') ||
+                        message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
+                        message.includes('abort') || message.includes('assert') ||
+                        message.includes('hnswlib-wasm/dist/hnswlib')) {
+                        return;
+                    }
+                    originalConsoleError.apply(console, args);
+                };
+                try {
+                    const { loadHnswlib } = await import('hnswlib-wasm/dist/hnswlib.js');
+                    this.hnswlib = await loadHnswlib();
+                }
+                finally {
+                    // Restore original output streams
+                    process.stderr.write = originalStderrWrite;
+                    console.error = originalConsoleError;
+                }
+            }
+            // Create new HNSW index (third parameter is autoSaveFilename, but we'll handle persistence manually)
+            this.index = new this.hnswlib.HierarchicalNSW('cosine', this.options.dimensions, '');
+            this.index.initIndex(this.options.maxElements, this.options.M || 16, this.options.efConstruction || 200, this.options.seed || 100);
+            this.currentSize = 0;
+            console.log(`Initialized HNSW index with ${this.options.dimensions} dimensions using hnswlib-wasm`);
+        }, 'Vector Index Initialization', {
+            category: ErrorCategory.INDEX,
+            severity: ErrorSeverity.FATAL
+        });
+    }
+    /**
+     * Load existing index from file using hnswlib-wasm
+     */
+    async loadIndex() {
+        if (!existsSync(this.indexPath)) {
+            throw new Error(`Index file not found: ${this.indexPath}`);
+        }
+        try {
+            // Load the hnswlib module
+            if (!this.hnswlib) {
+                // Temporarily suppress stderr output during hnswlib loading to avoid IndexedDB warnings
+                const originalStderrWrite = process.stderr.write;
+                const originalConsoleError = console.error;
+                process.stderr.write = function (chunk, encoding, callback) {
+                    const message = chunk.toString();
+                    // Suppress specific IndexedDB/IDBFS related errors and WebAssembly errors
+                    if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
+                        message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
+                        message.includes('jsFS Error') || message.includes('syncing FS') ||
+                        message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
+                        message.includes('abort') || message.includes('assert') ||
+                        message.includes('hnswlib-wasm/dist/hnswlib')) {
+                        if (callback)
+                            callback();
+                        return true;
+                    }
+                    return originalStderrWrite.call(this, chunk, encoding, callback);
+                };
+                console.error = (...args) => {
+                    const message = args.join(' ');
+                    if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
+                        message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
+                        message.includes('jsFS Error') || message.includes('syncing FS') ||
+                        message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
+                        message.includes('abort') || message.includes('assert') ||
+                        message.includes('hnswlib-wasm/dist/hnswlib')) {
+                        return;
+                    }
+                    originalConsoleError.apply(console, args);
+                };
+                try {
+                    const { loadHnswlib } = await import('hnswlib-wasm/dist/hnswlib.js');
+                    this.hnswlib = await loadHnswlib();
+                }
+                finally {
+                    // Restore original output streams
+                    process.stderr.write = originalStderrWrite;
+                    console.error = originalConsoleError;
+                }
+            }
+            // Create new HNSW index (third parameter is autoSaveFilename, but we'll handle persistence manually)
+            this.index = new this.hnswlib.HierarchicalNSW('cosine', this.options.dimensions, '');
+            // Load from JSON format since IDBFS doesn't work in Node.js
+            const data = readFileSync(this.indexPath, 'utf-8');
+            const stored = JSON.parse(data);
+            // Update options from stored data
+            this.options.maxElements = stored.maxElements || this.options.maxElements;
+            this.options.M = stored.M || this.options.M;
+            this.options.efConstruction = stored.efConstruction || this.options.efConstruction;
+            this.options.seed = stored.seed || this.options.seed;
+            // Recreate the index from stored data
+            this.index.initIndex(this.options.maxElements, this.options.M || 16, this.options.efConstruction || 200, this.options.seed || 100);
+            // Clear and repopulate vector storage
+            this.vectorStorage.clear();
+            // Add all stored vectors back
+            for (const item of stored.vectors || []) {
+                const vector = new Float32Array(item.vector);
+                this.index.addPoint(vector, item.id, false);
+                this.vectorStorage.set(item.id, vector);
+            }
+            this.currentSize = stored.vectors?.length || 0;
+            console.log(`Loaded HNSW index with ${this.currentSize} vectors from ${this.indexPath}`);
+        }
+        catch (error) {
+            throw new Error(`Failed to load index from ${this.indexPath}: ${error}`);
+        }
+    }
+    /**
+     * Save index to file using JSON format (since IDBFS doesn't work in Node.js)
+     */
+    async saveIndex() {
+        if (!this.index) {
+            throw new Error('Index not initialized');
+        }
+        try {
+            // Convert stored vectors to serializable format
+            const vectors = Array.from(this.vectorStorage.entries()).map(([id, vector]) => ({
+                id,
+                vector: Array.from(vector)
+            }));
+            const stored = {
+                dimensions: this.options.dimensions,
+                maxElements: this.options.maxElements,
+                M: this.options.M || 16,
+                efConstruction: this.options.efConstruction || 200,
+                seed: this.options.seed || 100,
+                currentSize: this.currentSize,
+                vectors: vectors
+            };
+            writeFileSync(this.indexPath, JSON.stringify(stored, null, 2));
+            console.log(`Saved HNSW index with ${this.currentSize} vectors to ${this.indexPath}`);
+        }
+        catch (error) {
+            throw new Error(`Failed to save index to ${this.indexPath}: ${error}`);
+        }
+    }
+    /**
+     * Add a single vector to the HNSW index
+     */
+    addVector(embeddingId, vector) {
+        if (!this.index) {
+            throw new Error('Index not initialized');
+        }
+        if (vector.length !== this.options.dimensions) {
+            throw new Error(`Vector dimension mismatch: expected ${this.options.dimensions}, got ${vector.length}`);
+        }
+        try {
+            this.index.addPoint(vector, embeddingId, false);
+            // Store vector for persistence
+            this.vectorStorage.set(embeddingId, new Float32Array(vector));
+            this.currentSize++;
+        }
+        catch (error) {
+            throw new Error(`Failed to add vector ${embeddingId}: ${error}`);
+        }
+    }
+    /**
+     * Add multiple vectors to the index in batch
+     */
+    addVectors(vectors) {
+        for (const { id, vector } of vectors) {
+            this.addVector(id, vector);
+        }
+    }
+    /**
+     * Search for k nearest neighbors using hnswlib-wasm
+     */
+    search(queryVector, k = 5) {
+        if (!this.index) {
+            throw new Error('Index not initialized');
+        }
+        if (queryVector.length !== this.options.dimensions) {
+            throw new Error(`Query vector dimension mismatch: expected ${this.options.dimensions}, got ${queryVector.length}`);
+        }
+        if (this.currentSize === 0) {
+            return { neighbors: [], distances: [] };
+        }
+        try {
+            const result = this.index.searchKnn(queryVector, Math.min(k, this.currentSize), undefined);
+            return {
+                neighbors: result.neighbors,
+                distances: result.distances
+            };
+        }
+        catch (error) {
+            throw new Error(`Search failed: ${error}`);
+        }
+    }
+    /**
+     * Get current number of vectors in the index
+     */
+    getCurrentCount() {
+        return this.currentSize;
+    }
+    /**
+     * Check if index exists on disk
+     */
+    indexExists() {
+        return existsSync(this.indexPath);
+    }
+    /**
+     * Set search parameters for query time
+     */
+    setEf(ef) {
+        if (!this.index) {
+            throw new Error('Index not initialized');
+        }
+        try {
+            // hnswlib-wasm might not have setEf method, check if it exists
+            if (typeof this.index.setEfSearch === 'function') {
+                this.index.setEfSearch(ef);
+                console.log(`Set efSearch to ${ef}`);
+            }
+            else {
+                console.log(`setEfSearch not available in hnswlib-wasm`);
+            }
+        }
+        catch (error) {
+            console.log(`Failed to set ef: ${error}`);
+        }
+    }
+    /**
+     * Resize index to accommodate more vectors
+     */
+    resizeIndex(newMaxElements) {
+        if (!this.index) {
+            throw new Error('Index not initialized');
+        }
+        if (newMaxElements <= this.options.maxElements) {
+            throw new Error(`New max elements (${newMaxElements}) must be greater than current (${this.options.maxElements})`);
+        }
+        try {
+            this.index.resizeIndex(newMaxElements);
+            this.options.maxElements = newMaxElements;
+            console.log(`Resized index to accommodate ${newMaxElements} vectors`);
+        }
+        catch (error) {
+            throw new Error(`Failed to resize index: ${error}`);
+        }
+    }
+}
+//# sourceMappingURL=vector-index.js.map

package/dist/vector-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-index.js","sourceRoot":"","sources":["../src/vector-index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAe,aAAa,EAAE,aAAa,EAAe,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAezG,mDAAmD;AACnD,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;IAClC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,2CAA2C,EAAE;QACjE,GAAG,EAAE,kBAAkB;QACvB,iBAAiB,EAAE,IAAI;QACvB,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,kEAAkE;IACjE,MAAc,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IACnC,MAAc,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;IAC9C,MAAc,CAAC,cAAc,GAAG,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC;IAE3D,kEAAkE;IACjE,MAAc,CAAC,SAAS,GAAG,SAAS,CAAC;IAEtC,8DAA8D;IAC9D,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE;QAC7C,KAAK,EAAE,SAAS;QAChB,QAAQ,EAAE,KAAK;QACf,YAAY,EAAE,IAAI;KACnB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,WAAW;IACd,KAAK,GAAQ,IAAI,CAAC;IAClB,OAAO,GAAQ,IAAI,CAAC;IACpB,SAAS,CAAS;IAClB,OAAO,CAAqB;IAC5B,WAAW,GAAG,CAAC,CAAC;IAChB,aAAa,GAA8B,IAAI,GAAG,EAAE,CAAC,CAAC,kBAAkB;IAEhF,YAAY,SAAiB,EAAE,OAA2B;QACxD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,GAAG;YACnB,CAAC,EAAE,EAAE;YACL,IAAI,EAAE,GAAG;YACT,GAAG,OAAO;SACX,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,WAAW,CACf,KAAK,IAAI,EAAE;YACT,0BAA0B;YAC1B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClB,wFAAwF;gBACxF,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;gBACjD,MAAM,oBAAoB,GAAG,OAAO,CAAC,KAAK,CAAC;gBAE3C,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,UAAS,KAAU,EAAE,QAAc,EAAE,QAAc;oBACxE,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;oBACjC,0EAA0E;oBAC1E,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAC;wBACxE,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC9E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAChE,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBACvD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;wBAClD,IAAI,QAAQ;4BAAE,QAAQ,EAAE,CAAC;wBACzB,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,OAAO,mBAAmB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBACnE,CAAC,CAAC;gBAEF,OAAO,CAAC,KAAK,GAAG,CAAC,GAAG,IAAW,EAAE,EAAE;oBACjC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC/B,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAC;wBACxE,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC9E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAChE,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBACvD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;wBAClD,OAAO;oBACT,CAAC;oBACD,oBAAoB,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC5C,CAAC,CAAC;gBAEF,IAAI,CAAC;oBACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAC;oBACrE,IAAI,CAAC,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;gBACrC,CAAC;wBAAS,CAAC;oBACT,kCAAkC;oBAClC,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,mBAAmB,CAAC;oBAC3C,OAAO,CAAC,KAAK,GAAG,oBAAoB,CAAC;gBACvC,CAAC;YACH,CAAC;YAED,qGAAqG;YACrG,IAAI,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACrF,IAAI,CAAC,KAAK,CAAC,SAAS,CAClB,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EACpB,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,GAAG,EAClC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG,CACzB,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,CAAC,OAAO,CAAC,UAAU,gCAAgC,CAAC,CAAC;QACtG,CAAC,EACD,6BAA6B,EAC7B;YACE,QAAQ,EAAE,aAAa,CAAC,KAAK;YAC7B,QAAQ,EAAE,aAAa,CAAC,KAAK;SAC9B,CACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC;YACH,0BAA0B;YAC1B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClB,wFAAwF;gBACxF,MAAM,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;gBACjD,MAAM,oBAAoB,GAAG,OAAO,CAAC,KAAK,CAAC;gBAE3C,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,UAAS,KAAU,EAAE,QAAc,EAAE,QAAc;oBACxE,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;oBACjC,0EAA0E;oBAC1E,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAC;wBACxE,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC9E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAChE,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBACvD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;wBAClD,IAAI,QAAQ;4BAAE,QAAQ,EAAE,CAAC;wBACzB,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,OAAO,mBAAmB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBACnE,CAAC,CAAC;gBAEF,OAAO,CAAC,KAAK,GAAG,CAAC,GAAG,IAAW,EAAE,EAAE;oBACjC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC/B,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAC;wBACxE,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC9E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAChE,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;wBAC5E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBACvD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;wBAClD,OAAO;oBACT,CAAC;oBACD,oBAAoB,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC5C,CAAC,CAAC;gBAEF,IAAI,CAAC;oBACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAC;oBACrE,IAAI,CAAC,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;gBACrC,CAAC;wBAAS,CAAC;oBACT,kCAAkC;oBAClC,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,mBAAmB,CAAC;oBAC3C,OAAO,CAAC,KAAK,GAAG,oBAAoB,CAAC;gBACvC,CAAC;YACH,CAAC;YAED,qGAAqG;YACrG,IAAI,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAErF,4DAA4D;YAC5D,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhC,kCAAkC;YAClC,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YAC1E,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5C,IAAI,CAAC,OAAO,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YACnF,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAErD,sCAAsC;YACtC,IAAI,CAAC,KAAK,CAAC,SAAS,CAClB,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EACpB,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,GAAG,EAClC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG,CACzB,CAAC;YAEF,sCAAsC;YACtC,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;YAE3B,8BAA8B;YAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC7C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAC1C,CAAC;YAED,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,WAAW,iBAAiB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC3F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,gDAAgD;YAChD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC9E,EAAE;gBACF,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;aAC3B,CAAC,CAAC,CAAC;YAEJ,MAAM,MAAM,GAAG;gBACb,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;gBACnC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW;gBACrC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;gBACvB,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,GAAG;gBAClD,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG;gBAC9B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,OAAO,EAAE,OAAO;aACjB,CAAC;YAEF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/D,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,CAAC,WAAW,eAAe,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACxF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,WAAmB,EAAE,MAAoB;QACjD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,uCAAuC,IAAI,CAAC,OAAO,CAAC,UAAU,SAAS,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YAChD,+BAA+B;YAC/B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,WAAW,KAAK,KAAK,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,OAAoD;QAC7D,KAAK,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;YACrC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,WAAyB,EAAE,IAAY,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,6CAA6C,IAAI,CAAC,OAAO,CAAC,UAAU,SAAS,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACrH,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,SAAS,CAAC,CAAC;YAC3F,OAAO;gBACL,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,EAAU;QACd,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,+DAA+D;YAC/D,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;gBACjD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,cAAsB;QAChC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,cAAc,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,qBAAqB,cAAc,mCAAmC,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC;QACrH,CAAC;QAED,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,cAAc,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,gCAAgC,cAAc,UAAU,CAAC,CAAC;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;CACF"}

package/package.json

@@ -0,0 +1,80 @@
+{
+  "name": "rag-lite-ts",
+  "version": "1.0.1",
+  "description": "Local-first TypeScript retrieval engine for semantic search over static documents",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "raglite": "./dist/cli.js",
+    "raglite-mcp": "./dist/mcp-server.js"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./mcp": {
+      "import": "./dist/mcp-server.js",
+      "types": "./dist/mcp-server.d.ts"
+    }
+  },
+  "files": [
+    "dist/**/*",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "build:test": "tsc --project tsconfig.test.json",
+    "clean": "rimraf dist",
+    "dev": "tsc --watch",
+    "test": "npm run build:test && node --test dist/tokenizer.test.js dist/chunker.test.js dist/embedder.test.js dist/vector-index.test.js dist/index-manager.test.js dist/search.test.js dist/file-processor.test.js dist/mcp-server.test.js dist/preprocess.test.js dist/config.test.js dist/preprocessors/integration.test.js",
+    "test:integration": "npm run build && npm run build:test && node --test dist/integration.test.js",
+    "test:all": "npm run test && npm run test:integration",
+    "prepublishOnly": "npm run clean && npm run build"
+  },
+  "keywords": [
+    "rag",
+    "retrieval",
+    "semantic-search",
+    "embeddings",
+    "typescript",
+    "local-first",
+    "mdx",
+    "mermaid",
+    "preprocessing",
+    "pdf",
+    "docx",
+    "document-processing"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/raglite/rag-lite-ts.git"
+  },
+  "bugs": {
+    "url": "https://github.com/raglite/rag-lite-ts/issues"
+  },
+  "homepage": "https://github.com/raglite/rag-lite-ts#readme",
+  "author": "RAG-lite TS Contributors",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^3.7.5",
+    "@modelcontextprotocol/sdk": "^1.18.2",
+    "hnswlib-wasm": "^0.8.2",
+    "jsdom": "^27.0.0",
+    "mammoth": "^1.11.0",
+    "pdf-parse": "^2.1.10",
+    "sqlite3": "^5.1.6"
+  },
+  "devDependencies": {
+    "@types/jsdom": "^21.1.7",
+    "@types/node": "^20.11.0",
+    "js-yaml": "^4.1.0",
+    "rimraf": "^5.0.5",
+    "typescript": "^5.3.0"
+  }
+}