rag-lite-ts 2.1.1 → 2.3.0

package/package.json

@@ -1,35 +1,60 @@
 {
   "name": "rag-lite-ts",
-  "version": "2.1.1",
+  "version": "2.3.0",
   "description": "Local-first TypeScript retrieval engine with Chameleon Multimodal Architecture for semantic search over text and image content",
   "type": "module",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "main": "./dist/cjs/index.js",
+  "types": "./dist/cjs/index.d.ts",
   "bin": {
-    "raglite": "./dist/cli.js",
-    "raglite-mcp": "./dist/mcp-server.js"
+    "raglite": "./dist/esm/cli.js",
+    "raglite-mcp": "./dist/esm/mcp-server.js"
   },
   "exports": {
     ".": {
-      "import": "./dist/index.js",
-      "types": "./dist/index.d.ts"
+      "types": "./dist/esm/index.d.ts",
+      "import": {
+        "types": "./dist/esm/index.d.ts",
+        "default": "./dist/esm/index.js"
+      },
+      "require": {
+        "types": "./dist/cjs/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
     },
     "./mcp": {
-      "import": "./dist/mcp-server.js",
-      "types": "./dist/mcp-server.d.ts"
+      "types": "./dist/esm/mcp-server.d.ts",
+      "import": {
+        "types": "./dist/esm/mcp-server.d.ts",
+        "default": "./dist/esm/mcp-server.js"
+      },
+      "require": {
+        "types": "./dist/cjs/mcp-server.d.ts",
+        "default": "./dist/cjs/mcp-server.js"
+      }
     }
   },
   "files": [
-    "dist/**/*.js",
-    "dist/**/*.d.ts",
+    "dist/esm/**/*.js",
+    "dist/cjs/**/*.js",
+    "dist/esm/**/*.d.ts",
+    "dist/cjs/**/*.d.ts",
     "!dist/**/*.map",
+    "ui/backend/dist/**",
+    "ui/frontend/dist/**",
+    "!ui/**/src/**",
+    "!ui/**/node_modules/**",
     "README.md",
     "LICENSE"
   ],
   "scripts": {
-    "build": "tsc",
+    "build": "npm run build:esm && npm run build:cjs",
+    "build:esm": "tsc -p tsconfig.esm.json",
+    "build:cjs": "tsc -p tsconfig.cjs.json",
     "build:test": "tsc --project tsconfig.test.json",
-    "clean": "rimraf dist",
+    "build:ui:backend": "cd ui/backend && npm run build",
+    "build:ui:frontend": "cd ui/frontend && npm run build",
+    "build:ui": "npm run build:ui:backend && npm run build:ui:frontend",
+    "clean": "rimraf dist ui/backend/dist ui/frontend/dist",
     "dev": "tsc --watch",
     "test": "npm run build:test && node --expose-gc --test --test-concurrency=1 dist/__tests__/core dist/__tests__/text dist/__tests__/preprocessors dist/__tests__/cli dist/__tests__/factories",
     "test:verbose": "npm run build:test && node --expose-gc --test --test-concurrency=1 --test-reporter=tap dist/__tests__/core dist/__tests__/text dist/__tests__/preprocessors dist/__tests__/cli dist/__tests__/factories",
@@ -45,7 +70,7 @@
     "test:integration:verbose": "npm run build && npm run build:test && node --expose-gc --test --test-concurrency=1 --test-reporter=tap dist/__tests__/integration",
     "test:all": "npm run build:test && node --expose-gc --test --test-concurrency=1 dist/__tests__",
     "test:all:verbose": "npm run build:test && node --expose-gc --test --test-concurrency=1 --test-reporter=tap dist/__tests__",
-    "prepublishOnly": "npm run clean && npm run build"
+    "prepublishOnly": "npm run clean && npm run build && npm run build:ui"
   },
   "keywords": [
     "rag",

package/dist/core/vector-index.d.ts

@@ -1,72 +0,0 @@
-/**
- * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
- * Model-agnostic. No transformer or modality-specific logic.
- */
-export interface VectorIndexOptions {
-    dimensions: number;
-    maxElements: number;
-    efConstruction?: number;
-    M?: number;
-    seed?: number;
-}
-export interface SearchResult {
-    neighbors: number[];
-    distances: number[];
-}
-export declare class VectorIndex {
-    private index;
-    private hnswlib;
-    private indexPath;
-    private options;
-    private currentSize;
-    private vectorStorage;
-    constructor(indexPath: string, options: VectorIndexOptions);
-    /**
-     * Initialize the HNSW index with cosine similarity using hnswlib-wasm
-     */
-    initialize(): Promise<void>;
-    /**
-     * Load existing index from file using hnswlib-wasm
-     */
-    loadIndex(): Promise<void>;
-    /**
-     * Save index to binary format
-     */
-    saveIndex(): Promise<void>;
-    /**
-     * Add a single vector to the HNSW index
-     */
-    addVector(embeddingId: number, vector: Float32Array): void;
-    /**
-     * Add multiple vectors to the index in batch
-     */
-    addVectors(vectors: Array<{
-        id: number;
-        vector: Float32Array;
-    }>): void;
-    /**
-     * Search for k nearest neighbors using hnswlib-wasm
-     */
-    search(queryVector: Float32Array, k?: number): SearchResult;
-    /**
-     * Get current number of vectors in the index
-     */
-    getCurrentCount(): number;
-    /**
-     * Check if index exists on disk
-     */
-    indexExists(): boolean;
-    /**
-     * Set search parameters for query time
-     */
-    setEf(ef: number): void;
-    /**
-     * Resize index to accommodate more vectors
-     */
-    resizeIndex(newMaxElements: number): void;
-    /**
-     * Get index options (for external access to configuration)
-     */
-    getOptions(): VectorIndexOptions;
-}
-//# sourceMappingURL=vector-index.d.ts.map

package/dist/core/vector-index.js

@@ -1,331 +0,0 @@
-/**
- * CORE MODULE — Shared between text-only (rag-lite-ts) and future multimodal (rag-lite-mm)
- * Model-agnostic. No transformer or modality-specific logic.
- */
-import { existsSync } from 'fs';
-import { JSDOM } from 'jsdom';
-import { ErrorCategory, ErrorSeverity, safeExecute } from './error-handler.js';
-import { createMissingFileError, createDimensionMismatchError } from './actionable-error-messages.js';
-import { BinaryIndexFormat } from './binary-index-format.js';
-// Set up browser-like environment for hnswlib-wasm
-if (typeof window === 'undefined') {
-    const dom = new JSDOM('<!DOCTYPE html><html><body></body></html>', {
-        url: 'http://localhost',
-        pretendToBeVisual: true,
-        resources: 'usable'
-    });
-    // Type assertion to avoid TypeScript issues with global polyfills
-    global.window = dom.window;
-    global.document = dom.window.document;
-    global.XMLHttpRequest = dom.window.XMLHttpRequest;
-    // Disable IndexedDB to prevent hnswlib-wasm from trying to use it
-    global.indexedDB = undefined;
-    // Override indexedDB on the window object to return undefined
-    Object.defineProperty(dom.window, 'indexedDB', {
-        value: undefined,
-        writable: false,
-        configurable: true
-    });
-}
-export class VectorIndex {
-    index = null;
-    hnswlib = null;
-    indexPath;
-    options;
-    currentSize = 0;
-    vectorStorage = new Map(); // For persistence
-    constructor(indexPath, options) {
-        this.indexPath = indexPath;
-        this.options = {
-            efConstruction: 200,
-            M: 16,
-            seed: 100,
-            ...options
-        };
-    }
-    /**
-     * Initialize the HNSW index with cosine similarity using hnswlib-wasm
-     */
-    async initialize() {
-        await safeExecute(async () => {
-            // Load the hnswlib module
-            if (!this.hnswlib) {
-                // Temporarily suppress stderr output during hnswlib loading to avoid IndexedDB warnings
-                const originalStderrWrite = process.stderr.write;
-                const originalConsoleError = console.error;
-                process.stderr.write = function (chunk, encoding, callback) {
-                    const message = chunk.toString();
-                    // Suppress specific IndexedDB/IDBFS related errors and WebAssembly errors
-                    if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
-                        message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
-                        message.includes('jsFS Error') || message.includes('syncing FS') ||
-                        message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
-                        message.includes('abort') || message.includes('assert') ||
-                        message.includes('hnswlib-wasm/dist/hnswlib')) {
-                        if (callback)
-                            callback();
-                        return true;
-                    }
-                    return originalStderrWrite.call(this, chunk, encoding, callback);
-                };
-                console.error = (...args) => {
-                    const message = args.join(' ');
-                    if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
-                        message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
-                        message.includes('jsFS Error') || message.includes('syncing FS') ||
-                        message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
-                        message.includes('abort') || message.includes('assert') ||
-                        message.includes('hnswlib-wasm/dist/hnswlib')) {
-                        return;
-                    }
-                    originalConsoleError.apply(console, args);
-                };
-                try {
-                    const { loadHnswlib } = await import('hnswlib-wasm/dist/hnswlib.js');
-                    this.hnswlib = await loadHnswlib();
-                }
-                finally {
-                    // Restore original output streams
-                    process.stderr.write = originalStderrWrite;
-                    console.error = originalConsoleError;
-                }
-            }
-            // Create new HNSW index (third parameter is autoSaveFilename, but we'll handle persistence manually)
-            this.index = new this.hnswlib.HierarchicalNSW('cosine', this.options.dimensions, '');
-            this.index.initIndex(this.options.maxElements, this.options.M || 16, this.options.efConstruction || 200, this.options.seed || 100);
-            this.currentSize = 0;
-            console.log(`Initialized HNSW index with ${this.options.dimensions} dimensions using hnswlib-wasm`);
-        }, 'Vector Index Initialization', {
-            category: ErrorCategory.INDEX,
-            severity: ErrorSeverity.FATAL
-        });
-    }
-    /**
-     * Load existing index from file using hnswlib-wasm
-     */
-    async loadIndex() {
-        if (!existsSync(this.indexPath)) {
-            throw createMissingFileError(this.indexPath, 'index', {
-                operationContext: 'VectorIndex.loadIndex'
-            });
-        }
-        try {
-            // Load the hnswlib module
-            if (!this.hnswlib) {
-                // Temporarily suppress stderr output during hnswlib loading to avoid IndexedDB warnings
-                const originalStderrWrite = process.stderr.write;
-                const originalConsoleError = console.error;
-                process.stderr.write = function (chunk, encoding, callback) {
-                    const message = chunk.toString();
-                    // Suppress specific IndexedDB/IDBFS related errors and WebAssembly errors
-                    if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
-                        message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
-                        message.includes('jsFS Error') || message.includes('syncing FS') ||
-                        message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
-                        message.includes('abort') || message.includes('assert') ||
-                        message.includes('hnswlib-wasm/dist/hnswlib')) {
-                        if (callback)
-                            callback();
-                        return true;
-                    }
-                    return originalStderrWrite.call(this, chunk, encoding, callback);
-                };
-                console.error = (...args) => {
-                    const message = args.join(' ');
-                    if (message.includes('IDBFS') || message.includes('indexedDB not supported') ||
-                        message.includes('EmscriptenFileSystemManager') || message.includes('Aborted') ||
-                        message.includes('jsFS Error') || message.includes('syncing FS') ||
-                        message.includes('RuntimeError: unreachable') || message.includes('___trap') ||
-                        message.includes('abort') || message.includes('assert') ||
-                        message.includes('hnswlib-wasm/dist/hnswlib')) {
-                        return;
-                    }
-                    originalConsoleError.apply(console, args);
-                };
-                try {
-                    const { loadHnswlib } = await import('hnswlib-wasm/dist/hnswlib.js');
-                    this.hnswlib = await loadHnswlib();
-                }
-                finally {
-                    // Restore original output streams
-                    process.stderr.write = originalStderrWrite;
-                    console.error = originalConsoleError;
-                }
-            }
-            // Create new HNSW index (third parameter is autoSaveFilename, but we'll handle persistence manually)
-            this.index = new this.hnswlib.HierarchicalNSW('cosine', this.options.dimensions, '');
-            // Load from binary format
-            const data = await BinaryIndexFormat.load(this.indexPath);
-            // Validate dimensions
-            if (data.dimensions !== this.options.dimensions) {
-                console.log(`⚠️  Dimension mismatch detected:`);
-                console.log(`   Stored dimensions: ${data.dimensions}`);
-                console.log(`   Expected dimensions: ${this.options.dimensions}`);
-                console.log(`   Number of vectors: ${data.vectors.length}`);
-                if (data.vectors.length > 0) {
-                    console.log(`   Actual vector length: ${data.vectors[0].vector.length}`);
-                }
-                throw createDimensionMismatchError(this.options.dimensions, data.dimensions, 'vector index loading', { operationContext: 'VectorIndex.loadIndex' });
-            }
-            // Update options from stored data
-            this.options.maxElements = data.maxElements;
-            this.options.M = data.M;
-            this.options.efConstruction = data.efConstruction;
-            this.options.seed = data.seed;
-            // Initialize HNSW index
-            this.index.initIndex(this.options.maxElements, this.options.M, this.options.efConstruction, this.options.seed);
-            // Clear and repopulate vector storage
-            this.vectorStorage.clear();
-            // Add all stored vectors to HNSW index
-            for (const item of data.vectors) {
-                this.index.addPoint(item.vector, item.id, false);
-                this.vectorStorage.set(item.id, item.vector);
-            }
-            this.currentSize = data.currentSize;
-            console.log(`✓ Loaded HNSW index with ${this.currentSize} vectors from ${this.indexPath}`);
-        }
-        catch (error) {
-            throw new Error(`Failed to load index from ${this.indexPath}: ${error}`);
-        }
-    }
-    /**
-     * Save index to binary format
-     */
-    async saveIndex() {
-        if (!this.index) {
-            throw new Error('Index not initialized');
-        }
-        try {
-            // Collect all vectors from storage
-            const vectors = Array.from(this.vectorStorage.entries()).map(([id, vector]) => ({
-                id,
-                vector
-            }));
-            // Save to binary format
-            await BinaryIndexFormat.save(this.indexPath, {
-                dimensions: this.options.dimensions,
-                maxElements: this.options.maxElements,
-                M: this.options.M || 16,
-                efConstruction: this.options.efConstruction || 200,
-                seed: this.options.seed || 100,
-                currentSize: this.currentSize,
-                vectors
-            });
-            console.log(`✓ Saved HNSW index with ${this.currentSize} vectors to ${this.indexPath}`);
-        }
-        catch (error) {
-            throw new Error(`Failed to save index to ${this.indexPath}: ${error}`);
-        }
-    }
-    /**
-     * Add a single vector to the HNSW index
-     */
-    addVector(embeddingId, vector) {
-        if (!this.index) {
-            throw new Error('Index not initialized');
-        }
-        if (vector.length !== this.options.dimensions) {
-            throw createDimensionMismatchError(this.options.dimensions, vector.length, 'vector addition', { operationContext: 'VectorIndex.addVector' });
-        }
-        try {
-            this.index.addPoint(vector, embeddingId, false);
-            // Store vector for persistence
-            this.vectorStorage.set(embeddingId, new Float32Array(vector));
-            this.currentSize++;
-        }
-        catch (error) {
-            throw new Error(`Failed to add vector ${embeddingId}: ${error}`);
-        }
-    }
-    /**
-     * Add multiple vectors to the index in batch
-     */
-    addVectors(vectors) {
-        for (const { id, vector } of vectors) {
-            this.addVector(id, vector);
-        }
-    }
-    /**
-     * Search for k nearest neighbors using hnswlib-wasm
-     */
-    search(queryVector, k = 5) {
-        if (!this.index) {
-            throw new Error('Index not initialized');
-        }
-        if (queryVector.length !== this.options.dimensions) {
-            throw createDimensionMismatchError(this.options.dimensions, queryVector.length, 'vector search', { operationContext: 'VectorIndex.search' });
-        }
-        if (this.currentSize === 0) {
-            return { neighbors: [], distances: [] };
-        }
-        try {
-            const result = this.index.searchKnn(queryVector, Math.min(k, this.currentSize), undefined);
-            return {
-                neighbors: result.neighbors,
-                distances: result.distances
-            };
-        }
-        catch (error) {
-            throw new Error(`Search failed: ${error}`);
-        }
-    }
-    /**
-     * Get current number of vectors in the index
-     */
-    getCurrentCount() {
-        return this.currentSize;
-    }
-    /**
-     * Check if index exists on disk
-     */
-    indexExists() {
-        return existsSync(this.indexPath);
-    }
-    /**
-     * Set search parameters for query time
-     */
-    setEf(ef) {
-        if (!this.index) {
-            throw new Error('Index not initialized');
-        }
-        try {
-            // hnswlib-wasm might not have setEf method, check if it exists
-            if (typeof this.index.setEfSearch === 'function') {
-                this.index.setEfSearch(ef);
-                console.log(`Set efSearch to ${ef}`);
-            }
-            else {
-                console.log(`setEfSearch not available in hnswlib-wasm`);
-            }
-        }
-        catch (error) {
-            console.log(`Failed to set ef: ${error}`);
-        }
-    }
-    /**
-     * Resize index to accommodate more vectors
-     */
-    resizeIndex(newMaxElements) {
-        if (!this.index) {
-            throw new Error('Index not initialized');
-        }
-        if (newMaxElements <= this.options.maxElements) {
-            throw new Error(`New max elements (${newMaxElements}) must be greater than current (${this.options.maxElements})`);
-        }
-        try {
-            this.index.resizeIndex(newMaxElements);
-            this.options.maxElements = newMaxElements;
-            console.log(`Resized index to accommodate ${newMaxElements} vectors`);
-        }
-        catch (error) {
-            throw new Error(`Failed to resize index: ${error}`);
-        }
-    }
-    /**
-     * Get index options (for external access to configuration)
-     */
-    getOptions() {
-        return { ...this.options };
-    }
-}
-//# sourceMappingURL=vector-index.js.map