rag-lite-ts 2.0.2 → 2.0.3

package/README.md

@@ -438,6 +438,33 @@ Now Claude can search your docs directly! Works with any MCP-compatible AI tool.
 </tr>
 </table>
 
+### 📁 Supported File Formats
+
+RAG-lite TS supports the following file formats with full processing implementations:
+
+**Text Mode:**
+- Markdown: `.md`, `.mdx`
+- Plain text: `.txt`
+- Documents: `.pdf`, `.docx`
+
+**Multimodal Mode** (includes all text formats plus):
+- Images: `.jpg`, `.jpeg`, `.png`, `.gif`, `.webp`, `.bmp`
+
+All formats work seamlessly with both single file and directory ingestion:
+
+```bash
+# Single file ingestion
+raglite ingest ./document.pdf
+raglite ingest ./readme.md
+raglite ingest ./notes.txt
+
+# Directory ingestion (processes all supported formats)
+raglite ingest ./docs/
+
+# Multimodal ingestion (includes images)
+raglite ingest ./mixed-content/ --mode multimodal
+```
+
 ## 🔧 How It Works
 
 RAG-lite TS follows a clean, efficient pipeline:

package/dist/cli/indexer.js

@@ -148,12 +148,31 @@ export async function runIngest(path, options = {}) {
         const pathType = stats.isDirectory() ? 'directory' : 'file';
         // Validate file type for single files
         if (stats.isFile()) {
-            const validExtensions = ['.md', '.txt'];
+            const mode = options.mode || 'text';
+            // Only formats with actual processing implementations
+            const textExtensions = ['.md', '.txt', '.mdx', '.pdf', '.docx'];
+            const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp'];
+            const validExtensions = mode === 'multimodal'
+                ? [...textExtensions, ...imageExtensions]
+                : textExtensions;
             const hasValidExtension = validExtensions.some(ext => path.toLowerCase().endsWith(ext));
             if (!hasValidExtension) {
                 console.error(`Error: Unsupported file type: ${path}`);
                 console.error('');
-                console.error('Supported file types: .md, .txt');
+                if (mode === 'multimodal') {
+                    console.error('Supported file types in multimodal mode:');
+                    console.error('  Text: .md, .txt, .mdx');
+                    console.error('  Documents: .pdf, .docx');
+                    console.error('  Images: .jpg, .jpeg, .png, .gif, .webp, .bmp');
+                }
+                else {
+                    console.error('Supported file types in text mode:');
+                    console.error('  Text: .md, .txt, .mdx');
+                    console.error('  Documents: .pdf, .docx');
+                    console.error('');
+                    console.error('For image files, use --mode multimodal:');
+                    console.error('  raglite ingest <path> --mode multimodal');
+                }
                 console.error('');
                 console.error('If you want to ingest multiple files, provide a directory path instead.');
                 process.exit(EXIT_CODES.INVALID_ARGUMENTS);

package/dist/cli.js

@@ -26,7 +26,7 @@ Commands:
   help              Show this help message
 
 Examples:
-  raglite ingest ./docs/           # Ingest all .md/.txt files in docs/
+  raglite ingest ./docs/           # Ingest all .md/.txt/.docx/.pdf files in docs/
   raglite ingest ./readme.md       # Ingest single file
   raglite ingest ./docs/ --model Xenova/all-mpnet-base-v2  # Use higher quality model
   raglite ingest ./docs/ --mode multimodal  # Enable multimodal processing
@@ -126,7 +126,7 @@ function validateArgs(command, args, options) {
                 console.error('Usage: raglite ingest <path>');
                 console.error('');
                 console.error('Examples:');
-                console.error('  raglite ingest ./docs/           # Ingest all .md/.txt files in docs/');
+                console.error('  raglite ingest ./docs/           # Ingest all .md/.txt/.docx/.pdf files in docs/');
                 console.error('  raglite ingest ./readme.md       # Ingest single file');
                 console.error('  raglite ingest ./docs/ --model Xenova/all-mpnet-base-v2  # Use higher quality model');
                 console.error('  raglite ingest ./docs/ --mode multimodal  # Enable multimodal processing');

package/dist/factories/text-factory.js

@@ -421,18 +421,35 @@ export class TextIngestionFactory {
                 console.log(`📁 Creating index directory: ${indexDir}`);
                 mkdirSync(indexDir, { recursive: true });
             }
-            // Step 1: Get model-specific defaults and merge with options
-            const modelDefaults = getModelDefaults(options.embeddingModel || config.embedding_model);
+            // Step 1: Determine effective mode and select appropriate default model
+            const effectiveMode = options.mode || 'text';
+            // Step 1.5: Select model based on mode if not explicitly provided
+            let effectiveModel;
+            if (options.embeddingModel) {
+                // Use explicitly provided model
+                effectiveModel = options.embeddingModel;
+            }
+            else {
+                // Select default model based on mode
+                if (effectiveMode === 'multimodal') {
+                    const { DEFAULT_MODELS } = await import('../core/model-registry.js');
+                    effectiveModel = DEFAULT_MODELS['clip'];
+                    console.log(`📊 No model specified for multimodal mode, using default: ${effectiveModel}`);
+                }
+                else {
+                    effectiveModel = config.embedding_model;
+                }
+            }
+            // Step 2: Get model-specific defaults and merge with options
+            const modelDefaults = getModelDefaults(effectiveModel);
             const effectiveBatchSize = options.batchSize ?? modelDefaults.batch_size;
             const effectiveChunkSize = options.chunkSize ?? modelDefaults.chunk_size;
             const effectiveChunkOverlap = options.chunkOverlap ?? modelDefaults.chunk_overlap;
-            // Step 1.5: Validate mode-model compatibility at creation time
-            const effectiveMode = options.mode || 'text';
-            const effectiveModel = options.embeddingModel || config.embedding_model;
+            // Step 3: Validate mode-model compatibility at creation time
             console.log('🔍 Validating mode-model compatibility...');
             validateModeModelCompatibilityOrThrow(effectiveMode, effectiveModel);
             console.log('✓ Mode-model compatibility validated');
-            // Step 2: Initialize embedding function based on mode
+            // Step 4: Initialize embedding function based on mode
             let embedFn;
             if (effectiveMode === 'multimodal') {
                 console.log('📊 Loading CLIP embedding model for multimodal mode...');
@@ -463,10 +480,10 @@ export class TextIngestionFactory {
             await initializeSchema(db);
             console.log('✓ Database connection established');
             // Step 3.1: Handle mode storage during ingestion
-            await this.handleModeStorage(db, options, modelDefaults);
-            // Step 4: Initialize index manager
+            await this.handleModeStorage(db, options, modelDefaults, effectiveModel);
+            // Step 5: Initialize index manager
             console.log('📇 Initializing vector index...');
-            const indexManager = new IndexManager(indexPath, dbPath, modelDefaults.dimensions, options.embeddingModel || config.embedding_model);
+            const indexManager = new IndexManager(indexPath, dbPath, modelDefaults.dimensions, effectiveModel);
             // Check if we need to force recreation due to model change
             let forceRecreate = false;
             if (options.forceRebuild && existsSync(indexPath) && existsSync(dbPath)) {
@@ -477,9 +494,8 @@ export class TextIngestionFactory {
                 const tempDb = await openDatabase(dbPath);
                 try {
                     const storedModel = await getStoredModelInfo(tempDb);
-                    const currentModel = options.embeddingModel || config.embedding_model;
-                    if (storedModel && storedModel.modelName !== currentModel) {
-                        console.log(`🔄 Model change detected: ${storedModel.modelName} → ${currentModel}`);
+                    if (storedModel && storedModel.modelName !== effectiveModel) {
+                        console.log(`🔄 Model change detected: ${storedModel.modelName} → ${effectiveModel}`);
                         console.log(`🔄 Dimensions change: ${storedModel.dimensions} → ${modelDefaults.dimensions}`);
                     }
                     else if (storedModel && storedModel.dimensions !== modelDefaults.dimensions) {
@@ -503,9 +519,8 @@ export class TextIngestionFactory {
                 // Update stored model info when rebuilding or creating new index
                 if (options.forceRebuild || forceRecreate) {
                     const { setStoredModelInfo } = await import('../core/db.js');
-                    const currentModel = options.embeddingModel || config.embedding_model;
-                    await setStoredModelInfo(db, currentModel, modelDefaults.dimensions);
-                    console.log(`✓ Updated stored model info: ${currentModel} (${modelDefaults.dimensions} dimensions)`);
+                    await setStoredModelInfo(db, effectiveModel, modelDefaults.dimensions);
+                    console.log(`✓ Updated stored model info: ${effectiveModel} (${modelDefaults.dimensions} dimensions)`);
                 }
             }
             else {
@@ -555,11 +570,10 @@ export class TextIngestionFactory {
      * Creates or validates system info based on the provided mode and options
      * @private
      */
-    static async handleModeStorage(db, options, modelDefaults) {
+    static async handleModeStorage(db, options, modelDefaults, effectiveModel) {
         const { getSystemInfo, setSystemInfo } = await import('../core/db.js');
-        // Determine the effective mode and model
+        // Determine the effective mode and reranking strategy
         const effectiveMode = options.mode || 'text';
-        const effectiveModel = options.embeddingModel || config.embedding_model;
         const effectiveRerankingStrategy = options.rerankingStrategy || 'cross-encoder';
         // Determine model type based on model name
         let modelType;

package/dist/indexer.js

@@ -16,11 +16,14 @@ async function main() {
         console.error('  <path>    File or directory path to ingest (.md and .txt files)');
         console.error('');
         console.error('Examples:');
-        console.error('  node indexer.js ./docs/           # Ingest all .md/.txt files in docs/');
+        console.error('  node indexer.js ./docs/           # Ingest all .md/.txt/.pdf/.docs files in docs/');
         console.error('  node indexer.js ./readme.md       # Ingest single file');
         console.error('  node indexer.js ../project/docs/  # Ingest from parent directory');
         console.error('');
-        console.error('Supported file types: .md (Markdown), .txt (Plain text)');
+        console.error('Supported file types:');
+        console.error('  Text: .md, .txt, .mdx');
+        console.error('  Documents: .pdf, .docx');
+        console.error('  Images (multimodal mode): .jpg, .jpeg, .png, .gif, .webp, .bmp');
         console.error('');
         console.error('After ingestion, use: node search.js "your query"');
         process.exit(EXIT_CODES.INVALID_ARGUMENTS);

package/dist/mcp-server.js

@@ -501,16 +501,23 @@ class RagLiteMCPServer {
             catch (error) {
                 throw new Error(`Cannot access path: ${args.path}. Check permissions.`);
             }
-            // Validate file type for single files
+            // Validate mode parameter
+            const mode = args.mode || 'text';
+            // Validate file type for single files (only formats with actual processing implementations)
             if (stats.isFile()) {
-                const validExtensions = ['.md', '.txt'];
+                const textExtensions = ['.md', '.txt', '.mdx', '.pdf', '.docx'];
+                const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp'];
+                const validExtensions = mode === 'multimodal'
+                    ? [...textExtensions, ...imageExtensions]
+                    : textExtensions;
                 const hasValidExtension = validExtensions.some(ext => args.path.toLowerCase().endsWith(ext));
                 if (!hasValidExtension) {
-                    throw new Error(`Unsupported file type: ${args.path}. Supported types: .md, .txt`);
+                    const supportedTypes = mode === 'multimodal'
+                        ? '.md, .txt, .mdx, .pdf, .docx, .jpg, .jpeg, .png, .gif, .webp, .bmp'
+                        : '.md, .txt, .mdx, .pdf, .docx';
+                    throw new Error(`Unsupported file type: ${args.path}. Supported types: ${supportedTypes}`);
                 }
             }
-            // Validate mode parameter
-            const mode = args.mode || 'text';
             if (!['text', 'multimodal'].includes(mode)) {
                 throw new Error(`Invalid mode: ${mode}. Supported modes: text, multimodal`);
             }
@@ -585,8 +592,8 @@ class RagLiteMCPServer {
                     chunks_per_second: result.processingTimeMs > 0 ?
                         Math.round(result.chunksCreated / (result.processingTimeMs / 1000) * 100) / 100 : 0,
                     supported_file_types: mode === 'multimodal'
-                        ? ['md', 'txt', 'jpg', 'jpeg', 'png', 'gif', 'webp']
-                        : ['md', 'txt'],
+                        ? ['md', 'txt', 'mdx', 'pdf', 'docx', 'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp']
+                        : ['md', 'txt', 'mdx', 'pdf', 'docx'],
                     success: true
                 };
                 return {
@@ -1132,7 +1139,7 @@ class RagLiteMCPServer {
                         text_search: true,
                         image_search: false,
                         multimodal_reranking: false,
-                        supported_file_types: ['md', 'txt']
+                        supported_file_types: ['md', 'txt', 'mdx', 'pdf', 'docx']
                     };
                 }
                 else if (systemInfo.mode === 'multimodal') {
@@ -1140,7 +1147,7 @@ class RagLiteMCPServer {
                         text_search: true,
                         image_search: true,
                         multimodal_reranking: true,
-                        supported_file_types: ['md', 'txt', 'jpg', 'png', 'gif', 'webp']
+                        supported_file_types: ['md', 'txt', 'mdx', 'pdf', 'docx', 'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp']
                     };
                 }
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rag-lite-ts",
-  "version": "2.0.2",
+  "version": "2.0.3",
   "description": "Local-first TypeScript retrieval engine with Chameleon Multimodal Architecture for semantic search over text and image content",
   "type": "module",
   "main": "./dist/index.js",