rag-lite-ts 1.0.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +651 -109
- package/dist/cli/indexer.js +262 -46
- package/dist/cli/search.js +54 -32
- package/dist/cli.js +185 -28
- package/dist/config.d.ts +34 -73
- package/dist/config.js +50 -255
- package/dist/core/abstract-embedder.d.ts +125 -0
- package/dist/core/abstract-embedder.js +264 -0
- package/dist/core/actionable-error-messages.d.ts +60 -0
- package/dist/core/actionable-error-messages.js +397 -0
- package/dist/core/adapters.d.ts +93 -0
- package/dist/core/adapters.js +139 -0
- package/dist/core/batch-processing-optimizer.d.ts +155 -0
- package/dist/core/batch-processing-optimizer.js +541 -0
- package/dist/core/chunker.d.ts +119 -0
- package/dist/core/chunker.js +73 -0
- package/dist/core/cli-database-utils.d.ts +53 -0
- package/dist/core/cli-database-utils.js +239 -0
- package/dist/core/config.d.ts +102 -0
- package/dist/core/config.js +247 -0
- package/dist/core/content-errors.d.ts +111 -0
- package/dist/core/content-errors.js +362 -0
- package/dist/core/content-manager.d.ts +343 -0
- package/dist/core/content-manager.js +1504 -0
- package/dist/core/content-performance-optimizer.d.ts +150 -0
- package/dist/core/content-performance-optimizer.js +516 -0
- package/dist/core/content-resolver.d.ts +104 -0
- package/dist/core/content-resolver.js +285 -0
- package/dist/core/cross-modal-search.d.ts +164 -0
- package/dist/core/cross-modal-search.js +342 -0
- package/dist/core/database-connection-manager.d.ts +109 -0
- package/dist/core/database-connection-manager.js +304 -0
- package/dist/core/db.d.ts +245 -0
- package/dist/core/db.js +952 -0
- package/dist/core/embedder-factory.d.ts +176 -0
- package/dist/core/embedder-factory.js +338 -0
- package/dist/{error-handler.d.ts → core/error-handler.d.ts} +23 -2
- package/dist/{error-handler.js → core/error-handler.js} +51 -8
- package/dist/core/index.d.ts +59 -0
- package/dist/core/index.js +69 -0
- package/dist/core/ingestion.d.ts +213 -0
- package/dist/core/ingestion.js +812 -0
- package/dist/core/interfaces.d.ts +408 -0
- package/dist/core/interfaces.js +106 -0
- package/dist/core/lazy-dependency-loader.d.ts +152 -0
- package/dist/core/lazy-dependency-loader.js +453 -0
- package/dist/core/mode-detection-service.d.ts +150 -0
- package/dist/core/mode-detection-service.js +565 -0
- package/dist/core/mode-model-validator.d.ts +92 -0
- package/dist/core/mode-model-validator.js +203 -0
- package/dist/core/model-registry.d.ts +120 -0
- package/dist/core/model-registry.js +415 -0
- package/dist/core/model-validator.d.ts +217 -0
- package/dist/core/model-validator.js +782 -0
- package/dist/{path-manager.d.ts → core/path-manager.d.ts} +5 -0
- package/dist/{path-manager.js → core/path-manager.js} +5 -0
- package/dist/core/polymorphic-search-factory.d.ts +154 -0
- package/dist/core/polymorphic-search-factory.js +344 -0
- package/dist/core/raglite-paths.d.ts +121 -0
- package/dist/core/raglite-paths.js +145 -0
- package/dist/core/reranking-config.d.ts +42 -0
- package/dist/core/reranking-config.js +156 -0
- package/dist/core/reranking-factory.d.ts +92 -0
- package/dist/core/reranking-factory.js +591 -0
- package/dist/core/reranking-strategies.d.ts +325 -0
- package/dist/core/reranking-strategies.js +720 -0
- package/dist/core/resource-cleanup.d.ts +163 -0
- package/dist/core/resource-cleanup.js +371 -0
- package/dist/core/resource-manager.d.ts +212 -0
- package/dist/core/resource-manager.js +564 -0
- package/dist/core/search-pipeline.d.ts +111 -0
- package/dist/core/search-pipeline.js +287 -0
- package/dist/core/search.d.ts +131 -0
- package/dist/core/search.js +296 -0
- package/dist/core/streaming-operations.d.ts +145 -0
- package/dist/core/streaming-operations.js +409 -0
- package/dist/core/types.d.ts +66 -0
- package/dist/core/types.js +6 -0
- package/dist/core/universal-embedder.d.ts +177 -0
- package/dist/core/universal-embedder.js +139 -0
- package/dist/core/validation-messages.d.ts +99 -0
- package/dist/core/validation-messages.js +334 -0
- package/dist/{vector-index.d.ts → core/vector-index.d.ts} +4 -0
- package/dist/{vector-index.js → core/vector-index.js} +21 -3
- package/dist/dom-polyfills.d.ts +6 -0
- package/dist/dom-polyfills.js +40 -0
- package/dist/factories/index.d.ts +43 -0
- package/dist/factories/index.js +44 -0
- package/dist/factories/text-factory.d.ts +560 -0
- package/dist/factories/text-factory.js +968 -0
- package/dist/file-processor.d.ts +90 -4
- package/dist/file-processor.js +723 -20
- package/dist/index-manager.d.ts +3 -2
- package/dist/index-manager.js +13 -11
- package/dist/index.d.ts +72 -8
- package/dist/index.js +102 -16
- package/dist/indexer.js +1 -1
- package/dist/ingestion.d.ts +44 -154
- package/dist/ingestion.js +75 -671
- package/dist/mcp-server.d.ts +35 -3
- package/dist/mcp-server.js +1186 -79
- package/dist/multimodal/clip-embedder.d.ts +314 -0
- package/dist/multimodal/clip-embedder.js +945 -0
- package/dist/multimodal/index.d.ts +6 -0
- package/dist/multimodal/index.js +6 -0
- package/dist/preprocess.js +1 -1
- package/dist/run-error-recovery-tests.d.ts +7 -0
- package/dist/run-error-recovery-tests.js +101 -0
- package/dist/search-standalone.js +1 -1
- package/dist/search.d.ts +51 -69
- package/dist/search.js +117 -412
- package/dist/test-utils.d.ts +8 -26
- package/dist/text/chunker.d.ts +33 -0
- package/dist/{chunker.js → text/chunker.js} +98 -75
- package/dist/{embedder.d.ts → text/embedder.d.ts} +22 -1
- package/dist/{embedder.js → text/embedder.js} +84 -10
- package/dist/text/index.d.ts +8 -0
- package/dist/text/index.js +9 -0
- package/dist/text/preprocessors/index.d.ts +17 -0
- package/dist/text/preprocessors/index.js +38 -0
- package/dist/text/preprocessors/mdx.d.ts +25 -0
- package/dist/text/preprocessors/mdx.js +101 -0
- package/dist/text/preprocessors/mermaid.d.ts +68 -0
- package/dist/text/preprocessors/mermaid.js +330 -0
- package/dist/text/preprocessors/registry.d.ts +56 -0
- package/dist/text/preprocessors/registry.js +180 -0
- package/dist/text/reranker.d.ts +59 -0
- package/dist/{reranker.js → text/reranker.js} +138 -53
- package/dist/text/sentence-transformer-embedder.d.ts +96 -0
- package/dist/text/sentence-transformer-embedder.js +340 -0
- package/dist/{tokenizer.d.ts → text/tokenizer.d.ts} +1 -0
- package/dist/{tokenizer.js → text/tokenizer.js} +7 -2
- package/dist/types.d.ts +40 -1
- package/dist/utils/vector-math.d.ts +31 -0
- package/dist/utils/vector-math.js +70 -0
- package/package.json +16 -4
- package/dist/api-errors.d.ts.map +0 -1
- package/dist/api-errors.js.map +0 -1
- package/dist/chunker.d.ts +0 -47
- package/dist/chunker.d.ts.map +0 -1
- package/dist/chunker.js.map +0 -1
- package/dist/cli/indexer.d.ts.map +0 -1
- package/dist/cli/indexer.js.map +0 -1
- package/dist/cli/search.d.ts.map +0 -1
- package/dist/cli/search.js.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js.map +0 -1
- package/dist/db.d.ts +0 -90
- package/dist/db.d.ts.map +0 -1
- package/dist/db.js +0 -340
- package/dist/db.js.map +0 -1
- package/dist/embedder.d.ts.map +0 -1
- package/dist/embedder.js.map +0 -1
- package/dist/error-handler.d.ts.map +0 -1
- package/dist/error-handler.js.map +0 -1
- package/dist/file-processor.d.ts.map +0 -1
- package/dist/file-processor.js.map +0 -1
- package/dist/index-manager.d.ts.map +0 -1
- package/dist/index-manager.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/indexer.d.ts.map +0 -1
- package/dist/indexer.js.map +0 -1
- package/dist/ingestion.d.ts.map +0 -1
- package/dist/ingestion.js.map +0 -1
- package/dist/mcp-server.d.ts.map +0 -1
- package/dist/mcp-server.js.map +0 -1
- package/dist/path-manager.d.ts.map +0 -1
- package/dist/path-manager.js.map +0 -1
- package/dist/preprocess.d.ts.map +0 -1
- package/dist/preprocess.js.map +0 -1
- package/dist/preprocessors/index.d.ts.map +0 -1
- package/dist/preprocessors/index.js.map +0 -1
- package/dist/preprocessors/mdx.d.ts.map +0 -1
- package/dist/preprocessors/mdx.js.map +0 -1
- package/dist/preprocessors/mermaid.d.ts.map +0 -1
- package/dist/preprocessors/mermaid.js.map +0 -1
- package/dist/preprocessors/registry.d.ts.map +0 -1
- package/dist/preprocessors/registry.js.map +0 -1
- package/dist/reranker.d.ts +0 -40
- package/dist/reranker.d.ts.map +0 -1
- package/dist/reranker.js.map +0 -1
- package/dist/resource-manager-demo.d.ts +0 -7
- package/dist/resource-manager-demo.d.ts.map +0 -1
- package/dist/resource-manager-demo.js +0 -52
- package/dist/resource-manager-demo.js.map +0 -1
- package/dist/resource-manager.d.ts +0 -129
- package/dist/resource-manager.d.ts.map +0 -1
- package/dist/resource-manager.js +0 -389
- package/dist/resource-manager.js.map +0 -1
- package/dist/search-standalone.d.ts.map +0 -1
- package/dist/search-standalone.js.map +0 -1
- package/dist/search.d.ts.map +0 -1
- package/dist/search.js.map +0 -1
- package/dist/test-utils.d.ts.map +0 -1
- package/dist/test-utils.js.map +0 -1
- package/dist/tokenizer.d.ts.map +0 -1
- package/dist/tokenizer.js.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/vector-index.d.ts.map +0 -1
- package/dist/vector-index.js.map +0 -1
package/dist/mcp-server.js
CHANGED
|
@@ -1,25 +1,36 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* MCP server entry point for rag-lite-ts
|
|
3
|
+
* MCP server entry point for rag-lite-ts with Chameleon Multimodal Architecture
|
|
4
4
|
*
|
|
5
|
-
* This is a thin wrapper around
|
|
5
|
+
* This is a thin wrapper around the polymorphic search and ingestion functions
|
|
6
6
|
* that exposes them as MCP tools without creating REST/GraphQL endpoints.
|
|
7
7
|
*
|
|
8
|
+
* The MCP server supports both text-only and multimodal modes:
|
|
9
|
+
* - Text mode: Optimized for text documents using sentence-transformer models
|
|
10
|
+
* - Multimodal mode: Supports mixed text and image content using CLIP models
|
|
11
|
+
*
|
|
12
|
+
* Key Features:
|
|
13
|
+
* - Automatic mode detection from database configuration
|
|
14
|
+
* - Polymorphic runtime that adapts to stored mode settings
|
|
15
|
+
* - Support for multiple embedding models and reranking strategies
|
|
16
|
+
* - Content type filtering and multimodal search capabilities
|
|
17
|
+
* - Comprehensive model and strategy information tools
|
|
18
|
+
*
|
|
8
19
|
* The MCP server lives in the same package as CLI with dual entry points
|
|
9
20
|
* and provides proper MCP tool definitions for search and indexing capabilities.
|
|
10
21
|
*
|
|
11
|
-
* Requirements addressed: 6.2, 6.4, 6.5, 6.6
|
|
22
|
+
* Requirements addressed: 6.2, 6.4, 6.5, 6.6, 9.1, 9.2, 9.3
|
|
12
23
|
*/
|
|
13
24
|
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
|
14
25
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
15
26
|
import { CallToolRequestSchema, ListToolsRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
|
|
16
27
|
import { existsSync, statSync } from 'fs';
|
|
17
28
|
import { resolve } from 'path';
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import { config,
|
|
29
|
+
import { PolymorphicSearchFactory } from './core/polymorphic-search-factory.js';
|
|
30
|
+
import { TextIngestionFactory } from './factories/text-factory.js';
|
|
31
|
+
import { openDatabase } from './core/db.js';
|
|
32
|
+
import { DatabaseConnectionManager } from './core/database-connection-manager.js';
|
|
33
|
+
import { config, validateCoreConfig, ConfigurationError } from './core/config.js';
|
|
23
34
|
/**
|
|
24
35
|
* MCP Server class that wraps RAG-lite TS functionality
|
|
25
36
|
* Implements MCP protocol interface without creating REST/GraphQL endpoints
|
|
@@ -44,13 +55,15 @@ class RagLiteMCPServer {
|
|
|
44
55
|
* Add proper MCP tool definitions for search and indexing capabilities
|
|
45
56
|
*/
|
|
46
57
|
setupToolHandlers() {
|
|
47
|
-
// List available tools
|
|
58
|
+
// List available tools - with dynamic descriptions based on database mode
|
|
48
59
|
this.server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
60
|
+
// Detect current mode and capabilities from database
|
|
61
|
+
const capabilities = await this.detectCapabilities();
|
|
49
62
|
return {
|
|
50
63
|
tools: [
|
|
51
64
|
{
|
|
52
65
|
name: 'search',
|
|
53
|
-
description:
|
|
66
|
+
description: this.generateSearchDescription(capabilities),
|
|
54
67
|
inputSchema: {
|
|
55
68
|
type: 'object',
|
|
56
69
|
properties: {
|
|
@@ -79,18 +92,34 @@ class RagLiteMCPServer {
|
|
|
79
92
|
},
|
|
80
93
|
{
|
|
81
94
|
name: 'ingest',
|
|
82
|
-
description: 'Ingest documents from a file or directory path.
|
|
95
|
+
description: 'Ingest documents from a file or directory path. Supports both text-only and multimodal modes. In text mode, processes .md and .txt files. In multimodal mode, also processes images (.jpg, .png, .gif, .webp).',
|
|
83
96
|
inputSchema: {
|
|
84
97
|
type: 'object',
|
|
85
98
|
properties: {
|
|
86
99
|
path: {
|
|
87
100
|
type: 'string',
|
|
88
|
-
description: 'File or directory path to ingest. Can be a single
|
|
101
|
+
description: 'File or directory path to ingest. Can be a single file or directory containing supported files.'
|
|
102
|
+
},
|
|
103
|
+
mode: {
|
|
104
|
+
type: 'string',
|
|
105
|
+
description: 'Processing mode: text for text-only content, multimodal for mixed text and image content (default: text)',
|
|
106
|
+
enum: ['text', 'multimodal'],
|
|
107
|
+
default: 'text'
|
|
89
108
|
},
|
|
90
109
|
model: {
|
|
91
110
|
type: 'string',
|
|
92
|
-
description: 'Embedding model to use
|
|
93
|
-
enum: [
|
|
111
|
+
description: 'Embedding model to use. For text mode: sentence-transformers/all-MiniLM-L6-v2 (default), Xenova/all-mpnet-base-v2. For multimodal mode: Xenova/clip-vit-base-patch32 (default), Xenova/clip-vit-base-patch16',
|
|
112
|
+
enum: [
|
|
113
|
+
'sentence-transformers/all-MiniLM-L6-v2',
|
|
114
|
+
'Xenova/all-mpnet-base-v2',
|
|
115
|
+
'Xenova/clip-vit-base-patch32',
|
|
116
|
+
'Xenova/clip-vit-base-patch16'
|
|
117
|
+
]
|
|
118
|
+
},
|
|
119
|
+
rerank_strategy: {
|
|
120
|
+
type: 'string',
|
|
121
|
+
description: 'Reranking strategy for multimodal mode. Options: text-derived (default), metadata, hybrid, disabled',
|
|
122
|
+
enum: ['text-derived', 'metadata', 'hybrid', 'disabled']
|
|
94
123
|
},
|
|
95
124
|
force_rebuild: {
|
|
96
125
|
type: 'boolean',
|
|
@@ -102,6 +131,44 @@ class RagLiteMCPServer {
|
|
|
102
131
|
additionalProperties: false
|
|
103
132
|
}
|
|
104
133
|
},
|
|
134
|
+
{
|
|
135
|
+
name: 'ingest_image',
|
|
136
|
+
description: 'Ingest image files or image URLs into the multimodal index. Automatically sets mode to multimodal and uses CLIP embeddings. Supports local image files and remote image URLs (http/https).',
|
|
137
|
+
inputSchema: {
|
|
138
|
+
type: 'object',
|
|
139
|
+
properties: {
|
|
140
|
+
source: {
|
|
141
|
+
type: 'string',
|
|
142
|
+
description: 'Image file path or URL. Can be a local file path (e.g., ./image.jpg) or a remote URL (e.g., https://example.com/image.jpg). Supported formats: jpg, jpeg, png, gif, webp.'
|
|
143
|
+
},
|
|
144
|
+
model: {
|
|
145
|
+
type: 'string',
|
|
146
|
+
description: 'CLIP model to use for image embedding (default: Xenova/clip-vit-base-patch32)',
|
|
147
|
+
enum: [
|
|
148
|
+
'Xenova/clip-vit-base-patch32',
|
|
149
|
+
'Xenova/clip-vit-base-patch16'
|
|
150
|
+
],
|
|
151
|
+
default: 'Xenova/clip-vit-base-patch32'
|
|
152
|
+
},
|
|
153
|
+
rerank_strategy: {
|
|
154
|
+
type: 'string',
|
|
155
|
+
description: 'Reranking strategy for multimodal mode. Options: text-derived (default), metadata, hybrid, disabled',
|
|
156
|
+
enum: ['text-derived', 'metadata', 'hybrid', 'disabled'],
|
|
157
|
+
default: 'text-derived'
|
|
158
|
+
},
|
|
159
|
+
title: {
|
|
160
|
+
type: 'string',
|
|
161
|
+
description: 'Optional title for the image. If not provided, will use filename or URL.'
|
|
162
|
+
},
|
|
163
|
+
metadata: {
|
|
164
|
+
type: 'object',
|
|
165
|
+
description: 'Optional metadata to associate with the image (e.g., tags, description, source info)'
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
required: ['source'],
|
|
169
|
+
additionalProperties: false
|
|
170
|
+
}
|
|
171
|
+
},
|
|
105
172
|
{
|
|
106
173
|
name: 'rebuild_index',
|
|
107
174
|
description: 'Rebuild the entire vector index from scratch. Useful when model version changes or for maintenance. This will regenerate all embeddings.',
|
|
@@ -119,6 +186,104 @@ class RagLiteMCPServer {
|
|
|
119
186
|
properties: {},
|
|
120
187
|
additionalProperties: false
|
|
121
188
|
}
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
name: 'get_mode_info',
|
|
192
|
+
description: 'Get current system mode and configuration information including detected mode, model, and reranking strategy.',
|
|
193
|
+
inputSchema: {
|
|
194
|
+
type: 'object',
|
|
195
|
+
properties: {},
|
|
196
|
+
additionalProperties: false
|
|
197
|
+
}
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
name: 'multimodal_search',
|
|
201
|
+
description: 'Search indexed documents with multimodal capabilities and content type filtering. Returns relevant document chunks with content type information. Image results include base64-encoded image data for display in MCP clients. Supports cross-modal search in multimodal mode (text queries can find images, image queries can find text).',
|
|
202
|
+
inputSchema: {
|
|
203
|
+
type: 'object',
|
|
204
|
+
properties: {
|
|
205
|
+
query: {
|
|
206
|
+
type: 'string',
|
|
207
|
+
description: 'Search query string to find relevant documents. In multimodal mode, text queries can find semantically similar images.',
|
|
208
|
+
minLength: 1,
|
|
209
|
+
maxLength: 500
|
|
210
|
+
},
|
|
211
|
+
top_k: {
|
|
212
|
+
type: 'number',
|
|
213
|
+
description: 'Number of results to return (default: 10, max: 100)',
|
|
214
|
+
minimum: 1,
|
|
215
|
+
maximum: 100,
|
|
216
|
+
default: 10
|
|
217
|
+
},
|
|
218
|
+
rerank: {
|
|
219
|
+
type: 'boolean',
|
|
220
|
+
description: 'Enable reranking for better result quality (default: false)',
|
|
221
|
+
default: false
|
|
222
|
+
},
|
|
223
|
+
content_type: {
|
|
224
|
+
type: 'string',
|
|
225
|
+
description: 'Filter results by content type (text, image, pdf, docx). If not specified, returns all content types. Use "image" to find only images, "text" for only text.',
|
|
226
|
+
enum: ['text', 'image', 'pdf', 'docx']
|
|
227
|
+
}
|
|
228
|
+
},
|
|
229
|
+
required: ['query'],
|
|
230
|
+
additionalProperties: false
|
|
231
|
+
}
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
name: 'list_supported_models',
|
|
235
|
+
description: 'List all supported embedding models with their capabilities, dimensions, and supported content types.',
|
|
236
|
+
inputSchema: {
|
|
237
|
+
type: 'object',
|
|
238
|
+
properties: {
|
|
239
|
+
model_type: {
|
|
240
|
+
type: 'string',
|
|
241
|
+
description: 'Filter models by type (sentence-transformer, clip). If not specified, returns all models.',
|
|
242
|
+
enum: ['sentence-transformer', 'clip']
|
|
243
|
+
},
|
|
244
|
+
content_type: {
|
|
245
|
+
type: 'string',
|
|
246
|
+
description: 'Filter models by supported content type (text, image). If not specified, returns all models.',
|
|
247
|
+
enum: ['text', 'image']
|
|
248
|
+
}
|
|
249
|
+
},
|
|
250
|
+
additionalProperties: false
|
|
251
|
+
}
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
name: 'list_reranking_strategies',
|
|
255
|
+
description: 'List all supported reranking strategies for different modes with their descriptions and requirements.',
|
|
256
|
+
inputSchema: {
|
|
257
|
+
type: 'object',
|
|
258
|
+
properties: {
|
|
259
|
+
mode: {
|
|
260
|
+
type: 'string',
|
|
261
|
+
description: 'Filter strategies by mode (text, multimodal). If not specified, returns strategies for all modes.',
|
|
262
|
+
enum: ['text', 'multimodal']
|
|
263
|
+
}
|
|
264
|
+
},
|
|
265
|
+
additionalProperties: false
|
|
266
|
+
}
|
|
267
|
+
},
|
|
268
|
+
{
|
|
269
|
+
name: 'get_system_stats',
|
|
270
|
+
description: 'Get comprehensive system statistics including mode-specific metrics, performance data, and resource usage.',
|
|
271
|
+
inputSchema: {
|
|
272
|
+
type: 'object',
|
|
273
|
+
properties: {
|
|
274
|
+
include_performance: {
|
|
275
|
+
type: 'boolean',
|
|
276
|
+
description: 'Include performance metrics and timing data (default: false)',
|
|
277
|
+
default: false
|
|
278
|
+
},
|
|
279
|
+
include_content_breakdown: {
|
|
280
|
+
type: 'boolean',
|
|
281
|
+
description: 'Include breakdown of content by type (default: false)',
|
|
282
|
+
default: false
|
|
283
|
+
}
|
|
284
|
+
},
|
|
285
|
+
additionalProperties: false
|
|
286
|
+
}
|
|
122
287
|
}
|
|
123
288
|
],
|
|
124
289
|
};
|
|
@@ -132,10 +297,22 @@ class RagLiteMCPServer {
|
|
|
132
297
|
return await this.handleSearch(args);
|
|
133
298
|
case 'ingest':
|
|
134
299
|
return await this.handleIngest(args);
|
|
300
|
+
case 'ingest_image':
|
|
301
|
+
return await this.handleIngestImage(args);
|
|
135
302
|
case 'rebuild_index':
|
|
136
303
|
return await this.handleRebuildIndex(args);
|
|
137
304
|
case 'get_stats':
|
|
138
305
|
return await this.handleGetStats(args);
|
|
306
|
+
case 'get_mode_info':
|
|
307
|
+
return await this.handleGetModeInfo(args);
|
|
308
|
+
case 'multimodal_search':
|
|
309
|
+
return await this.handleMultimodalSearch(args);
|
|
310
|
+
case 'list_supported_models':
|
|
311
|
+
return await this.handleListSupportedModels(args);
|
|
312
|
+
case 'list_reranking_strategies':
|
|
313
|
+
return await this.handleListRerankingStrategies(args);
|
|
314
|
+
case 'get_system_stats':
|
|
315
|
+
return await this.handleGetSystemStats(args);
|
|
139
316
|
default:
|
|
140
317
|
throw new Error(`Unknown tool: ${name}`);
|
|
141
318
|
}
|
|
@@ -198,20 +375,41 @@ class RagLiteMCPServer {
|
|
|
198
375
|
const startTime = Date.now();
|
|
199
376
|
const results = await this.searchEngine.search(args.query, searchOptions);
|
|
200
377
|
const searchTime = Date.now() - startTime;
|
|
201
|
-
// Format results for MCP response
|
|
378
|
+
// Format results for MCP response with content type information
|
|
202
379
|
const formattedResults = {
|
|
203
380
|
query: args.query,
|
|
204
381
|
results_count: results.length,
|
|
205
382
|
search_time_ms: searchTime,
|
|
206
|
-
results: results.map((result, index) =>
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
383
|
+
results: await Promise.all(results.map(async (result, index) => {
|
|
384
|
+
const formattedResult = {
|
|
385
|
+
rank: index + 1,
|
|
386
|
+
score: Math.round(result.score * 100) / 100, // Round to 2 decimal places
|
|
387
|
+
content_type: result.contentType,
|
|
388
|
+
document: {
|
|
389
|
+
id: result.document.id,
|
|
390
|
+
title: result.document.title,
|
|
391
|
+
source: result.document.source,
|
|
392
|
+
content_type: result.document.contentType
|
|
393
|
+
},
|
|
394
|
+
text: result.content
|
|
395
|
+
};
|
|
396
|
+
// For image content, include base64-encoded image data for MCP clients
|
|
397
|
+
if (result.contentType === 'image' && result.document.contentId) {
|
|
398
|
+
try {
|
|
399
|
+
const imageData = await this.searchEngine.getContent(result.document.contentId, 'base64');
|
|
400
|
+
formattedResult.image_data = imageData;
|
|
401
|
+
formattedResult.image_format = 'base64';
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
// If image retrieval fails, include error but don't fail the entire search
|
|
405
|
+
formattedResult.image_error = error instanceof Error ? error.message : 'Failed to retrieve image';
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
// Include metadata if available
|
|
409
|
+
if (result.metadata) {
|
|
410
|
+
formattedResult.metadata = result.metadata;
|
|
411
|
+
}
|
|
412
|
+
return formattedResult;
|
|
215
413
|
}))
|
|
216
414
|
};
|
|
217
415
|
return {
|
|
@@ -311,19 +509,62 @@ class RagLiteMCPServer {
|
|
|
311
509
|
throw new Error(`Unsupported file type: ${args.path}. Supported types: .md, .txt`);
|
|
312
510
|
}
|
|
313
511
|
}
|
|
512
|
+
// Validate mode parameter
|
|
513
|
+
const mode = args.mode || 'text';
|
|
514
|
+
if (!['text', 'multimodal'].includes(mode)) {
|
|
515
|
+
throw new Error(`Invalid mode: ${mode}. Supported modes: text, multimodal`);
|
|
516
|
+
}
|
|
314
517
|
// Validate model parameter if provided
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
518
|
+
const supportedModels = [
|
|
519
|
+
'sentence-transformers/all-MiniLM-L6-v2',
|
|
520
|
+
'Xenova/all-mpnet-base-v2',
|
|
521
|
+
'Xenova/clip-vit-base-patch32',
|
|
522
|
+
'Xenova/clip-vit-base-patch16'
|
|
523
|
+
];
|
|
524
|
+
if (args.model && !supportedModels.includes(args.model)) {
|
|
525
|
+
throw new Error(`Unsupported model: ${args.model}. Supported models: ${supportedModels.join(', ')}`);
|
|
526
|
+
}
|
|
527
|
+
// Validate model compatibility with mode
|
|
528
|
+
if (mode === 'text' && args.model) {
|
|
529
|
+
const textModels = ['sentence-transformers/all-MiniLM-L6-v2', 'Xenova/all-mpnet-base-v2'];
|
|
530
|
+
if (!textModels.includes(args.model)) {
|
|
531
|
+
throw new Error(`Model ${args.model} is not compatible with text mode. Use: ${textModels.join(', ')}`);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
if (mode === 'multimodal' && args.model) {
|
|
535
|
+
const multimodalModels = ['Xenova/clip-vit-base-patch32', 'Xenova/clip-vit-base-patch16'];
|
|
536
|
+
if (!multimodalModels.includes(args.model)) {
|
|
537
|
+
throw new Error(`Model ${args.model} is not compatible with multimodal mode. Use: ${multimodalModels.join(', ')}`);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
// Validate reranking strategy for multimodal mode
|
|
541
|
+
if (args.rerank_strategy) {
|
|
542
|
+
if (mode === 'text') {
|
|
543
|
+
throw new Error('Reranking strategy parameter is only supported in multimodal mode');
|
|
544
|
+
}
|
|
545
|
+
const validStrategies = ['text-derived', 'metadata', 'hybrid', 'disabled'];
|
|
546
|
+
if (!validStrategies.includes(args.rerank_strategy)) {
|
|
547
|
+
throw new Error(`Invalid reranking strategy: ${args.rerank_strategy}. Supported strategies: ${validStrategies.join(', ')}`);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
// Prepare factory options
|
|
551
|
+
const factoryOptions = {
|
|
552
|
+
mode: mode
|
|
553
|
+
};
|
|
554
|
+
if (args.model) {
|
|
555
|
+
factoryOptions.embeddingModel = args.model;
|
|
556
|
+
}
|
|
557
|
+
if (args.rerank_strategy && mode === 'multimodal') {
|
|
558
|
+
factoryOptions.rerankingStrategy = args.rerank_strategy;
|
|
559
|
+
}
|
|
560
|
+
if (args.force_rebuild) {
|
|
561
|
+
factoryOptions.forceRebuild = true;
|
|
562
|
+
}
|
|
563
|
+
// Create and run ingestion pipeline using text factory
|
|
564
|
+
// The TextIngestionFactory already supports mode and reranking strategy parameters
|
|
565
|
+
const pipeline = await TextIngestionFactory.create(config.db_file, config.index_file, factoryOptions);
|
|
323
566
|
try {
|
|
324
|
-
const result = await pipeline.ingestPath(resolvedPath
|
|
325
|
-
forceRebuild: args.force_rebuild || false
|
|
326
|
-
});
|
|
567
|
+
const result = await pipeline.ingestPath(resolvedPath);
|
|
327
568
|
// Reset search engine initialization flag since index may have changed
|
|
328
569
|
this.isSearchEngineInitialized = false;
|
|
329
570
|
this.searchEngine = null;
|
|
@@ -331,6 +572,9 @@ class RagLiteMCPServer {
|
|
|
331
572
|
const ingestionSummary = {
|
|
332
573
|
path: resolvedPath,
|
|
333
574
|
path_type: stats.isDirectory() ? 'directory' : 'file',
|
|
575
|
+
mode: mode,
|
|
576
|
+
model: args.model || (mode === 'multimodal' ? 'Xenova/clip-vit-base-patch32' : 'sentence-transformers/all-MiniLM-L6-v2'),
|
|
577
|
+
reranking_strategy: args.rerank_strategy || (mode === 'multimodal' ? 'text-derived' : 'cross-encoder'),
|
|
334
578
|
documents_processed: result.documentsProcessed,
|
|
335
579
|
chunks_created: result.chunksCreated,
|
|
336
580
|
embeddings_generated: result.embeddingsGenerated,
|
|
@@ -340,6 +584,9 @@ class RagLiteMCPServer {
|
|
|
340
584
|
processing_time_seconds: Math.round(result.processingTimeMs / 1000 * 100) / 100,
|
|
341
585
|
chunks_per_second: result.processingTimeMs > 0 ?
|
|
342
586
|
Math.round(result.chunksCreated / (result.processingTimeMs / 1000) * 100) / 100 : 0,
|
|
587
|
+
supported_file_types: mode === 'multimodal'
|
|
588
|
+
? ['md', 'txt', 'jpg', 'jpeg', 'png', 'gif', 'webp']
|
|
589
|
+
: ['md', 'txt'],
|
|
343
590
|
success: true
|
|
344
591
|
};
|
|
345
592
|
return {
|
|
@@ -443,30 +690,286 @@ class RagLiteMCPServer {
|
|
|
443
690
|
throw error;
|
|
444
691
|
}
|
|
445
692
|
}
|
|
693
|
+
/**
|
|
694
|
+
* Handle ingest image tool calls
|
|
695
|
+
* Specialized tool for ingesting images from local files or URLs
|
|
696
|
+
*/
|
|
697
|
+
async handleIngestImage(args) {
|
|
698
|
+
try {
|
|
699
|
+
// Validate arguments
|
|
700
|
+
if (!args.source || typeof args.source !== 'string') {
|
|
701
|
+
throw new Error('Source parameter is required and must be a string (file path or URL)');
|
|
702
|
+
}
|
|
703
|
+
const source = args.source.trim();
|
|
704
|
+
if (source.length === 0) {
|
|
705
|
+
throw new Error('Source cannot be empty');
|
|
706
|
+
}
|
|
707
|
+
// Check if source is a URL or local file
|
|
708
|
+
const isUrl = source.startsWith('http://') || source.startsWith('https://');
|
|
709
|
+
let resolvedPath;
|
|
710
|
+
let isTemporaryFile = false;
|
|
711
|
+
let tempFilePath = null;
|
|
712
|
+
if (isUrl) {
|
|
713
|
+
// Download image from URL to temporary location
|
|
714
|
+
console.error(`📥 Downloading image from URL: ${source}`);
|
|
715
|
+
try {
|
|
716
|
+
// Import required modules for URL download
|
|
717
|
+
const https = await import('https');
|
|
718
|
+
const http = await import('http');
|
|
719
|
+
const { promises: fs } = await import('fs');
|
|
720
|
+
const { join } = await import('path');
|
|
721
|
+
const { tmpdir } = await import('os');
|
|
722
|
+
const { randomBytes } = await import('crypto');
|
|
723
|
+
// Generate temporary file path
|
|
724
|
+
const tempDir = tmpdir();
|
|
725
|
+
const randomName = randomBytes(16).toString('hex');
|
|
726
|
+
const urlExt = source.split('.').pop()?.split('?')[0] || 'jpg';
|
|
727
|
+
tempFilePath = join(tempDir, `mcp-image-${randomName}.${urlExt}`);
|
|
728
|
+
// Download the image
|
|
729
|
+
await new Promise((resolve, reject) => {
|
|
730
|
+
const protocol = source.startsWith('https://') ? https : http;
|
|
731
|
+
protocol.get(source, (response) => {
|
|
732
|
+
if (response.statusCode === 301 || response.statusCode === 302) {
|
|
733
|
+
// Handle redirects
|
|
734
|
+
const redirectUrl = response.headers.location;
|
|
735
|
+
if (redirectUrl) {
|
|
736
|
+
const redirectProtocol = redirectUrl.startsWith('https://') ? https : http;
|
|
737
|
+
redirectProtocol.get(redirectUrl, (redirectResponse) => {
|
|
738
|
+
if (redirectResponse.statusCode !== 200) {
|
|
739
|
+
reject(new Error(`Failed to download image: HTTP ${redirectResponse.statusCode}`));
|
|
740
|
+
return;
|
|
741
|
+
}
|
|
742
|
+
const fileStream = require('fs').createWriteStream(tempFilePath);
|
|
743
|
+
redirectResponse.pipe(fileStream);
|
|
744
|
+
fileStream.on('finish', () => {
|
|
745
|
+
fileStream.close();
|
|
746
|
+
resolve();
|
|
747
|
+
});
|
|
748
|
+
fileStream.on('error', reject);
|
|
749
|
+
}).on('error', reject);
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
else if (response.statusCode !== 200) {
|
|
753
|
+
reject(new Error(`Failed to download image: HTTP ${response.statusCode}`));
|
|
754
|
+
return;
|
|
755
|
+
}
|
|
756
|
+
else {
|
|
757
|
+
const fileStream = require('fs').createWriteStream(tempFilePath);
|
|
758
|
+
response.pipe(fileStream);
|
|
759
|
+
fileStream.on('finish', () => {
|
|
760
|
+
fileStream.close();
|
|
761
|
+
resolve();
|
|
762
|
+
});
|
|
763
|
+
fileStream.on('error', reject);
|
|
764
|
+
}
|
|
765
|
+
}).on('error', reject);
|
|
766
|
+
});
|
|
767
|
+
resolvedPath = tempFilePath;
|
|
768
|
+
isTemporaryFile = true;
|
|
769
|
+
console.error(`✅ Image downloaded to: ${tempFilePath}`);
|
|
770
|
+
}
|
|
771
|
+
catch (downloadError) {
|
|
772
|
+
throw new Error(`Failed to download image from URL: ${downloadError instanceof Error ? downloadError.message : 'Unknown error'}`);
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
else {
|
|
776
|
+
// Local file path
|
|
777
|
+
resolvedPath = resolve(source);
|
|
778
|
+
if (!existsSync(resolvedPath)) {
|
|
779
|
+
throw new Error(`Image file does not exist: ${source}`);
|
|
780
|
+
}
|
|
781
|
+
// Validate it's a file
|
|
782
|
+
try {
|
|
783
|
+
const stats = statSync(resolvedPath);
|
|
784
|
+
if (!stats.isFile()) {
|
|
785
|
+
throw new Error(`Source is not a file: ${source}`);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
catch (error) {
|
|
789
|
+
throw new Error(`Cannot access image file: ${source}. Check permissions.`);
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
// Validate image file extension
|
|
793
|
+
const validExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
|
|
794
|
+
const hasValidExtension = validExtensions.some(ext => resolvedPath.toLowerCase().endsWith(ext));
|
|
795
|
+
if (!hasValidExtension) {
|
|
796
|
+
if (isTemporaryFile && tempFilePath) {
|
|
797
|
+
// Clean up temp file
|
|
798
|
+
try {
|
|
799
|
+
const { promises: fs } = await import('fs');
|
|
800
|
+
await fs.unlink(tempFilePath);
|
|
801
|
+
}
|
|
802
|
+
catch { }
|
|
803
|
+
}
|
|
804
|
+
throw new Error(`Unsupported image format. Supported formats: ${validExtensions.join(', ')}`);
|
|
805
|
+
}
|
|
806
|
+
// Prepare factory options for multimodal mode
|
|
807
|
+
const factoryOptions = {
|
|
808
|
+
mode: 'multimodal' // Always use multimodal mode for image ingestion
|
|
809
|
+
};
|
|
810
|
+
if (args.model) {
|
|
811
|
+
factoryOptions.embeddingModel = args.model;
|
|
812
|
+
}
|
|
813
|
+
else {
|
|
814
|
+
factoryOptions.embeddingModel = 'Xenova/clip-vit-base-patch32'; // Default CLIP model
|
|
815
|
+
}
|
|
816
|
+
if (args.rerank_strategy) {
|
|
817
|
+
factoryOptions.rerankingStrategy = args.rerank_strategy;
|
|
818
|
+
}
|
|
819
|
+
else {
|
|
820
|
+
factoryOptions.rerankingStrategy = 'text-derived'; // Default for multimodal
|
|
821
|
+
}
|
|
822
|
+
// Create and run ingestion pipeline
|
|
823
|
+
const pipeline = await TextIngestionFactory.create(config.db_file, config.index_file, factoryOptions);
|
|
824
|
+
try {
|
|
825
|
+
const result = await pipeline.ingestFile(resolvedPath);
|
|
826
|
+
// Reset search engine initialization flag since index may have changed
|
|
827
|
+
this.isSearchEngineInitialized = false;
|
|
828
|
+
this.searchEngine = null;
|
|
829
|
+
// Format results for MCP response
|
|
830
|
+
const ingestionSummary = {
|
|
831
|
+
source: isUrl ? source : resolvedPath,
|
|
832
|
+
source_type: isUrl ? 'url' : 'file',
|
|
833
|
+
mode: 'multimodal',
|
|
834
|
+
model: args.model || 'Xenova/clip-vit-base-patch32',
|
|
835
|
+
reranking_strategy: args.rerank_strategy || 'text-derived',
|
|
836
|
+
documents_processed: result.documentsProcessed,
|
|
837
|
+
chunks_created: result.chunksCreated,
|
|
838
|
+
embeddings_generated: result.embeddingsGenerated,
|
|
839
|
+
document_errors: result.documentErrors,
|
|
840
|
+
embedding_errors: result.embeddingErrors,
|
|
841
|
+
processing_time_ms: result.processingTimeMs,
|
|
842
|
+
processing_time_seconds: Math.round(result.processingTimeMs / 1000 * 100) / 100,
|
|
843
|
+
content_type: 'image',
|
|
844
|
+
title: args.title || (isUrl ? source.split('/').pop() : resolvedPath.split(/[/\\]/).pop()),
|
|
845
|
+
metadata: args.metadata || {},
|
|
846
|
+
success: true
|
|
847
|
+
};
|
|
848
|
+
return {
|
|
849
|
+
content: [
|
|
850
|
+
{
|
|
851
|
+
type: 'text',
|
|
852
|
+
text: JSON.stringify(ingestionSummary, null, 2),
|
|
853
|
+
},
|
|
854
|
+
],
|
|
855
|
+
};
|
|
856
|
+
}
|
|
857
|
+
finally {
|
|
858
|
+
await pipeline.cleanup();
|
|
859
|
+
// Clean up temporary file if it was downloaded
|
|
860
|
+
if (isTemporaryFile && tempFilePath) {
|
|
861
|
+
try {
|
|
862
|
+
const { promises: fs } = await import('fs');
|
|
863
|
+
await fs.unlink(tempFilePath);
|
|
864
|
+
console.error(`🧹 Cleaned up temporary file: ${tempFilePath}`);
|
|
865
|
+
}
|
|
866
|
+
catch (cleanupError) {
|
|
867
|
+
console.error(`⚠️ Failed to clean up temporary file: ${cleanupError}`);
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
catch (error) {
|
|
873
|
+
// Handle model mismatch errors specifically
|
|
874
|
+
if (error instanceof Error && error.message.includes('Model mismatch detected')) {
|
|
875
|
+
const modelMismatchError = {
|
|
876
|
+
error: 'MODEL_MISMATCH',
|
|
877
|
+
message: 'Cannot perform image ingestion due to model mismatch',
|
|
878
|
+
details: error.message,
|
|
879
|
+
resolution: {
|
|
880
|
+
action: 'manual_intervention_required',
|
|
881
|
+
explanation: 'The embedding model configuration does not match the indexed data. Please verify your setup before proceeding.',
|
|
882
|
+
options: [
|
|
883
|
+
'Check if the model mismatch is intentional',
|
|
884
|
+
'If you want to use a different model, manually run the rebuild_index tool',
|
|
885
|
+
'Verify your model configuration matches your indexing setup'
|
|
886
|
+
],
|
|
887
|
+
warning: 'Rebuilding will regenerate all embeddings and may take significant time'
|
|
888
|
+
}
|
|
889
|
+
};
|
|
890
|
+
return {
|
|
891
|
+
content: [
|
|
892
|
+
{
|
|
893
|
+
type: 'text',
|
|
894
|
+
text: JSON.stringify(modelMismatchError, null, 2),
|
|
895
|
+
},
|
|
896
|
+
],
|
|
897
|
+
};
|
|
898
|
+
}
|
|
899
|
+
// Re-throw other errors to be handled by the main error handler
|
|
900
|
+
throw error;
|
|
901
|
+
}
|
|
902
|
+
}
|
|
446
903
|
/**
|
|
447
904
|
* Handle rebuild index tool calls
|
|
448
905
|
* Wraps existing rebuild functionality as MCP tool
|
|
449
906
|
*/
|
|
450
907
|
async handleRebuildIndex(_args) {
|
|
451
908
|
try {
|
|
452
|
-
//
|
|
453
|
-
await
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
909
|
+
// Create ingestion pipeline with force rebuild using factory
|
|
910
|
+
const pipeline = await TextIngestionFactory.create(config.db_file, config.index_file, { forceRebuild: true });
|
|
911
|
+
try {
|
|
912
|
+
// Get all documents from database and re-ingest them
|
|
913
|
+
const db = await openDatabase(config.db_file);
|
|
914
|
+
try {
|
|
915
|
+
const documents = await db.all('SELECT DISTINCT source FROM documents ORDER BY source');
|
|
916
|
+
if (documents.length === 0) {
|
|
917
|
+
throw new Error('No documents found in database. Nothing to rebuild.');
|
|
918
|
+
}
|
|
919
|
+
let totalResult = {
|
|
920
|
+
documentsProcessed: 0,
|
|
921
|
+
chunksCreated: 0,
|
|
922
|
+
embeddingsGenerated: 0,
|
|
923
|
+
documentErrors: 0,
|
|
924
|
+
embeddingErrors: 0,
|
|
925
|
+
processingTimeMs: 0
|
|
926
|
+
};
|
|
927
|
+
// Re-ingest each document
|
|
928
|
+
for (const doc of documents) {
|
|
929
|
+
if (existsSync(doc.source)) {
|
|
930
|
+
const result = await pipeline.ingestFile(doc.source);
|
|
931
|
+
totalResult.documentsProcessed += result.documentsProcessed;
|
|
932
|
+
totalResult.chunksCreated += result.chunksCreated;
|
|
933
|
+
totalResult.embeddingsGenerated += result.embeddingsGenerated;
|
|
934
|
+
totalResult.documentErrors += result.documentErrors;
|
|
935
|
+
totalResult.embeddingErrors += result.embeddingErrors;
|
|
936
|
+
totalResult.processingTimeMs += result.processingTimeMs;
|
|
937
|
+
}
|
|
938
|
+
else {
|
|
939
|
+
totalResult.documentErrors++;
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
// Reset search engine initialization flag since index was rebuilt
|
|
943
|
+
this.isSearchEngineInitialized = false;
|
|
944
|
+
this.searchEngine = null;
|
|
945
|
+
const rebuildSummary = {
|
|
946
|
+
operation: 'rebuild_index',
|
|
947
|
+
success: true,
|
|
948
|
+
message: 'Vector index has been successfully rebuilt. All embeddings have been regenerated with the current model.',
|
|
949
|
+
documents_processed: totalResult.documentsProcessed,
|
|
950
|
+
chunks_created: totalResult.chunksCreated,
|
|
951
|
+
embeddings_generated: totalResult.embeddingsGenerated,
|
|
952
|
+
document_errors: totalResult.documentErrors,
|
|
953
|
+
embedding_errors: totalResult.embeddingErrors,
|
|
954
|
+
processing_time_ms: totalResult.processingTimeMs,
|
|
955
|
+
processing_time_seconds: Math.round(totalResult.processingTimeMs / 1000 * 100) / 100
|
|
956
|
+
};
|
|
957
|
+
return {
|
|
958
|
+
content: [
|
|
959
|
+
{
|
|
960
|
+
type: 'text',
|
|
961
|
+
text: JSON.stringify(rebuildSummary, null, 2),
|
|
962
|
+
},
|
|
963
|
+
],
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
finally {
|
|
967
|
+
await db.close();
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
finally {
|
|
971
|
+
await pipeline.cleanup();
|
|
972
|
+
}
|
|
470
973
|
}
|
|
471
974
|
catch (error) {
|
|
472
975
|
throw new Error(`Index rebuild failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
@@ -485,7 +988,7 @@ class RagLiteMCPServer {
|
|
|
485
988
|
};
|
|
486
989
|
// Get model information and compatibility status
|
|
487
990
|
const { getModelDefaults } = await import('./config.js');
|
|
488
|
-
const { getStoredModelInfo } = await import('./db.js');
|
|
991
|
+
const { getStoredModelInfo } = await import('./core/db.js');
|
|
489
992
|
const currentModel = config.embedding_model;
|
|
490
993
|
const currentDefaults = getModelDefaults(currentModel);
|
|
491
994
|
stats.model_info = {
|
|
@@ -581,7 +1084,478 @@ class RagLiteMCPServer {
|
|
|
581
1084
|
}
|
|
582
1085
|
}
|
|
583
1086
|
/**
|
|
584
|
-
*
|
|
1087
|
+
* Handle get mode info tool calls
|
|
1088
|
+
* Provides information about the current system mode and configuration
|
|
1089
|
+
*/
|
|
1090
|
+
async handleGetModeInfo(_args) {
|
|
1091
|
+
try {
|
|
1092
|
+
const modeInfo = {
|
|
1093
|
+
database_exists: existsSync(config.db_file),
|
|
1094
|
+
index_exists: existsSync(config.index_file)
|
|
1095
|
+
};
|
|
1096
|
+
if (!modeInfo.database_exists) {
|
|
1097
|
+
modeInfo.mode_status = 'No database found - system not initialized';
|
|
1098
|
+
modeInfo.default_mode = 'text';
|
|
1099
|
+
modeInfo.message = 'Run ingestion first to initialize the system with a specific mode';
|
|
1100
|
+
return {
|
|
1101
|
+
content: [
|
|
1102
|
+
{
|
|
1103
|
+
type: 'text',
|
|
1104
|
+
text: JSON.stringify(modeInfo, null, 2),
|
|
1105
|
+
},
|
|
1106
|
+
],
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
// Import mode detection service
|
|
1110
|
+
const { ModeDetectionService } = await import('./core/mode-detection-service.js');
|
|
1111
|
+
const modeService = new ModeDetectionService(config.db_file);
|
|
1112
|
+
try {
|
|
1113
|
+
const systemInfo = await modeService.detectMode();
|
|
1114
|
+
modeInfo.mode_status = 'Mode detected from database';
|
|
1115
|
+
modeInfo.current_mode = systemInfo.mode;
|
|
1116
|
+
modeInfo.model_name = systemInfo.modelName;
|
|
1117
|
+
modeInfo.model_type = systemInfo.modelType;
|
|
1118
|
+
modeInfo.model_dimensions = systemInfo.modelDimensions;
|
|
1119
|
+
modeInfo.supported_content_types = systemInfo.supportedContentTypes;
|
|
1120
|
+
modeInfo.reranking_strategy = systemInfo.rerankingStrategy;
|
|
1121
|
+
if (systemInfo.rerankingModel) {
|
|
1122
|
+
modeInfo.reranking_model = systemInfo.rerankingModel;
|
|
1123
|
+
}
|
|
1124
|
+
if (systemInfo.rerankingConfig) {
|
|
1125
|
+
modeInfo.reranking_config = systemInfo.rerankingConfig;
|
|
1126
|
+
}
|
|
1127
|
+
modeInfo.created_at = systemInfo.createdAt;
|
|
1128
|
+
modeInfo.updated_at = systemInfo.updatedAt;
|
|
1129
|
+
// Add mode-specific capabilities
|
|
1130
|
+
if (systemInfo.mode === 'text') {
|
|
1131
|
+
modeInfo.capabilities = {
|
|
1132
|
+
text_search: true,
|
|
1133
|
+
image_search: false,
|
|
1134
|
+
multimodal_reranking: false,
|
|
1135
|
+
supported_file_types: ['md', 'txt']
|
|
1136
|
+
};
|
|
1137
|
+
}
|
|
1138
|
+
else if (systemInfo.mode === 'multimodal') {
|
|
1139
|
+
modeInfo.capabilities = {
|
|
1140
|
+
text_search: true,
|
|
1141
|
+
image_search: true,
|
|
1142
|
+
multimodal_reranking: true,
|
|
1143
|
+
supported_file_types: ['md', 'txt', 'jpg', 'png', 'gif', 'webp']
|
|
1144
|
+
};
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
catch (error) {
|
|
1148
|
+
modeInfo.mode_status = 'Error detecting mode from database';
|
|
1149
|
+
modeInfo.error = error instanceof Error ? error.message : 'Unknown error';
|
|
1150
|
+
modeInfo.fallback_mode = 'text';
|
|
1151
|
+
modeInfo.message = 'System will fall back to text mode. Consider rebuilding the database.';
|
|
1152
|
+
}
|
|
1153
|
+
return {
|
|
1154
|
+
content: [
|
|
1155
|
+
{
|
|
1156
|
+
type: 'text',
|
|
1157
|
+
text: JSON.stringify(modeInfo, null, 2),
|
|
1158
|
+
},
|
|
1159
|
+
],
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
catch (error) {
|
|
1163
|
+
throw new Error(`Failed to get mode info: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
/**
|
|
1167
|
+
* Handle multimodal search tool calls with content type filtering
|
|
1168
|
+
* Extends regular search with multimodal capabilities and content type filtering
|
|
1169
|
+
*/
|
|
1170
|
+
async handleMultimodalSearch(args) {
|
|
1171
|
+
try {
|
|
1172
|
+
// Validate arguments (same as regular search)
|
|
1173
|
+
if (!args.query || typeof args.query !== 'string') {
|
|
1174
|
+
throw new Error('Query parameter is required and must be a string');
|
|
1175
|
+
}
|
|
1176
|
+
if (args.query.trim().length === 0) {
|
|
1177
|
+
throw new Error('Query cannot be empty');
|
|
1178
|
+
}
|
|
1179
|
+
if (args.query.length > 500) {
|
|
1180
|
+
throw new Error('Query is too long (maximum 500 characters)');
|
|
1181
|
+
}
|
|
1182
|
+
// Validate optional parameters
|
|
1183
|
+
if (args.top_k !== undefined) {
|
|
1184
|
+
if (typeof args.top_k !== 'number' || args.top_k < 1 || args.top_k > 100) {
|
|
1185
|
+
throw new Error('top_k must be a number between 1 and 100');
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
if (args.rerank !== undefined && typeof args.rerank !== 'boolean') {
|
|
1189
|
+
throw new Error('rerank must be a boolean');
|
|
1190
|
+
}
|
|
1191
|
+
if (args.content_type !== undefined) {
|
|
1192
|
+
const validContentTypes = ['text', 'image', 'pdf', 'docx'];
|
|
1193
|
+
if (!validContentTypes.includes(args.content_type)) {
|
|
1194
|
+
throw new Error(`content_type must be one of: ${validContentTypes.join(', ')}`);
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
// Check if database and index exist
|
|
1198
|
+
if (!existsSync(config.db_file)) {
|
|
1199
|
+
throw new Error('No database found. You need to ingest documents first using the ingest tool.');
|
|
1200
|
+
}
|
|
1201
|
+
if (!existsSync(config.index_file)) {
|
|
1202
|
+
throw new Error('No vector index found. The ingestion may not have completed successfully. Try using the ingest tool or rebuild_index tool.');
|
|
1203
|
+
}
|
|
1204
|
+
// Initialize search engine if needed
|
|
1205
|
+
if (!this.isSearchEngineInitialized) {
|
|
1206
|
+
await this.initializeSearchEngine();
|
|
1207
|
+
}
|
|
1208
|
+
// Prepare search options with content type filtering
|
|
1209
|
+
const searchOptions = {
|
|
1210
|
+
top_k: args.top_k || config.top_k || 10,
|
|
1211
|
+
rerank: args.rerank !== undefined ? args.rerank : config.rerank_enabled,
|
|
1212
|
+
contentType: args.content_type // Add content type filtering
|
|
1213
|
+
};
|
|
1214
|
+
// Perform search using existing search functionality
|
|
1215
|
+
const startTime = Date.now();
|
|
1216
|
+
const results = await this.searchEngine.search(args.query, searchOptions);
|
|
1217
|
+
const searchTime = Date.now() - startTime;
|
|
1218
|
+
// Format results for MCP response with content type information and image data
|
|
1219
|
+
const formattedResults = {
|
|
1220
|
+
query: args.query,
|
|
1221
|
+
content_type_filter: args.content_type || 'all',
|
|
1222
|
+
results_count: results.length,
|
|
1223
|
+
search_time_ms: searchTime,
|
|
1224
|
+
results: await Promise.all(results.map(async (result, index) => {
|
|
1225
|
+
const formattedResult = {
|
|
1226
|
+
rank: index + 1,
|
|
1227
|
+
score: Math.round(result.score * 100) / 100,
|
|
1228
|
+
content_type: result.contentType,
|
|
1229
|
+
document: {
|
|
1230
|
+
id: result.document.id,
|
|
1231
|
+
title: result.document.title,
|
|
1232
|
+
source: result.document.source,
|
|
1233
|
+
content_type: result.document.contentType
|
|
1234
|
+
},
|
|
1235
|
+
text: result.content,
|
|
1236
|
+
metadata: result.metadata
|
|
1237
|
+
};
|
|
1238
|
+
// For image content, include base64-encoded image data for MCP clients
|
|
1239
|
+
if (result.contentType === 'image' && result.document.contentId) {
|
|
1240
|
+
try {
|
|
1241
|
+
const imageData = await this.searchEngine.getContent(result.document.contentId, 'base64');
|
|
1242
|
+
formattedResult.image_data = imageData;
|
|
1243
|
+
formattedResult.image_format = 'base64';
|
|
1244
|
+
}
|
|
1245
|
+
catch (error) {
|
|
1246
|
+
// If image retrieval fails, include error but don't fail the entire search
|
|
1247
|
+
formattedResult.image_error = error instanceof Error ? error.message : 'Failed to retrieve image';
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
return formattedResult;
|
|
1251
|
+
}))
|
|
1252
|
+
};
|
|
1253
|
+
return {
|
|
1254
|
+
content: [
|
|
1255
|
+
{
|
|
1256
|
+
type: 'text',
|
|
1257
|
+
text: JSON.stringify(formattedResults, null, 2),
|
|
1258
|
+
},
|
|
1259
|
+
],
|
|
1260
|
+
};
|
|
1261
|
+
}
|
|
1262
|
+
catch (error) {
|
|
1263
|
+
// Handle the same errors as regular search
|
|
1264
|
+
if (error instanceof Error && error.message.includes('Model mismatch detected')) {
|
|
1265
|
+
const modelMismatchError = {
|
|
1266
|
+
error: 'MODEL_MISMATCH',
|
|
1267
|
+
message: 'Cannot perform multimodal search due to model mismatch',
|
|
1268
|
+
details: error.message,
|
|
1269
|
+
resolution: {
|
|
1270
|
+
action: 'manual_intervention_required',
|
|
1271
|
+
explanation: 'The embedding model configuration does not match the indexed data. Please verify your setup before proceeding.',
|
|
1272
|
+
options: [
|
|
1273
|
+
'Check if the model mismatch is intentional',
|
|
1274
|
+
'If you want to use a different model, manually run the rebuild_index tool',
|
|
1275
|
+
'Verify your model configuration matches your indexing setup'
|
|
1276
|
+
],
|
|
1277
|
+
warning: 'Rebuilding will regenerate all embeddings and may take significant time'
|
|
1278
|
+
}
|
|
1279
|
+
};
|
|
1280
|
+
return {
|
|
1281
|
+
content: [
|
|
1282
|
+
{
|
|
1283
|
+
type: 'text',
|
|
1284
|
+
text: JSON.stringify(modelMismatchError, null, 2),
|
|
1285
|
+
},
|
|
1286
|
+
],
|
|
1287
|
+
};
|
|
1288
|
+
}
|
|
1289
|
+
// Re-throw other errors to be handled by the main error handler
|
|
1290
|
+
throw error;
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
/**
|
|
1294
|
+
* Handle list supported models tool calls
|
|
1295
|
+
* Lists all supported embedding models with their capabilities
|
|
1296
|
+
*/
|
|
1297
|
+
async handleListSupportedModels(args) {
|
|
1298
|
+
try {
|
|
1299
|
+
// Import model registry
|
|
1300
|
+
const { ModelRegistry } = await import('./core/model-registry.js');
|
|
1301
|
+
const { getSupportedModelsForContentType } = await import('./core/embedder-factory.js');
|
|
1302
|
+
let models;
|
|
1303
|
+
// Filter by model type if specified
|
|
1304
|
+
if (args.model_type) {
|
|
1305
|
+
models = ModelRegistry.getSupportedModels(args.model_type);
|
|
1306
|
+
}
|
|
1307
|
+
else if (args.content_type) {
|
|
1308
|
+
// Filter by content type if specified
|
|
1309
|
+
models = getSupportedModelsForContentType(args.content_type);
|
|
1310
|
+
}
|
|
1311
|
+
else {
|
|
1312
|
+
// Get all models
|
|
1313
|
+
models = ModelRegistry.getSupportedModels();
|
|
1314
|
+
}
|
|
1315
|
+
// Get detailed information for each model
|
|
1316
|
+
const modelDetails = models.map(modelName => {
|
|
1317
|
+
const info = ModelRegistry.getModelInfo(modelName);
|
|
1318
|
+
return {
|
|
1319
|
+
name: modelName,
|
|
1320
|
+
type: info?.type,
|
|
1321
|
+
dimensions: info?.dimensions,
|
|
1322
|
+
supported_content_types: info?.supportedContentTypes || [],
|
|
1323
|
+
memory_requirement: info?.requirements?.minimumMemory,
|
|
1324
|
+
description: `${info?.type} model for ${info?.supportedContentTypes?.join(', ')} content`,
|
|
1325
|
+
is_default: ModelRegistry.getDefaultModel(info?.type) === modelName,
|
|
1326
|
+
capabilities: {
|
|
1327
|
+
supports_text: info?.capabilities?.supportsText || false,
|
|
1328
|
+
supports_images: info?.capabilities?.supportsImages || false,
|
|
1329
|
+
supports_batch_processing: info?.capabilities?.supportsBatchProcessing || false,
|
|
1330
|
+
max_batch_size: info?.capabilities?.maxBatchSize,
|
|
1331
|
+
max_text_length: info?.capabilities?.maxTextLength,
|
|
1332
|
+
supported_image_formats: info?.capabilities?.supportedImageFormats || []
|
|
1333
|
+
},
|
|
1334
|
+
requirements: {
|
|
1335
|
+
transformers_js_version: info?.requirements?.transformersJsVersion,
|
|
1336
|
+
minimum_memory_mb: info?.requirements?.minimumMemory,
|
|
1337
|
+
required_features: info?.requirements?.requiredFeatures || [],
|
|
1338
|
+
platform_support: info?.requirements?.platformSupport || []
|
|
1339
|
+
}
|
|
1340
|
+
};
|
|
1341
|
+
});
|
|
1342
|
+
const response = {
|
|
1343
|
+
filter_applied: {
|
|
1344
|
+
model_type: args.model_type || 'all',
|
|
1345
|
+
content_type: args.content_type || 'all'
|
|
1346
|
+
},
|
|
1347
|
+
total_models: modelDetails.length,
|
|
1348
|
+
models: modelDetails,
|
|
1349
|
+
model_types: {
|
|
1350
|
+
'sentence-transformer': 'Text-only embedding models optimized for semantic similarity',
|
|
1351
|
+
'clip': 'Multimodal models supporting both text and image embeddings'
|
|
1352
|
+
},
|
|
1353
|
+
usage_examples: {
|
|
1354
|
+
text_only: 'Use sentence-transformer models for text documents, markdown files, and text-based search',
|
|
1355
|
+
multimodal: 'Use CLIP models when working with mixed content including images, diagrams, and visual content'
|
|
1356
|
+
}
|
|
1357
|
+
};
|
|
1358
|
+
return {
|
|
1359
|
+
content: [
|
|
1360
|
+
{
|
|
1361
|
+
type: 'text',
|
|
1362
|
+
text: JSON.stringify(response, null, 2),
|
|
1363
|
+
},
|
|
1364
|
+
],
|
|
1365
|
+
};
|
|
1366
|
+
}
|
|
1367
|
+
catch (error) {
|
|
1368
|
+
throw new Error(`Failed to list supported models: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Handle list reranking strategies tool calls
|
|
1373
|
+
* Lists all supported reranking strategies for different modes
|
|
1374
|
+
*/
|
|
1375
|
+
async handleListRerankingStrategies(args) {
|
|
1376
|
+
try {
|
|
1377
|
+
// Import reranking configuration
|
|
1378
|
+
const { getSupportedStrategies, getDefaultRerankingConfig } = await import('./core/reranking-config.js');
|
|
1379
|
+
const modes = args.mode ? [args.mode] : ['text', 'multimodal'];
|
|
1380
|
+
const strategiesByMode = {};
|
|
1381
|
+
for (const mode of modes) {
|
|
1382
|
+
const supportedStrategies = getSupportedStrategies(mode);
|
|
1383
|
+
const defaultConfig = getDefaultRerankingConfig(mode);
|
|
1384
|
+
strategiesByMode[mode] = {
|
|
1385
|
+
default_strategy: defaultConfig.strategy,
|
|
1386
|
+
mode_description: mode === 'text'
|
|
1387
|
+
? 'Text-only mode optimized for document and text-based content'
|
|
1388
|
+
: 'Multimodal mode supporting mixed text and image content',
|
|
1389
|
+
supported_strategies: supportedStrategies.map(strategy => {
|
|
1390
|
+
const strategyInfo = {
|
|
1391
|
+
name: strategy,
|
|
1392
|
+
is_default: strategy === defaultConfig.strategy,
|
|
1393
|
+
performance_impact: 'medium'
|
|
1394
|
+
};
|
|
1395
|
+
// Add descriptions for each strategy
|
|
1396
|
+
switch (strategy) {
|
|
1397
|
+
case 'cross-encoder':
|
|
1398
|
+
strategyInfo.description = 'Uses cross-encoder models to rerank results based on query-document relevance';
|
|
1399
|
+
strategyInfo.requirements = ['Cross-encoder model (e.g., ms-marco-MiniLM-L-6-v2)'];
|
|
1400
|
+
strategyInfo.supported_content_types = ['text'];
|
|
1401
|
+
strategyInfo.performance_impact = 'high';
|
|
1402
|
+
strategyInfo.accuracy = 'high';
|
|
1403
|
+
strategyInfo.use_cases = ['Text documents', 'Academic papers', 'Technical documentation'];
|
|
1404
|
+
break;
|
|
1405
|
+
case 'text-derived':
|
|
1406
|
+
strategyInfo.description = 'Converts images to text descriptions using image-to-text models, then applies cross-encoder reranking';
|
|
1407
|
+
strategyInfo.requirements = ['Image-to-text model (e.g., vit-gpt2-image-captioning)', 'Cross-encoder model'];
|
|
1408
|
+
strategyInfo.supported_content_types = ['text', 'image'];
|
|
1409
|
+
strategyInfo.performance_impact = 'high';
|
|
1410
|
+
strategyInfo.accuracy = 'high';
|
|
1411
|
+
strategyInfo.use_cases = ['Mixed content with images', 'Visual documentation', 'Diagrams and charts'];
|
|
1412
|
+
break;
|
|
1413
|
+
case 'metadata':
|
|
1414
|
+
strategyInfo.description = 'Uses file metadata, filenames, and content properties for scoring without model inference';
|
|
1415
|
+
strategyInfo.requirements = ['None - uses file system metadata only'];
|
|
1416
|
+
strategyInfo.supported_content_types = ['text', 'image', 'pdf', 'docx'];
|
|
1417
|
+
strategyInfo.performance_impact = 'low';
|
|
1418
|
+
strategyInfo.accuracy = 'medium';
|
|
1419
|
+
strategyInfo.use_cases = ['Fast retrieval', 'Filename-based search', 'Content type filtering'];
|
|
1420
|
+
break;
|
|
1421
|
+
case 'hybrid':
|
|
1422
|
+
strategyInfo.description = 'Combines multiple reranking signals (semantic + metadata) with configurable weights';
|
|
1423
|
+
strategyInfo.requirements = ['Text-derived reranker', 'Metadata reranker'];
|
|
1424
|
+
strategyInfo.supported_content_types = ['text', 'image', 'pdf', 'docx'];
|
|
1425
|
+
strategyInfo.performance_impact = 'high';
|
|
1426
|
+
strategyInfo.accuracy = 'very high';
|
|
1427
|
+
strategyInfo.use_cases = ['Best overall accuracy', 'Complex multimodal collections', 'Production systems'];
|
|
1428
|
+
strategyInfo.default_weights = { semantic: 0.7, metadata: 0.3 };
|
|
1429
|
+
break;
|
|
1430
|
+
case 'disabled':
|
|
1431
|
+
strategyInfo.description = 'No reranking applied - results ordered by vector similarity scores only';
|
|
1432
|
+
strategyInfo.requirements = ['None'];
|
|
1433
|
+
strategyInfo.supported_content_types = ['text', 'image', 'pdf', 'docx'];
|
|
1434
|
+
strategyInfo.performance_impact = 'none';
|
|
1435
|
+
strategyInfo.accuracy = 'baseline';
|
|
1436
|
+
strategyInfo.use_cases = ['Maximum performance', 'Simple similarity search', 'Development/testing'];
|
|
1437
|
+
break;
|
|
1438
|
+
}
|
|
1439
|
+
return strategyInfo;
|
|
1440
|
+
})
|
|
1441
|
+
};
|
|
1442
|
+
}
|
|
1443
|
+
const response = {
|
|
1444
|
+
filter_applied: {
|
|
1445
|
+
mode: args.mode || 'all'
|
|
1446
|
+
},
|
|
1447
|
+
strategies_by_mode: strategiesByMode,
|
|
1448
|
+
recommendations: {
|
|
1449
|
+
text_mode: 'Use cross-encoder for best accuracy, disabled for best performance',
|
|
1450
|
+
multimodal_mode: 'Use hybrid for best accuracy, text-derived for good balance, metadata for fast retrieval',
|
|
1451
|
+
development: 'Start with disabled or metadata for fast iteration, upgrade to cross-encoder/text-derived for production'
|
|
1452
|
+
}
|
|
1453
|
+
};
|
|
1454
|
+
return {
|
|
1455
|
+
content: [
|
|
1456
|
+
{
|
|
1457
|
+
type: 'text',
|
|
1458
|
+
text: JSON.stringify(response, null, 2),
|
|
1459
|
+
},
|
|
1460
|
+
],
|
|
1461
|
+
};
|
|
1462
|
+
}
|
|
1463
|
+
catch (error) {
|
|
1464
|
+
throw new Error(`Failed to list reranking strategies: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
/**
|
|
1468
|
+
* Handle get system stats tool calls with mode-specific metrics
|
|
1469
|
+
* Provides comprehensive system statistics including mode-specific data
|
|
1470
|
+
*/
|
|
1471
|
+
async handleGetSystemStats(args) {
|
|
1472
|
+
try {
|
|
1473
|
+
// Start with basic stats from existing get_stats handler
|
|
1474
|
+
const basicStats = await this.handleGetStats({});
|
|
1475
|
+
const basicStatsData = JSON.parse(basicStats.content[0].text);
|
|
1476
|
+
// Enhanced stats with mode-specific information
|
|
1477
|
+
const enhancedStats = {
|
|
1478
|
+
...basicStatsData,
|
|
1479
|
+
mode_specific_metrics: {}
|
|
1480
|
+
};
|
|
1481
|
+
// Add mode detection information
|
|
1482
|
+
if (basicStatsData.database_exists) {
|
|
1483
|
+
try {
|
|
1484
|
+
const { ModeDetectionService } = await import('./core/mode-detection-service.js');
|
|
1485
|
+
const modeService = new ModeDetectionService(config.db_file);
|
|
1486
|
+
const systemInfo = await modeService.detectMode();
|
|
1487
|
+
enhancedStats.mode_specific_metrics = {
|
|
1488
|
+
current_mode: systemInfo.mode,
|
|
1489
|
+
model_name: systemInfo.modelName,
|
|
1490
|
+
model_type: systemInfo.modelType,
|
|
1491
|
+
model_dimensions: systemInfo.modelDimensions,
|
|
1492
|
+
supported_content_types: systemInfo.supportedContentTypes,
|
|
1493
|
+
reranking_strategy: systemInfo.rerankingStrategy
|
|
1494
|
+
};
|
|
1495
|
+
// Add content breakdown if requested
|
|
1496
|
+
if (args.include_content_breakdown) {
|
|
1497
|
+
const db = await openDatabase(config.db_file);
|
|
1498
|
+
try {
|
|
1499
|
+
// Get document count by content type
|
|
1500
|
+
const docsByType = await db.all(`
|
|
1501
|
+
SELECT content_type, COUNT(*) as count
|
|
1502
|
+
FROM documents
|
|
1503
|
+
GROUP BY content_type
|
|
1504
|
+
`);
|
|
1505
|
+
// Get chunk count by content type
|
|
1506
|
+
const chunksByType = await db.all(`
|
|
1507
|
+
SELECT content_type, COUNT(*) as count
|
|
1508
|
+
FROM chunks
|
|
1509
|
+
GROUP BY content_type
|
|
1510
|
+
`);
|
|
1511
|
+
enhancedStats.content_breakdown = {
|
|
1512
|
+
documents_by_type: docsByType.reduce((acc, row) => {
|
|
1513
|
+
acc[row.content_type] = row.count;
|
|
1514
|
+
return acc;
|
|
1515
|
+
}, {}),
|
|
1516
|
+
chunks_by_type: chunksByType.reduce((acc, row) => {
|
|
1517
|
+
acc[row.content_type] = row.count;
|
|
1518
|
+
return acc;
|
|
1519
|
+
}, {})
|
|
1520
|
+
};
|
|
1521
|
+
}
|
|
1522
|
+
finally {
|
|
1523
|
+
await db.close();
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
// Add performance metrics if requested
|
|
1527
|
+
if (args.include_performance && this.isSearchEngineInitialized && this.searchEngine) {
|
|
1528
|
+
const searchStats = await this.searchEngine.getStats();
|
|
1529
|
+
enhancedStats.performance_metrics = {
|
|
1530
|
+
search_engine_initialized: true,
|
|
1531
|
+
index_size: searchStats.indexSize,
|
|
1532
|
+
reranking_enabled: searchStats.rerankingEnabled,
|
|
1533
|
+
total_chunks: searchStats.totalChunks,
|
|
1534
|
+
// Add timing information if available
|
|
1535
|
+
last_search_time_ms: undefined, // Would need to track this
|
|
1536
|
+
average_search_time_ms: undefined // Would need to track this
|
|
1537
|
+
};
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
catch (error) {
|
|
1541
|
+
enhancedStats.mode_specific_metrics.error = error instanceof Error ? error.message : 'Unknown error';
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
return {
|
|
1545
|
+
content: [
|
|
1546
|
+
{
|
|
1547
|
+
type: 'text',
|
|
1548
|
+
text: JSON.stringify(enhancedStats, null, 2),
|
|
1549
|
+
},
|
|
1550
|
+
],
|
|
1551
|
+
};
|
|
1552
|
+
}
|
|
1553
|
+
catch (error) {
|
|
1554
|
+
throw new Error(`Failed to get system stats: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
1555
|
+
}
|
|
1556
|
+
}
|
|
1557
|
+
/**
|
|
1558
|
+
* Initialize search engine components using factory
|
|
585
1559
|
* Lazy initialization to avoid startup overhead when not needed
|
|
586
1560
|
*/
|
|
587
1561
|
async initializeSearchEngine() {
|
|
@@ -590,38 +1564,142 @@ class RagLiteMCPServer {
|
|
|
590
1564
|
}
|
|
591
1565
|
try {
|
|
592
1566
|
// Validate configuration
|
|
593
|
-
|
|
594
|
-
//
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
1567
|
+
validateCoreConfig(config);
|
|
1568
|
+
// Create search engine using PolymorphicSearchFactory (auto-detects mode)
|
|
1569
|
+
// This will automatically detect the mode from the database and create the appropriate engine
|
|
1570
|
+
console.error('🎭 MCP Server: Initializing search engine with automatic mode detection...');
|
|
1571
|
+
this.searchEngine = await PolymorphicSearchFactory.create(config.index_file, config.db_file);
|
|
1572
|
+
// Log successful initialization with mode information
|
|
1573
|
+
const stats = await this.searchEngine.getStats();
|
|
1574
|
+
// Try to get mode information for enhanced logging
|
|
1575
|
+
try {
|
|
1576
|
+
const { ModeDetectionService } = await import('./core/mode-detection-service.js');
|
|
1577
|
+
const modeService = new ModeDetectionService(config.db_file);
|
|
1578
|
+
const systemInfo = await modeService.detectMode();
|
|
1579
|
+
console.error(`✅ MCP Server: Search engine initialized successfully`);
|
|
1580
|
+
console.error(`🎭 Mode: ${systemInfo.mode} | Model: ${systemInfo.modelName}`);
|
|
1581
|
+
console.error(`📊 Total chunks: ${stats.totalChunks} | Reranking: ${stats.rerankingEnabled ? 'enabled' : 'disabled'}`);
|
|
1582
|
+
console.error(`🔧 Content types: ${systemInfo.supportedContentTypes.join(', ')}`);
|
|
1583
|
+
if (systemInfo.mode === 'multimodal') {
|
|
1584
|
+
console.error(`🌈 Multimodal capabilities: Text + Image processing enabled`);
|
|
1585
|
+
console.error(`⚡ Reranking strategy: ${systemInfo.rerankingStrategy}`);
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
catch (modeError) {
|
|
1589
|
+
// Fallback to basic logging if mode detection fails
|
|
1590
|
+
console.error(`✅ MCP Server: Search engine initialized successfully`);
|
|
1591
|
+
console.error(`📊 Total chunks: ${stats.totalChunks}, Reranking: ${stats.rerankingEnabled ? 'enabled' : 'disabled'}`);
|
|
1592
|
+
console.error(`⚠️ Mode detection unavailable: ${modeError instanceof Error ? modeError.message : 'Unknown error'}`);
|
|
601
1593
|
}
|
|
602
|
-
// Use the stored model info instead of config.embedding_model
|
|
603
|
-
const { getModelDefaults } = await import('./config.js');
|
|
604
|
-
const modelDefaults = getModelDefaults(storedModelInfo.modelName);
|
|
605
|
-
const embedder = await initializeEmbeddingEngine(storedModelInfo.modelName, modelDefaults.batch_size);
|
|
606
|
-
// Initialize index manager with stored model info
|
|
607
|
-
const { IndexManager } = await import('./index-manager.js');
|
|
608
|
-
const indexManager = new IndexManager(config.index_file, config.db_file, storedModelInfo.dimensions, storedModelInfo.modelName);
|
|
609
|
-
await indexManager.initialize();
|
|
610
|
-
// Create search engine
|
|
611
|
-
this.searchEngine = SearchEngine.createWithComponents(embedder, indexManager, db, config.rerank_enabled);
|
|
612
|
-
await this.searchEngine.initialize();
|
|
613
1594
|
this.isSearchEngineInitialized = true;
|
|
614
1595
|
}
|
|
615
1596
|
catch (error) {
|
|
1597
|
+
// Check if this is a mode detection error
|
|
1598
|
+
if (error instanceof Error && error.message.includes('mode detection')) {
|
|
1599
|
+
console.error('⚠️ MCP Server: Mode detection failed, falling back to text mode');
|
|
1600
|
+
throw new Error(`Mode detection failed: ${error.message}. The system will attempt to fall back to text mode.`);
|
|
1601
|
+
}
|
|
616
1602
|
// Check if this is a model mismatch error and re-throw with more context
|
|
617
1603
|
if (error instanceof Error && (error.message.includes('Model mismatch detected') || error.message.includes('dimension mismatch'))) {
|
|
618
|
-
|
|
1604
|
+
console.error('⚠️ MCP Server: Model compatibility issue detected');
|
|
1605
|
+
// Re-throw the original error - it already has good formatting from factory
|
|
619
1606
|
throw error;
|
|
620
1607
|
}
|
|
621
1608
|
// For other initialization errors, provide a generic wrapper
|
|
1609
|
+
console.error('❌ MCP Server: Search engine initialization failed');
|
|
622
1610
|
throw new Error(`Failed to initialize search engine: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
623
1611
|
}
|
|
624
1612
|
}
|
|
1613
|
+
/**
|
|
1614
|
+
* Detect capabilities of the current database
|
|
1615
|
+
* Returns information about mode, content types, and features
|
|
1616
|
+
*/
|
|
1617
|
+
async detectCapabilities() {
|
|
1618
|
+
// Default capabilities if database doesn't exist
|
|
1619
|
+
if (!existsSync(config.db_file)) {
|
|
1620
|
+
return {
|
|
1621
|
+
mode: 'unknown',
|
|
1622
|
+
contentTypes: [],
|
|
1623
|
+
modelName: 'none',
|
|
1624
|
+
hasImages: false,
|
|
1625
|
+
documentCount: 0
|
|
1626
|
+
};
|
|
1627
|
+
}
|
|
1628
|
+
try {
|
|
1629
|
+
const { ModeDetectionService } = await import('./core/mode-detection-service.js');
|
|
1630
|
+
const modeService = new ModeDetectionService(config.db_file);
|
|
1631
|
+
const systemInfo = await modeService.detectMode();
|
|
1632
|
+
// Check if database has any images
|
|
1633
|
+
const db = await openDatabase(config.db_file);
|
|
1634
|
+
let hasImages = false;
|
|
1635
|
+
let documentCount = 0;
|
|
1636
|
+
try {
|
|
1637
|
+
const imageCount = await db.get("SELECT COUNT(*) as count FROM documents WHERE content_type = 'image'");
|
|
1638
|
+
hasImages = (imageCount?.count || 0) > 0;
|
|
1639
|
+
const docCount = await db.get('SELECT COUNT(*) as count FROM documents');
|
|
1640
|
+
documentCount = docCount?.count || 0;
|
|
1641
|
+
}
|
|
1642
|
+
finally {
|
|
1643
|
+
await db.close();
|
|
1644
|
+
}
|
|
1645
|
+
return {
|
|
1646
|
+
mode: systemInfo.mode,
|
|
1647
|
+
contentTypes: systemInfo.supportedContentTypes,
|
|
1648
|
+
modelName: systemInfo.modelName,
|
|
1649
|
+
hasImages,
|
|
1650
|
+
documentCount
|
|
1651
|
+
};
|
|
1652
|
+
}
|
|
1653
|
+
catch (error) {
|
|
1654
|
+
// If detection fails, return unknown
|
|
1655
|
+
return {
|
|
1656
|
+
mode: 'unknown',
|
|
1657
|
+
contentTypes: [],
|
|
1658
|
+
modelName: 'unknown',
|
|
1659
|
+
hasImages: false,
|
|
1660
|
+
documentCount: 0
|
|
1661
|
+
};
|
|
1662
|
+
}
|
|
1663
|
+
}
|
|
1664
|
+
/**
|
|
1665
|
+
* Generate dynamic search tool description based on actual capabilities
|
|
1666
|
+
*/
|
|
1667
|
+
generateSearchDescription(capabilities) {
|
|
1668
|
+
const baseDesc = 'Search indexed documents using semantic similarity.';
|
|
1669
|
+
if (capabilities.mode === 'unknown' || capabilities.documentCount === 0) {
|
|
1670
|
+
return `${baseDesc} Database not initialized - ingest documents first.`;
|
|
1671
|
+
}
|
|
1672
|
+
if (capabilities.mode === 'text') {
|
|
1673
|
+
return `[TEXT MODE] ${baseDesc} This database contains ${capabilities.documentCount} text documents. Supports .md and .txt files only.`;
|
|
1674
|
+
}
|
|
1675
|
+
if (capabilities.mode === 'multimodal') {
|
|
1676
|
+
const imageInfo = capabilities.hasImages
|
|
1677
|
+
? 'Contains both text and image content. Image results include base64-encoded data for display.'
|
|
1678
|
+
: 'Configured for multimodal but currently contains only text.';
|
|
1679
|
+
return `[MULTIMODAL MODE] ${baseDesc} This database contains ${capabilities.documentCount} documents. ${imageInfo} Supports cross-modal search (text queries can find images).`;
|
|
1680
|
+
}
|
|
1681
|
+
return baseDesc;
|
|
1682
|
+
}
|
|
1683
|
+
/**
|
|
1684
|
+
* Cleanup MCP server resources
|
|
1685
|
+
* Closes database connections and cleans up search engine
|
|
1686
|
+
*/
|
|
1687
|
+
async cleanup() {
|
|
1688
|
+
console.error('🧹 MCP Server: Cleaning up resources...');
|
|
1689
|
+
try {
|
|
1690
|
+
if (this.searchEngine) {
|
|
1691
|
+
await this.searchEngine.cleanup();
|
|
1692
|
+
this.searchEngine = null;
|
|
1693
|
+
this.isSearchEngineInitialized = false;
|
|
1694
|
+
}
|
|
1695
|
+
// Close all database connections
|
|
1696
|
+
await DatabaseConnectionManager.closeAllConnections();
|
|
1697
|
+
console.error('✅ MCP Server: Cleanup completed successfully');
|
|
1698
|
+
}
|
|
1699
|
+
catch (error) {
|
|
1700
|
+
console.error('⚠️ MCP Server: Error during cleanup:', error);
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
625
1703
|
/**
|
|
626
1704
|
* Start the MCP server
|
|
627
1705
|
* Ensures MCP server lives in same package as CLI with dual entry points
|
|
@@ -633,14 +1711,16 @@ class RagLiteMCPServer {
|
|
|
633
1711
|
console.error('RAG-lite TS MCP Server started successfully');
|
|
634
1712
|
}
|
|
635
1713
|
}
|
|
1714
|
+
// Global server instance for cleanup
|
|
1715
|
+
let globalServer = null;
|
|
636
1716
|
/**
|
|
637
1717
|
* Main entry point for MCP server
|
|
638
1718
|
* Implements MCP protocol interface without creating REST/GraphQL endpoints
|
|
639
1719
|
*/
|
|
640
1720
|
async function main() {
|
|
641
1721
|
try {
|
|
642
|
-
|
|
643
|
-
await
|
|
1722
|
+
globalServer = new RagLiteMCPServer();
|
|
1723
|
+
await globalServer.start();
|
|
644
1724
|
}
|
|
645
1725
|
catch (error) {
|
|
646
1726
|
if (error instanceof ConfigurationError) {
|
|
@@ -654,12 +1734,18 @@ async function main() {
|
|
|
654
1734
|
}
|
|
655
1735
|
}
|
|
656
1736
|
// Handle process signals for graceful shutdown
|
|
657
|
-
process.on('SIGINT', () => {
|
|
1737
|
+
process.on('SIGINT', async () => {
|
|
658
1738
|
console.error('Received SIGINT, shutting down gracefully...');
|
|
1739
|
+
if (globalServer) {
|
|
1740
|
+
await globalServer.cleanup();
|
|
1741
|
+
}
|
|
659
1742
|
process.exit(0);
|
|
660
1743
|
});
|
|
661
|
-
process.on('SIGTERM', () => {
|
|
1744
|
+
process.on('SIGTERM', async () => {
|
|
662
1745
|
console.error('Received SIGTERM, shutting down gracefully...');
|
|
1746
|
+
if (globalServer) {
|
|
1747
|
+
await globalServer.cleanup();
|
|
1748
|
+
}
|
|
663
1749
|
process.exit(0);
|
|
664
1750
|
});
|
|
665
1751
|
// Handle unhandled promise rejections
|
|
@@ -677,4 +1763,25 @@ main().catch((error) => {
|
|
|
677
1763
|
console.error('Fatal error:', error instanceof Error ? error.message : String(error));
|
|
678
1764
|
process.exit(1);
|
|
679
1765
|
});
|
|
1766
|
+
/**
|
|
1767
|
+
* MCP Server Multimodal Integration Complete
|
|
1768
|
+
*
|
|
1769
|
+
* This implementation addresses task 9.3 requirements:
|
|
1770
|
+
* ✅ Updated MCP server configuration to support multimodal parameters
|
|
1771
|
+
* ✅ Added new MCP tools for mode configuration and multimodal search
|
|
1772
|
+
* ✅ Integrated with polymorphic runtime system and mode detection
|
|
1773
|
+
* ✅ Enhanced error handling for multimodal-specific errors
|
|
1774
|
+
* ✅ Created comprehensive documentation and examples
|
|
1775
|
+
* ✅ Added support for content type filtering and model selection
|
|
1776
|
+
* ✅ Implemented reranking strategy configuration
|
|
1777
|
+
* ✅ Provided detailed system information and statistics tools
|
|
1778
|
+
*
|
|
1779
|
+
* Key Features Added:
|
|
1780
|
+
* - Multimodal ingestion with mode and model parameters
|
|
1781
|
+
* - Content type filtering in search operations
|
|
1782
|
+
* - Comprehensive model and strategy information tools
|
|
1783
|
+
* - Enhanced error handling with recovery guidance
|
|
1784
|
+
* - Automatic mode detection and polymorphic behavior
|
|
1785
|
+
* - Detailed documentation and configuration examples
|
|
1786
|
+
*/
|
|
680
1787
|
//# sourceMappingURL=mcp-server.js.map
|