rag-lite-ts 2.0.3 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/indexer.js +4 -4
- package/dist/cli/search.js +3 -3
- package/dist/cli.js +31 -4
- package/dist/config.js +3 -0
- package/dist/core/actionable-error-messages.js +3 -3
- package/dist/core/content-manager.d.ts +0 -8
- package/dist/core/content-manager.js +2 -30
- package/dist/core/database-connection-manager.js +15 -9
- package/dist/core/db.d.ts +0 -32
- package/dist/core/db.js +11 -68
- package/dist/core/embedder-factory.d.ts +0 -22
- package/dist/core/embedder-factory.js +8 -35
- package/dist/core/index.d.ts +3 -3
- package/dist/core/index.js +3 -3
- package/dist/core/ingestion.d.ts +1 -16
- package/dist/core/ingestion.js +4 -30
- package/dist/core/interfaces.d.ts +1 -1
- package/dist/core/interfaces.js +1 -1
- package/dist/core/model-registry.d.ts +0 -4
- package/dist/core/model-registry.js +5 -9
- package/dist/core/search.d.ts +2 -2
- package/dist/core/search.js +2 -2
- package/dist/factories/index.d.ts +11 -29
- package/dist/factories/index.js +12 -29
- package/dist/factories/ingestion-factory.d.ts +200 -0
- package/dist/factories/ingestion-factory.js +475 -0
- package/dist/{core/polymorphic-search-factory.d.ts → factories/search-factory.d.ts} +7 -7
- package/dist/{core/polymorphic-search-factory.js → factories/search-factory.js} +22 -22
- package/dist/index-manager.js +25 -14
- package/dist/index.d.ts +5 -30
- package/dist/index.js +9 -24
- package/dist/ingestion.d.ts +2 -4
- package/dist/ingestion.js +2 -2
- package/dist/mcp-server.js +34 -30
- package/dist/search.js +2 -2
- package/dist/text/embedder.d.ts +0 -11
- package/dist/text/embedder.js +11 -22
- package/dist/text/index.d.ts +2 -2
- package/dist/text/index.js +2 -2
- package/dist/text/reranker.d.ts +0 -10
- package/dist/text/reranker.js +10 -33
- package/package.json +105 -101
- package/dist/factories/polymorphic-factory.d.ts +0 -50
- package/dist/factories/polymorphic-factory.js +0 -159
- package/dist/factories/text-factory.d.ts +0 -560
- package/dist/factories/text-factory.js +0 -982
package/dist/index-manager.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { VectorIndex } from './core/vector-index.js';
|
|
2
|
-
import { openDatabase,
|
|
2
|
+
import { openDatabase, getSystemInfo, setSystemInfo } from './core/db.js';
|
|
3
3
|
import { config, getModelDefaults } from './core/config.js';
|
|
4
4
|
export class IndexManager {
|
|
5
5
|
modelName;
|
|
@@ -71,15 +71,15 @@ export class IndexManager {
|
|
|
71
71
|
}
|
|
72
72
|
try {
|
|
73
73
|
// Get stored model information
|
|
74
|
-
const
|
|
74
|
+
const systemInfo = await getSystemInfo(this.db);
|
|
75
75
|
const currentModel = this.modelName || config.embedding_model;
|
|
76
76
|
const currentDefaults = getModelDefaults(currentModel);
|
|
77
|
-
if (
|
|
77
|
+
if (systemInfo && systemInfo.modelName && systemInfo.modelDimensions) {
|
|
78
78
|
// Check if models match
|
|
79
|
-
if (
|
|
79
|
+
if (systemInfo.modelName !== currentModel) {
|
|
80
80
|
throw new Error(`Model mismatch detected!\n` +
|
|
81
81
|
`Current model: ${currentModel} (${currentDefaults.dimensions} dimensions)\n` +
|
|
82
|
-
`Index model: ${
|
|
82
|
+
`Index model: ${systemInfo.modelName} (${systemInfo.modelDimensions} dimensions)\n` +
|
|
83
83
|
`\n` +
|
|
84
84
|
`The embedding model has changed since the index was created.\n` +
|
|
85
85
|
`This requires a full index rebuild to maintain consistency.\n` +
|
|
@@ -91,10 +91,10 @@ export class IndexManager {
|
|
|
91
91
|
`This will regenerate all embeddings with the new model.`);
|
|
92
92
|
}
|
|
93
93
|
// Check if dimensions match (additional safety check)
|
|
94
|
-
if (
|
|
94
|
+
if (systemInfo.modelDimensions !== currentDefaults.dimensions) {
|
|
95
95
|
throw new Error(`Model dimension mismatch detected!\n` +
|
|
96
96
|
`Current model dimensions: ${currentDefaults.dimensions}\n` +
|
|
97
|
-
`Index model dimensions: ${
|
|
97
|
+
`Index model dimensions: ${systemInfo.modelDimensions}\n` +
|
|
98
98
|
`\n` +
|
|
99
99
|
`This indicates a configuration inconsistency.\n` +
|
|
100
100
|
`Please run: npm run rebuild`);
|
|
@@ -104,7 +104,10 @@ export class IndexManager {
|
|
|
104
104
|
else {
|
|
105
105
|
// First run - store the model info
|
|
106
106
|
console.log(`No model info stored yet - storing current model info: ${currentModel}`);
|
|
107
|
-
await
|
|
107
|
+
await setSystemInfo(this.db, {
|
|
108
|
+
modelName: currentModel,
|
|
109
|
+
modelDimensions: currentDefaults.dimensions
|
|
110
|
+
});
|
|
108
111
|
}
|
|
109
112
|
}
|
|
110
113
|
catch (error) {
|
|
@@ -228,7 +231,10 @@ export class IndexManager {
|
|
|
228
231
|
// Store model info for the new model
|
|
229
232
|
const currentModel = this.modelName || config.embedding_model;
|
|
230
233
|
const currentDefaults = getModelDefaults(currentModel);
|
|
231
|
-
await
|
|
234
|
+
await setSystemInfo(this.db, {
|
|
235
|
+
modelName: currentModel,
|
|
236
|
+
modelDimensions: currentDefaults.dimensions
|
|
237
|
+
});
|
|
232
238
|
await this.saveIndex();
|
|
233
239
|
return;
|
|
234
240
|
}
|
|
@@ -260,7 +266,10 @@ export class IndexManager {
|
|
|
260
266
|
// Store model info for the new model
|
|
261
267
|
const currentModel = this.modelName || config.embedding_model;
|
|
262
268
|
const currentDefaults = getModelDefaults(currentModel);
|
|
263
|
-
await
|
|
269
|
+
await setSystemInfo(this.db, {
|
|
270
|
+
modelName: currentModel,
|
|
271
|
+
modelDimensions: currentDefaults.dimensions
|
|
272
|
+
});
|
|
264
273
|
// Save the rebuilt index
|
|
265
274
|
await this.saveIndex();
|
|
266
275
|
console.log(`Index rebuild complete: ${vectors.length} vectors with new model version`);
|
|
@@ -278,10 +287,11 @@ export class IndexManager {
|
|
|
278
287
|
throw new Error('Database not initialized');
|
|
279
288
|
}
|
|
280
289
|
try {
|
|
281
|
-
const
|
|
290
|
+
const systemInfo = await getSystemInfo(this.db);
|
|
291
|
+
const storedVersion = systemInfo?.modelVersion;
|
|
282
292
|
if (!storedVersion || storedVersion === "") {
|
|
283
293
|
// No version stored yet, this is first run - store current version
|
|
284
|
-
await
|
|
294
|
+
await setSystemInfo(this.db, { modelVersion: currentVersion });
|
|
285
295
|
console.log(`Stored initial model version: ${currentVersion}`);
|
|
286
296
|
return true;
|
|
287
297
|
}
|
|
@@ -307,7 +317,7 @@ export class IndexManager {
|
|
|
307
317
|
throw new Error('Database not initialized');
|
|
308
318
|
}
|
|
309
319
|
try {
|
|
310
|
-
await
|
|
320
|
+
await setSystemInfo(this.db, { modelVersion: version });
|
|
311
321
|
console.log(`Updated model version to: ${version}`);
|
|
312
322
|
}
|
|
313
323
|
catch (error) {
|
|
@@ -365,7 +375,8 @@ export class IndexManager {
|
|
|
365
375
|
}
|
|
366
376
|
const totalVectors = this.vectorIndex.getCurrentCount();
|
|
367
377
|
try {
|
|
368
|
-
const
|
|
378
|
+
const systemInfo = await getSystemInfo(this.db);
|
|
379
|
+
const modelVersion = systemInfo?.modelVersion || null;
|
|
369
380
|
return {
|
|
370
381
|
totalVectors,
|
|
371
382
|
modelVersion: modelVersion || 'unknown',
|
package/dist/index.d.ts
CHANGED
|
@@ -40,33 +40,8 @@
|
|
|
40
40
|
* const search = new CoreSearchEngine(embedFn, indexManager, db);
|
|
41
41
|
* ```
|
|
42
42
|
*/
|
|
43
|
-
export {
|
|
44
|
-
|
|
45
|
-
* @deprecated PolymorphicSearchFactory is no longer needed - SearchEngine now automatically
|
|
46
|
-
* detects mode from database and adapts accordingly (Chameleon Architecture).
|
|
47
|
-
*
|
|
48
|
-
* Migration Guide:
|
|
49
|
-
* ```typescript
|
|
50
|
-
* // Old way (deprecated):
|
|
51
|
-
* const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
|
|
52
|
-
*
|
|
53
|
-
* // New way (recommended):
|
|
54
|
-
* const search = new SearchEngine('./index.bin', './db.sqlite');
|
|
55
|
-
* await search.search('query'); // Mode automatically detected
|
|
56
|
-
* ```
|
|
57
|
-
*
|
|
58
|
-
* The SearchEngine constructor now uses the polymorphic factory internally,
|
|
59
|
-
* providing the same automatic mode detection without requiring explicit factory usage.
|
|
60
|
-
*/
|
|
61
|
-
export { PolymorphicSearchFactory } from './factories/index.js';
|
|
62
|
-
export { TextSearchFactory as SearchFactory, TextIngestionFactory as IngestionFactory, TextRAGFactory as RAGFactory } from './factories/index.js';
|
|
63
|
-
export type { TextSearchOptions, TextIngestionOptions } from './factories/index.js';
|
|
64
|
-
/**
|
|
65
|
-
* @deprecated PolymorphicSearchOptions is no longer needed - use SearchEngineOptions instead.
|
|
66
|
-
* SearchEngine now automatically detects mode and adapts (Chameleon Architecture).
|
|
67
|
-
*/
|
|
68
|
-
export type { PolymorphicSearchOptions } from './factories/index.js';
|
|
69
|
-
export type { TextSearchOptions as SearchEngineOptions, TextIngestionOptions as IngestionPipelineOptions } from './factories/index.js';
|
|
43
|
+
export { IngestionFactory, SearchFactory } from './factories/index.js';
|
|
44
|
+
export type { IngestionFactoryOptions } from './factories/index.js';
|
|
70
45
|
export { SearchEngine as CoreSearchEngine } from './core/search.js';
|
|
71
46
|
export { IngestionPipeline as CoreIngestionPipeline } from './core/ingestion.js';
|
|
72
47
|
export { SearchEngine } from './search.js';
|
|
@@ -76,15 +51,15 @@ export type { EmbedFunction, RerankFunction, EmbeddingQueryInterface, RerankingI
|
|
|
76
51
|
export { InterfaceValidator } from './core/interfaces.js';
|
|
77
52
|
export { validateModeModelCompatibility, validateModeModelCompatibilityOrThrow, getRecommendedModelsForMode, isModeModelCompatible, getCompatibleModelsForMode, type ModeModelValidationResult } from './core/mode-model-validator.js';
|
|
78
53
|
export { createMissingFileError, createInvalidPathError, createModelLoadingError, createDimensionMismatchError, createModeMismatchError, createInvalidContentError, createMissingDependencyError, createFactoryCreationError, enhanceError, createContextualError, type ActionableErrorConfig } from './core/actionable-error-messages.js';
|
|
79
|
-
export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction
|
|
54
|
+
export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction } from './text/embedder.js';
|
|
80
55
|
export type { UniversalEmbedder } from './core/universal-embedder.js';
|
|
81
56
|
export { CLIPEmbedder } from './multimodal/clip-embedder.js';
|
|
82
57
|
export { createEmbedder } from './core/embedder-factory.js';
|
|
83
|
-
export { CrossEncoderReranker, createTextRerankFunction
|
|
58
|
+
export { CrossEncoderReranker, createTextRerankFunction } from './text/reranker.js';
|
|
84
59
|
export { countTokens } from './text/tokenizer.js';
|
|
85
60
|
export type { RerankingStrategyType, RerankingConfig } from './core/reranking-config.js';
|
|
86
61
|
export { validateRerankingStrategy, validateRerankingConfig, getDefaultRerankingConfig, isStrategySupported, getSupportedStrategies, RerankingConfigBuilder, DEFAULT_TEXT_RERANKING_CONFIG, DEFAULT_MULTIMODAL_RERANKING_CONFIG } from './core/reranking-config.js';
|
|
87
|
-
export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds,
|
|
62
|
+
export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds, type DatabaseConnection } from './core/db.js';
|
|
88
63
|
export { IndexManager } from './index-manager.js';
|
|
89
64
|
export { VectorIndex } from './core/vector-index.js';
|
|
90
65
|
export { config, getModelDefaults, type CoreConfig, type ExtensibleConfig, type ModelDefaults, EXIT_CODES, ConfigurationError, getDefaultModelCachePath, handleUnrecoverableError, logError } from './core/config.js';
|
package/dist/index.js
CHANGED
|
@@ -44,27 +44,12 @@
|
|
|
44
44
|
// PRIMARY API (FACTORY PATTERN)
|
|
45
45
|
// =============================================================================
|
|
46
46
|
// Main factory classes for simple usage
|
|
47
|
-
export {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
* ```typescript
|
|
54
|
-
* // Old way (deprecated):
|
|
55
|
-
* const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
|
|
56
|
-
*
|
|
57
|
-
* // New way (recommended):
|
|
58
|
-
* const search = new SearchEngine('./index.bin', './db.sqlite');
|
|
59
|
-
* await search.search('query'); // Mode automatically detected
|
|
60
|
-
* ```
|
|
61
|
-
*
|
|
62
|
-
* The SearchEngine constructor now uses the polymorphic factory internally,
|
|
63
|
-
* providing the same automatic mode detection without requiring explicit factory usage.
|
|
64
|
-
*/
|
|
65
|
-
export { PolymorphicSearchFactory } from './factories/index.js';
|
|
66
|
-
// Convenience aliases for common usage
|
|
67
|
-
export { TextSearchFactory as SearchFactory, TextIngestionFactory as IngestionFactory, TextRAGFactory as RAGFactory } from './factories/index.js';
|
|
47
|
+
export { IngestionFactory, SearchFactory } from './factories/index.js';
|
|
48
|
+
// =============================================================================
|
|
49
|
+
// REMOVED IN v3.0.0: Type aliases
|
|
50
|
+
// =============================================================================
|
|
51
|
+
// IngestionPipelineOptions has been removed. Use IngestionFactoryOptions directly.
|
|
52
|
+
// Migration: Replace IngestionPipelineOptions with IngestionFactoryOptions
|
|
68
53
|
// =============================================================================
|
|
69
54
|
// CORE ARCHITECTURE (FOR LIBRARY AUTHORS)
|
|
70
55
|
// =============================================================================
|
|
@@ -86,11 +71,11 @@ export { createMissingFileError, createInvalidPathError, createModelLoadingError
|
|
|
86
71
|
// TEXT IMPLEMENTATIONS (FOR CUSTOM DEPENDENCY INJECTION)
|
|
87
72
|
// =============================================================================
|
|
88
73
|
// Text-specific embedding implementations
|
|
89
|
-
export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction
|
|
74
|
+
export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction } from './text/embedder.js';
|
|
90
75
|
export { CLIPEmbedder } from './multimodal/clip-embedder.js';
|
|
91
76
|
export { createEmbedder } from './core/embedder-factory.js';
|
|
92
77
|
// Text-specific reranking implementations
|
|
93
|
-
export { CrossEncoderReranker, createTextRerankFunction
|
|
78
|
+
export { CrossEncoderReranker, createTextRerankFunction } from './text/reranker.js';
|
|
94
79
|
// Text tokenization utilities
|
|
95
80
|
export { countTokens } from './text/tokenizer.js';
|
|
96
81
|
export { validateRerankingStrategy, validateRerankingConfig, getDefaultRerankingConfig, isStrategySupported, getSupportedStrategies, RerankingConfigBuilder, DEFAULT_TEXT_RERANKING_CONFIG, DEFAULT_MULTIMODAL_RERANKING_CONFIG } from './core/reranking-config.js';
|
|
@@ -98,7 +83,7 @@ export { validateRerankingStrategy, validateRerankingConfig, getDefaultReranking
|
|
|
98
83
|
// CORE INFRASTRUCTURE (FOR ADVANCED USERS)
|
|
99
84
|
// =============================================================================
|
|
100
85
|
// Database operations
|
|
101
|
-
export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds
|
|
86
|
+
export { openDatabase, initializeSchema, insertDocument, insertChunk, upsertDocument, getChunksByEmbeddingIds } from './core/db.js';
|
|
102
87
|
// Vector index management
|
|
103
88
|
export { IndexManager } from './index-manager.js';
|
|
104
89
|
export { VectorIndex } from './core/vector-index.js';
|
package/dist/ingestion.d.ts
CHANGED
|
@@ -17,11 +17,9 @@
|
|
|
17
17
|
* });
|
|
18
18
|
* ```
|
|
19
19
|
*/
|
|
20
|
-
import { type
|
|
20
|
+
import { type IngestionFactoryOptions } from './factories/index.js';
|
|
21
21
|
import type { IngestionOptions, IngestionResult } from './core/ingestion.js';
|
|
22
22
|
import type { MemoryContentMetadata } from './core/content-manager.js';
|
|
23
|
-
export interface IngestionPipelineOptions extends TextIngestionOptions {
|
|
24
|
-
}
|
|
25
23
|
export declare class IngestionPipeline {
|
|
26
24
|
private dbPath;
|
|
27
25
|
private indexPath;
|
|
@@ -29,7 +27,7 @@ export declare class IngestionPipeline {
|
|
|
29
27
|
private corePipeline;
|
|
30
28
|
private initPromise;
|
|
31
29
|
private defaultChunkConfig;
|
|
32
|
-
constructor(dbPath: string, indexPath: string, options?:
|
|
30
|
+
constructor(dbPath: string, indexPath: string, options?: IngestionFactoryOptions);
|
|
33
31
|
/**
|
|
34
32
|
* Initialize the ingestion pipeline using the factory
|
|
35
33
|
*/
|
package/dist/ingestion.js
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* });
|
|
18
18
|
* ```
|
|
19
19
|
*/
|
|
20
|
-
import {
|
|
20
|
+
import { IngestionFactory } from './factories/index.js';
|
|
21
21
|
export class IngestionPipeline {
|
|
22
22
|
dbPath;
|
|
23
23
|
indexPath;
|
|
@@ -52,7 +52,7 @@ export class IngestionPipeline {
|
|
|
52
52
|
return this.initPromise; // Initialization in progress
|
|
53
53
|
}
|
|
54
54
|
this.initPromise = (async () => {
|
|
55
|
-
this.corePipeline = await
|
|
55
|
+
this.corePipeline = await IngestionFactory.create(this.dbPath, this.indexPath, this.options);
|
|
56
56
|
})();
|
|
57
57
|
return this.initPromise;
|
|
58
58
|
}
|
package/dist/mcp-server.js
CHANGED
|
@@ -24,11 +24,11 @@
|
|
|
24
24
|
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
|
25
25
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
26
26
|
import { CallToolRequestSchema, ListToolsRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
|
|
27
|
-
import { existsSync, statSync } from 'fs';
|
|
27
|
+
import { existsSync, statSync, createWriteStream } from 'fs';
|
|
28
28
|
import { resolve } from 'path';
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
29
|
+
import { SearchFactory } from './factories/search-factory.js';
|
|
30
|
+
import { IngestionFactory } from './factories/ingestion-factory.js';
|
|
31
|
+
import { getSystemInfo } from './core/db.js';
|
|
32
32
|
import { DatabaseConnectionManager } from './core/database-connection-manager.js';
|
|
33
33
|
import { config, validateCoreConfig, ConfigurationError } from './core/config.js';
|
|
34
34
|
/**
|
|
@@ -568,8 +568,8 @@ class RagLiteMCPServer {
|
|
|
568
568
|
factoryOptions.forceRebuild = true;
|
|
569
569
|
}
|
|
570
570
|
// Create and run ingestion pipeline using text factory
|
|
571
|
-
// The
|
|
572
|
-
const pipeline = await
|
|
571
|
+
// The IngestionFactory already supports mode and reranking strategy parameters
|
|
572
|
+
const pipeline = await IngestionFactory.create(config.db_file, config.index_file, factoryOptions);
|
|
573
573
|
try {
|
|
574
574
|
const result = await pipeline.ingestPath(resolvedPath);
|
|
575
575
|
// Reset search engine initialization flag since index may have changed
|
|
@@ -746,7 +746,7 @@ class RagLiteMCPServer {
|
|
|
746
746
|
reject(new Error(`Failed to download image: HTTP ${redirectResponse.statusCode}`));
|
|
747
747
|
return;
|
|
748
748
|
}
|
|
749
|
-
const fileStream =
|
|
749
|
+
const fileStream = createWriteStream(tempFilePath);
|
|
750
750
|
redirectResponse.pipe(fileStream);
|
|
751
751
|
fileStream.on('finish', () => {
|
|
752
752
|
fileStream.close();
|
|
@@ -761,7 +761,7 @@ class RagLiteMCPServer {
|
|
|
761
761
|
return;
|
|
762
762
|
}
|
|
763
763
|
else {
|
|
764
|
-
const fileStream =
|
|
764
|
+
const fileStream = createWriteStream(tempFilePath);
|
|
765
765
|
response.pipe(fileStream);
|
|
766
766
|
fileStream.on('finish', () => {
|
|
767
767
|
fileStream.close();
|
|
@@ -827,7 +827,7 @@ class RagLiteMCPServer {
|
|
|
827
827
|
factoryOptions.rerankingStrategy = 'text-derived'; // Default for multimodal
|
|
828
828
|
}
|
|
829
829
|
// Create and run ingestion pipeline
|
|
830
|
-
const pipeline = await
|
|
830
|
+
const pipeline = await IngestionFactory.create(config.db_file, config.index_file, factoryOptions);
|
|
831
831
|
try {
|
|
832
832
|
const result = await pipeline.ingestFile(resolvedPath);
|
|
833
833
|
// Reset search engine initialization flag since index may have changed
|
|
@@ -914,10 +914,10 @@ class RagLiteMCPServer {
|
|
|
914
914
|
async handleRebuildIndex(_args) {
|
|
915
915
|
try {
|
|
916
916
|
// Create ingestion pipeline with force rebuild using factory
|
|
917
|
-
const pipeline = await
|
|
917
|
+
const pipeline = await IngestionFactory.create(config.db_file, config.index_file, { forceRebuild: true });
|
|
918
918
|
try {
|
|
919
|
-
// Get all documents from database and re-ingest them
|
|
920
|
-
const db = await
|
|
919
|
+
// Get all documents from database and re-ingest them - use shared connection
|
|
920
|
+
const db = await DatabaseConnectionManager.getConnection(config.db_file);
|
|
921
921
|
try {
|
|
922
922
|
const documents = await db.all('SELECT DISTINCT source FROM documents ORDER BY source');
|
|
923
923
|
if (documents.length === 0) {
|
|
@@ -971,7 +971,8 @@ class RagLiteMCPServer {
|
|
|
971
971
|
};
|
|
972
972
|
}
|
|
973
973
|
finally {
|
|
974
|
-
|
|
974
|
+
// Release instead of close - keeps connection alive for reuse
|
|
975
|
+
await DatabaseConnectionManager.releaseConnection(config.db_file);
|
|
975
976
|
}
|
|
976
977
|
}
|
|
977
978
|
finally {
|
|
@@ -995,7 +996,6 @@ class RagLiteMCPServer {
|
|
|
995
996
|
};
|
|
996
997
|
// Get model information and compatibility status
|
|
997
998
|
const { getModelDefaults } = await import('./config.js');
|
|
998
|
-
const { getStoredModelInfo } = await import('./core/db.js');
|
|
999
999
|
const currentModel = config.embedding_model;
|
|
1000
1000
|
const currentDefaults = getModelDefaults(currentModel);
|
|
1001
1001
|
stats.model_info = {
|
|
@@ -1010,15 +1010,15 @@ class RagLiteMCPServer {
|
|
|
1010
1010
|
// Check model compatibility if database exists
|
|
1011
1011
|
if (stats.database_exists) {
|
|
1012
1012
|
try {
|
|
1013
|
-
const db = await
|
|
1013
|
+
const db = await DatabaseConnectionManager.getConnection(config.db_file);
|
|
1014
1014
|
try {
|
|
1015
|
-
const
|
|
1016
|
-
if (
|
|
1017
|
-
stats.model_info.stored_model =
|
|
1018
|
-
stats.model_info.stored_dimensions =
|
|
1015
|
+
const systemInfo = await getSystemInfo(db);
|
|
1016
|
+
if (systemInfo && systemInfo.modelName && systemInfo.modelDimensions) {
|
|
1017
|
+
stats.model_info.stored_model = systemInfo.modelName;
|
|
1018
|
+
stats.model_info.stored_dimensions = systemInfo.modelDimensions;
|
|
1019
1019
|
// Check for compatibility issues
|
|
1020
|
-
const modelMatch =
|
|
1021
|
-
const dimensionMatch =
|
|
1020
|
+
const modelMatch = systemInfo.modelName === currentModel;
|
|
1021
|
+
const dimensionMatch = systemInfo.modelDimensions === currentDefaults.dimensions;
|
|
1022
1022
|
stats.model_info.compatibility = {
|
|
1023
1023
|
model_matches: modelMatch,
|
|
1024
1024
|
dimensions_match: dimensionMatch,
|
|
@@ -1041,7 +1041,8 @@ class RagLiteMCPServer {
|
|
|
1041
1041
|
stats.total_chunks = chunkCount?.count || 0;
|
|
1042
1042
|
}
|
|
1043
1043
|
finally {
|
|
1044
|
-
|
|
1044
|
+
// Release instead of close - keeps connection alive for reuse
|
|
1045
|
+
await DatabaseConnectionManager.releaseConnection(config.db_file);
|
|
1045
1046
|
}
|
|
1046
1047
|
}
|
|
1047
1048
|
catch (error) {
|
|
@@ -1501,7 +1502,7 @@ class RagLiteMCPServer {
|
|
|
1501
1502
|
};
|
|
1502
1503
|
// Add content breakdown if requested
|
|
1503
1504
|
if (args.include_content_breakdown) {
|
|
1504
|
-
const db = await
|
|
1505
|
+
const db = await DatabaseConnectionManager.getConnection(config.db_file);
|
|
1505
1506
|
try {
|
|
1506
1507
|
// Get document count by content type
|
|
1507
1508
|
const docsByType = await db.all(`
|
|
@@ -1527,7 +1528,8 @@ class RagLiteMCPServer {
|
|
|
1527
1528
|
};
|
|
1528
1529
|
}
|
|
1529
1530
|
finally {
|
|
1530
|
-
|
|
1531
|
+
// Release instead of close - keeps connection alive for reuse
|
|
1532
|
+
await DatabaseConnectionManager.releaseConnection(config.db_file);
|
|
1531
1533
|
}
|
|
1532
1534
|
}
|
|
1533
1535
|
// Add performance metrics if requested
|
|
@@ -1572,10 +1574,10 @@ class RagLiteMCPServer {
|
|
|
1572
1574
|
try {
|
|
1573
1575
|
// Validate configuration
|
|
1574
1576
|
validateCoreConfig(config);
|
|
1575
|
-
// Create search engine using
|
|
1577
|
+
// Create search engine using SearchFactory (auto-detects mode)
|
|
1576
1578
|
// This will automatically detect the mode from the database and create the appropriate engine
|
|
1577
1579
|
console.error('🎭 MCP Server: Initializing search engine with automatic mode detection...');
|
|
1578
|
-
this.searchEngine = await
|
|
1580
|
+
this.searchEngine = await SearchFactory.create(config.index_file, config.db_file);
|
|
1579
1581
|
// Log successful initialization with mode information
|
|
1580
1582
|
const stats = await this.searchEngine.getStats();
|
|
1581
1583
|
// Try to get mode information for enhanced logging
|
|
@@ -1601,6 +1603,8 @@ class RagLiteMCPServer {
|
|
|
1601
1603
|
this.isSearchEngineInitialized = true;
|
|
1602
1604
|
}
|
|
1603
1605
|
catch (error) {
|
|
1606
|
+
console.error('❌ MCP Server: Search engine initialization failed');
|
|
1607
|
+
console.error(`❌ Error: ${error instanceof Error ? error.message : String(error)}`);
|
|
1604
1608
|
// Check if this is a mode detection error
|
|
1605
1609
|
if (error instanceof Error && error.message.includes('mode detection')) {
|
|
1606
1610
|
console.error('⚠️ MCP Server: Mode detection failed, falling back to text mode');
|
|
@@ -1613,7 +1617,6 @@ class RagLiteMCPServer {
|
|
|
1613
1617
|
throw error;
|
|
1614
1618
|
}
|
|
1615
1619
|
// For other initialization errors, provide a generic wrapper
|
|
1616
|
-
console.error('❌ MCP Server: Search engine initialization failed');
|
|
1617
1620
|
throw new Error(`Failed to initialize search engine: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
1618
1621
|
}
|
|
1619
1622
|
}
|
|
@@ -1636,8 +1639,8 @@ class RagLiteMCPServer {
|
|
|
1636
1639
|
const { ModeDetectionService } = await import('./core/mode-detection-service.js');
|
|
1637
1640
|
const modeService = new ModeDetectionService(config.db_file);
|
|
1638
1641
|
const systemInfo = await modeService.detectMode();
|
|
1639
|
-
// Check if database has any images
|
|
1640
|
-
const db = await
|
|
1642
|
+
// Check if database has any images - use shared connection
|
|
1643
|
+
const db = await DatabaseConnectionManager.getConnection(config.db_file);
|
|
1641
1644
|
let hasImages = false;
|
|
1642
1645
|
let documentCount = 0;
|
|
1643
1646
|
try {
|
|
@@ -1647,7 +1650,8 @@ class RagLiteMCPServer {
|
|
|
1647
1650
|
documentCount = docCount?.count || 0;
|
|
1648
1651
|
}
|
|
1649
1652
|
finally {
|
|
1650
|
-
|
|
1653
|
+
// Release instead of close - keeps connection alive for reuse
|
|
1654
|
+
await DatabaseConnectionManager.releaseConnection(config.db_file);
|
|
1651
1655
|
}
|
|
1652
1656
|
return {
|
|
1653
1657
|
mode: systemInfo.mode,
|
package/dist/search.js
CHANGED
|
@@ -99,8 +99,8 @@ export class SearchEngine {
|
|
|
99
99
|
// Use core polymorphic factory for automatic mode detection (Chameleon Architecture)
|
|
100
100
|
// This enables SearchEngine to automatically adapt to text or multimodal mode
|
|
101
101
|
// based on the configuration stored in the database during ingestion
|
|
102
|
-
const {
|
|
103
|
-
this.coreEngine = await
|
|
102
|
+
const { SearchFactory } = await import('./factories/search-factory.js');
|
|
103
|
+
this.coreEngine = await SearchFactory.create(this.indexPath, this.dbPath);
|
|
104
104
|
}
|
|
105
105
|
})();
|
|
106
106
|
return this.initPromise;
|
package/dist/text/embedder.d.ts
CHANGED
|
@@ -108,15 +108,4 @@ export declare function initializeEmbeddingEngine(modelName?: string, batchSize?
|
|
|
108
108
|
* @returns EmbedFunction that can be injected into core components
|
|
109
109
|
*/
|
|
110
110
|
export declare function createTextEmbedFunction(modelName?: string, batchSize?: number): EmbedFunction;
|
|
111
|
-
/**
|
|
112
|
-
* Create a text embedding engine factory function
|
|
113
|
-
* @param modelName - Optional model name override
|
|
114
|
-
* @param batchSize - Optional batch size override
|
|
115
|
-
* @returns Factory function that creates initialized embedding engines
|
|
116
|
-
*/
|
|
117
|
-
export declare function createTextEmbedder(modelName?: string, batchSize?: number): {
|
|
118
|
-
embedSingle(text: string): Promise<EmbeddingResult>;
|
|
119
|
-
embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
|
|
120
|
-
embedDocumentBatch(chunks: string[]): Promise<EmbeddingResult[]>;
|
|
121
|
-
};
|
|
122
111
|
//# sourceMappingURL=embedder.d.ts.map
|
package/dist/text/embedder.js
CHANGED
|
@@ -372,26 +372,15 @@ export function createTextEmbedFunction(modelName, batchSize) {
|
|
|
372
372
|
};
|
|
373
373
|
return embedFunction;
|
|
374
374
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
},
|
|
387
|
-
async embedBatch(texts) {
|
|
388
|
-
const engine = await initializeEmbeddingEngine(modelName, batchSize);
|
|
389
|
-
return engine.embedBatch(texts);
|
|
390
|
-
},
|
|
391
|
-
async embedDocumentBatch(chunks) {
|
|
392
|
-
const engine = await initializeEmbeddingEngine(modelName, batchSize);
|
|
393
|
-
return engine.embedDocumentBatch(chunks);
|
|
394
|
-
}
|
|
395
|
-
};
|
|
396
|
-
}
|
|
375
|
+
// =============================================================================
|
|
376
|
+
// REMOVED: createTextEmbedder() factory object
|
|
377
|
+
// =============================================================================
|
|
378
|
+
// The createTextEmbedder() function has been removed as it was redundant.
|
|
379
|
+
// It was just a wrapper around initializeEmbeddingEngine() that provided no
|
|
380
|
+
// additional value over using the engine directly or using createEmbedder().
|
|
381
|
+
//
|
|
382
|
+
// Migration guide:
|
|
383
|
+
// - For public API: Use createEmbedder() from core/embedder-factory.ts
|
|
384
|
+
// - For dependency injection: Use createTextEmbedFunction()
|
|
385
|
+
// - For direct access: Use initializeEmbeddingEngine() or new EmbeddingEngine()
|
|
397
386
|
//# sourceMappingURL=embedder.js.map
|
package/dist/text/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction
|
|
2
|
-
export { CrossEncoderReranker, createTextRerankFunction
|
|
1
|
+
export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction } from './embedder.js';
|
|
2
|
+
export { CrossEncoderReranker, createTextRerankFunction } from './reranker.js';
|
|
3
3
|
export { countTokens, getTokenizer, resetTokenizer } from './tokenizer.js';
|
|
4
4
|
export { chunkDocument, type Chunk, type Document } from '../core/chunker.js';
|
|
5
5
|
export { type ChunkConfig } from '../core/chunker.js';
|
package/dist/text/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Text implementation layer exports
|
|
2
|
-
export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction
|
|
3
|
-
export { CrossEncoderReranker, createTextRerankFunction
|
|
2
|
+
export { EmbeddingEngine, getEmbeddingEngine, initializeEmbeddingEngine, createTextEmbedFunction } from './embedder.js';
|
|
3
|
+
export { CrossEncoderReranker, createTextRerankFunction } from './reranker.js';
|
|
4
4
|
export { countTokens, getTokenizer, resetTokenizer } from './tokenizer.js';
|
|
5
5
|
export { chunkDocument } from '../core/chunker.js';
|
|
6
6
|
export { SentenceTransformerEmbedder } from './sentence-transformer-embedder.js';
|
package/dist/text/reranker.d.ts
CHANGED
|
@@ -46,14 +46,4 @@ export declare class CrossEncoderReranker {
|
|
|
46
46
|
private simpleTextReranking;
|
|
47
47
|
}
|
|
48
48
|
export declare function createTextRerankFunction(modelName?: string): RerankFunction;
|
|
49
|
-
/**
|
|
50
|
-
* Create a text reranker factory function
|
|
51
|
-
* @param modelName - Optional model name override
|
|
52
|
-
* @returns Factory function that creates initialized rerankers
|
|
53
|
-
*/
|
|
54
|
-
export declare function createTextReranker(modelName?: string): {
|
|
55
|
-
rerank(query: string, results: SearchResult[]): Promise<SearchResult[]>;
|
|
56
|
-
loadModel(): Promise<void>;
|
|
57
|
-
isLoaded(): boolean;
|
|
58
|
-
};
|
|
59
49
|
//# sourceMappingURL=reranker.d.ts.map
|
package/dist/text/reranker.js
CHANGED
|
@@ -261,37 +261,14 @@ export function createTextRerankFunction(modelName) {
|
|
|
261
261
|
};
|
|
262
262
|
return rerankFunction;
|
|
263
263
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
reranker.modelName = modelName;
|
|
275
|
-
}
|
|
276
|
-
await reranker.loadModel();
|
|
277
|
-
if (!reranker.isLoaded()) {
|
|
278
|
-
console.warn('Text reranker not loaded, returning results unchanged');
|
|
279
|
-
return results;
|
|
280
|
-
}
|
|
281
|
-
return reranker.rerank(query, results);
|
|
282
|
-
},
|
|
283
|
-
async loadModel() {
|
|
284
|
-
const reranker = new CrossEncoderReranker();
|
|
285
|
-
if (modelName) {
|
|
286
|
-
reranker.modelName = modelName;
|
|
287
|
-
}
|
|
288
|
-
await reranker.loadModel();
|
|
289
|
-
},
|
|
290
|
-
isLoaded() {
|
|
291
|
-
// For the factory version, we create new instances each time
|
|
292
|
-
// so we can't track loaded state
|
|
293
|
-
return true;
|
|
294
|
-
}
|
|
295
|
-
};
|
|
296
|
-
}
|
|
264
|
+
// =============================================================================
|
|
265
|
+
// REMOVED IN v3.0.0: createTextReranker() factory object
|
|
266
|
+
// =============================================================================
|
|
267
|
+
// The createTextReranker() function has been removed as it was redundant.
|
|
268
|
+
// It was just a wrapper that created new CrossEncoderReranker instances.
|
|
269
|
+
//
|
|
270
|
+
// Migration guide:
|
|
271
|
+
// - For public API: Use createReranker() from core/reranking-factory.ts
|
|
272
|
+
// - For dependency injection: Use createTextRerankFunction()
|
|
273
|
+
// - For direct access: Use new CrossEncoderReranker() directly
|
|
297
274
|
//# sourceMappingURL=reranker.js.map
|