rag-lite-ts 2.0.2 → 2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/cli/indexer.js +25 -6
- package/dist/cli/search.js +3 -3
- package/dist/cli.js +33 -6
- package/dist/core/actionable-error-messages.js +3 -3
- package/dist/core/content-manager.d.ts +0 -8
- package/dist/core/content-manager.js +2 -30
- package/dist/core/database-connection-manager.js +10 -0
- package/dist/core/db.d.ts +0 -32
- package/dist/core/db.js +11 -68
- package/dist/core/embedder-factory.d.ts +0 -22
- package/dist/core/embedder-factory.js +8 -35
- package/dist/core/index.d.ts +3 -3
- package/dist/core/index.js +3 -3
- package/dist/core/ingestion.d.ts +1 -16
- package/dist/core/ingestion.js +1 -30
- package/dist/core/interfaces.d.ts +1 -1
- package/dist/core/interfaces.js +1 -1
- package/dist/core/model-registry.d.ts +0 -4
- package/dist/core/model-registry.js +5 -9
- package/dist/core/search.d.ts +2 -2
- package/dist/core/search.js +2 -2
- package/dist/factories/index.d.ts +11 -29
- package/dist/factories/index.js +12 -29
- package/dist/factories/ingestion-factory.d.ts +200 -0
- package/dist/factories/ingestion-factory.js +475 -0
- package/dist/{core/polymorphic-search-factory.d.ts → factories/search-factory.d.ts} +7 -7
- package/dist/{core/polymorphic-search-factory.js → factories/search-factory.js} +22 -22
- package/dist/index-manager.js +25 -14
- package/dist/index.d.ts +5 -30
- package/dist/index.js +9 -24
- package/dist/indexer.js +5 -2
- package/dist/ingestion.d.ts +2 -4
- package/dist/ingestion.js +2 -2
- package/dist/mcp-server.js +31 -25
- package/dist/search.js +2 -2
- package/dist/text/embedder.d.ts +0 -11
- package/dist/text/embedder.js +11 -22
- package/dist/text/index.d.ts +2 -2
- package/dist/text/index.js +2 -2
- package/dist/text/reranker.d.ts +0 -10
- package/dist/text/reranker.js +10 -33
- package/package.json +7 -3
- package/dist/factories/polymorphic-factory.d.ts +0 -50
- package/dist/factories/polymorphic-factory.js +0 -159
- package/dist/factories/text-factory.d.ts +0 -560
- package/dist/factories/text-factory.js +0 -968
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Polymorphic factory for creating mode-aware search engines
|
|
3
|
-
* Automatically detects mode from database and uses appropriate embedder
|
|
4
|
-
*
|
|
5
|
-
* This factory implements the Chameleon Architecture principle:
|
|
6
|
-
* - Detects mode (text/multimodal) from database configuration
|
|
7
|
-
* - Uses appropriate embedder based on detected mode
|
|
8
|
-
* - Provides seamless polymorphic behavior without user intervention
|
|
9
|
-
*
|
|
10
|
-
* @example
|
|
11
|
-
* ```typescript
|
|
12
|
-
* // Automatically detects mode and creates appropriate search engine
|
|
13
|
-
* const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
|
|
14
|
-
*
|
|
15
|
-
* // Works for both text and multimodal modes
|
|
16
|
-
* const results = await search.search('query');
|
|
17
|
-
* ```
|
|
18
|
-
*/
|
|
19
|
-
import { SearchEngine } from '../core/search.js';
|
|
20
|
-
export interface PolymorphicSearchOptions {
|
|
21
|
-
/** Whether to enable reranking (default: true) */
|
|
22
|
-
enableReranking?: boolean;
|
|
23
|
-
/** Top-k results to return (default: from config) */
|
|
24
|
-
topK?: number;
|
|
25
|
-
}
|
|
26
|
-
/**
|
|
27
|
-
* Factory for creating mode-aware search engines
|
|
28
|
-
* Automatically detects mode from database and uses appropriate embedder
|
|
29
|
-
*/
|
|
30
|
-
export declare class PolymorphicSearchFactory {
|
|
31
|
-
/**
|
|
32
|
-
* Create a SearchEngine that automatically adapts to the mode stored in the database
|
|
33
|
-
*
|
|
34
|
-
* This method:
|
|
35
|
-
* 1. Validates that required files exist
|
|
36
|
-
* 2. Opens database and reads system configuration
|
|
37
|
-
* 3. Detects mode (text/multimodal) from database
|
|
38
|
-
* 4. Creates appropriate embedder based on mode
|
|
39
|
-
* 5. Optionally creates reranker based on configuration
|
|
40
|
-
* 6. Returns fully configured SearchEngine
|
|
41
|
-
*
|
|
42
|
-
* @param indexPath - Path to the vector index file (must exist)
|
|
43
|
-
* @param dbPath - Path to the SQLite database file (must exist)
|
|
44
|
-
* @param options - Optional configuration overrides
|
|
45
|
-
* @returns Promise resolving to configured SearchEngine
|
|
46
|
-
* @throws {Error} If required files don't exist or initialization fails
|
|
47
|
-
*/
|
|
48
|
-
static create(indexPath: string, dbPath: string, options?: PolymorphicSearchOptions): Promise<SearchEngine>;
|
|
49
|
-
}
|
|
50
|
-
//# sourceMappingURL=polymorphic-factory.d.ts.map
|
|
@@ -1,159 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Polymorphic factory for creating mode-aware search engines
|
|
3
|
-
* Automatically detects mode from database and uses appropriate embedder
|
|
4
|
-
*
|
|
5
|
-
* This factory implements the Chameleon Architecture principle:
|
|
6
|
-
* - Detects mode (text/multimodal) from database configuration
|
|
7
|
-
* - Uses appropriate embedder based on detected mode
|
|
8
|
-
* - Provides seamless polymorphic behavior without user intervention
|
|
9
|
-
*
|
|
10
|
-
* @example
|
|
11
|
-
* ```typescript
|
|
12
|
-
* // Automatically detects mode and creates appropriate search engine
|
|
13
|
-
* const search = await PolymorphicSearchFactory.create('./index.bin', './db.sqlite');
|
|
14
|
-
*
|
|
15
|
-
* // Works for both text and multimodal modes
|
|
16
|
-
* const results = await search.search('query');
|
|
17
|
-
* ```
|
|
18
|
-
*/
|
|
19
|
-
import { SearchEngine } from '../core/search.js';
|
|
20
|
-
import { IndexManager } from '../index-manager.js';
|
|
21
|
-
import { openDatabase, getSystemInfo } from '../core/db.js';
|
|
22
|
-
import { createTextEmbedFunction } from '../text/embedder.js';
|
|
23
|
-
import { createTextRerankFunction } from '../text/reranker.js';
|
|
24
|
-
import { config, getModelDefaults } from '../core/config.js';
|
|
25
|
-
import { existsSync } from 'fs';
|
|
26
|
-
import { createMissingFileError, createInvalidPathError, createFactoryCreationError } from '../core/actionable-error-messages.js';
|
|
27
|
-
/**
|
|
28
|
-
* Factory for creating mode-aware search engines
|
|
29
|
-
* Automatically detects mode from database and uses appropriate embedder
|
|
30
|
-
*/
|
|
31
|
-
export class PolymorphicSearchFactory {
|
|
32
|
-
/**
|
|
33
|
-
* Create a SearchEngine that automatically adapts to the mode stored in the database
|
|
34
|
-
*
|
|
35
|
-
* This method:
|
|
36
|
-
* 1. Validates that required files exist
|
|
37
|
-
* 2. Opens database and reads system configuration
|
|
38
|
-
* 3. Detects mode (text/multimodal) from database
|
|
39
|
-
* 4. Creates appropriate embedder based on mode
|
|
40
|
-
* 5. Optionally creates reranker based on configuration
|
|
41
|
-
* 6. Returns fully configured SearchEngine
|
|
42
|
-
*
|
|
43
|
-
* @param indexPath - Path to the vector index file (must exist)
|
|
44
|
-
* @param dbPath - Path to the SQLite database file (must exist)
|
|
45
|
-
* @param options - Optional configuration overrides
|
|
46
|
-
* @returns Promise resolving to configured SearchEngine
|
|
47
|
-
* @throws {Error} If required files don't exist or initialization fails
|
|
48
|
-
*/
|
|
49
|
-
static async create(indexPath, dbPath, options = {}) {
|
|
50
|
-
try {
|
|
51
|
-
console.log('🏭 PolymorphicSearchFactory: Initializing mode-aware search engine...');
|
|
52
|
-
// Validate input paths
|
|
53
|
-
if (!indexPath || !dbPath) {
|
|
54
|
-
throw createInvalidPathError([
|
|
55
|
-
{ name: 'indexPath', value: indexPath },
|
|
56
|
-
{ name: 'dbPath', value: dbPath }
|
|
57
|
-
], { operationContext: 'PolymorphicSearchFactory.create' });
|
|
58
|
-
}
|
|
59
|
-
// Check if required files exist
|
|
60
|
-
if (!existsSync(indexPath)) {
|
|
61
|
-
throw createMissingFileError(indexPath, 'index', {
|
|
62
|
-
operationContext: 'PolymorphicSearchFactory.create'
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
if (!existsSync(dbPath)) {
|
|
66
|
-
throw createMissingFileError(dbPath, 'database', {
|
|
67
|
-
operationContext: 'PolymorphicSearchFactory.create'
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
|
-
// Step 1: Open database and detect mode
|
|
71
|
-
console.log('💾 Opening database and detecting mode...');
|
|
72
|
-
const db = await openDatabase(dbPath);
|
|
73
|
-
let mode = 'text';
|
|
74
|
-
let embeddingModel;
|
|
75
|
-
let modelDimensions;
|
|
76
|
-
try {
|
|
77
|
-
const systemInfo = await getSystemInfo(db);
|
|
78
|
-
if (systemInfo) {
|
|
79
|
-
mode = systemInfo.mode;
|
|
80
|
-
embeddingModel = systemInfo.modelName;
|
|
81
|
-
modelDimensions = systemInfo.modelDimensions;
|
|
82
|
-
console.log(`📊 Detected mode: ${mode}`);
|
|
83
|
-
console.log(`📊 Detected model: ${embeddingModel} (${modelDimensions} dimensions)`);
|
|
84
|
-
}
|
|
85
|
-
else {
|
|
86
|
-
// Fallback to default if no system info
|
|
87
|
-
embeddingModel = config.embedding_model;
|
|
88
|
-
const modelDefaults = getModelDefaults(embeddingModel);
|
|
89
|
-
modelDimensions = modelDefaults.dimensions;
|
|
90
|
-
console.log(`📊 No system info found, using default: ${embeddingModel} (${modelDimensions} dimensions)`);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
catch (error) {
|
|
94
|
-
// If getSystemInfo fails, use defaults
|
|
95
|
-
embeddingModel = config.embedding_model;
|
|
96
|
-
const modelDefaults = getModelDefaults(embeddingModel);
|
|
97
|
-
modelDimensions = modelDefaults.dimensions;
|
|
98
|
-
console.log(`📊 Using default configuration: ${embeddingModel} (${modelDimensions} dimensions)`);
|
|
99
|
-
}
|
|
100
|
-
// Step 2: Create appropriate embedder based on mode
|
|
101
|
-
let embedFn;
|
|
102
|
-
if (mode === 'multimodal') {
|
|
103
|
-
console.log('📊 Loading CLIP embedder for multimodal mode...');
|
|
104
|
-
const { createEmbedder } = await import('../core/embedder-factory.js');
|
|
105
|
-
const clipEmbedder = await createEmbedder(embeddingModel);
|
|
106
|
-
// Wrap CLIP embedder to match EmbedFunction signature
|
|
107
|
-
embedFn = async (content, contentType) => {
|
|
108
|
-
if (contentType === 'image') {
|
|
109
|
-
return await clipEmbedder.embedImage(content);
|
|
110
|
-
}
|
|
111
|
-
return await clipEmbedder.embedText(content);
|
|
112
|
-
};
|
|
113
|
-
console.log('✓ CLIP embedder loaded for multimodal mode');
|
|
114
|
-
}
|
|
115
|
-
else {
|
|
116
|
-
console.log('📊 Loading text embedder for text mode...');
|
|
117
|
-
embedFn = createTextEmbedFunction(embeddingModel);
|
|
118
|
-
console.log('✓ Text embedder loaded');
|
|
119
|
-
}
|
|
120
|
-
// Step 3: Initialize reranking function (optional)
|
|
121
|
-
let rerankFn;
|
|
122
|
-
if (options.enableReranking === true) {
|
|
123
|
-
console.log('🔄 Loading reranking model...');
|
|
124
|
-
rerankFn = createTextRerankFunction();
|
|
125
|
-
await rerankFn('test query', []);
|
|
126
|
-
console.log('✓ Reranking model loaded successfully');
|
|
127
|
-
}
|
|
128
|
-
else {
|
|
129
|
-
console.log('🔄 Reranking disabled (local-first, fast mode)');
|
|
130
|
-
}
|
|
131
|
-
// Step 4: Initialize database schema
|
|
132
|
-
const { initializeSchema } = await import('../core/db.js');
|
|
133
|
-
await initializeSchema(db);
|
|
134
|
-
console.log('✓ Database connection established');
|
|
135
|
-
// Step 5: Initialize index manager
|
|
136
|
-
console.log('📇 Loading vector index...');
|
|
137
|
-
const indexManager = new IndexManager(indexPath, dbPath, modelDimensions, embeddingModel);
|
|
138
|
-
await indexManager.initialize();
|
|
139
|
-
console.log('✓ Vector index loaded successfully');
|
|
140
|
-
// Step 6: Create ContentResolver
|
|
141
|
-
console.log('📁 Initializing content resolver...');
|
|
142
|
-
const { ContentResolver } = await import('../core/content-resolver.js');
|
|
143
|
-
const contentResolver = new ContentResolver(db);
|
|
144
|
-
console.log('✓ Content resolver ready');
|
|
145
|
-
// Step 7: Create SearchEngine with dependency injection
|
|
146
|
-
const searchEngine = new SearchEngine(embedFn, indexManager, db, rerankFn, contentResolver);
|
|
147
|
-
// Step 8: Validate the setup
|
|
148
|
-
const stats = await searchEngine.getStats();
|
|
149
|
-
console.log(`✓ Search engine ready: ${stats.totalChunks} chunks indexed, mode: ${mode}, reranking ${stats.rerankingEnabled ? 'enabled' : 'disabled'}`);
|
|
150
|
-
console.log('🎉 PolymorphicSearchFactory: Mode-aware search engine initialized successfully');
|
|
151
|
-
return searchEngine;
|
|
152
|
-
}
|
|
153
|
-
catch (error) {
|
|
154
|
-
console.error('❌ PolymorphicSearchFactory: Failed to create search engine');
|
|
155
|
-
throw createFactoryCreationError('PolymorphicSearchFactory', error instanceof Error ? error.message : 'Unknown error', { operationContext: 'polymorphic search engine creation' });
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
//# sourceMappingURL=polymorphic-factory.js.map
|