rag-lite-ts 2.2.0 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +88 -5
  2. package/dist/cjs/cli/indexer.js +73 -15
  3. package/dist/cjs/cli/search.js +77 -2
  4. package/dist/cjs/cli/ui-server.d.ts +5 -0
  5. package/dist/cjs/cli/ui-server.js +152 -0
  6. package/dist/cjs/cli.js +53 -7
  7. package/dist/cjs/core/abstract-generator.d.ts +97 -0
  8. package/dist/cjs/core/abstract-generator.js +222 -0
  9. package/dist/cjs/core/binary-index-format.js +53 -10
  10. package/dist/cjs/core/db.d.ts +56 -0
  11. package/dist/cjs/core/db.js +105 -0
  12. package/dist/cjs/core/generator-registry.d.ts +114 -0
  13. package/dist/cjs/core/generator-registry.js +280 -0
  14. package/dist/cjs/core/index.d.ts +4 -0
  15. package/dist/cjs/core/index.js +11 -0
  16. package/dist/cjs/core/ingestion.js +3 -0
  17. package/dist/cjs/core/knowledge-base-manager.d.ts +109 -0
  18. package/dist/cjs/core/knowledge-base-manager.js +256 -0
  19. package/dist/cjs/core/lazy-dependency-loader.d.ts +43 -0
  20. package/dist/cjs/core/lazy-dependency-loader.js +111 -2
  21. package/dist/cjs/core/prompt-templates.d.ts +138 -0
  22. package/dist/cjs/core/prompt-templates.js +225 -0
  23. package/dist/cjs/core/response-generator.d.ts +132 -0
  24. package/dist/cjs/core/response-generator.js +69 -0
  25. package/dist/cjs/core/search-pipeline.js +1 -1
  26. package/dist/cjs/core/search.d.ts +72 -1
  27. package/dist/cjs/core/search.js +80 -7
  28. package/dist/cjs/core/types.d.ts +1 -0
  29. package/dist/cjs/core/vector-index-messages.d.ts +52 -0
  30. package/dist/cjs/core/vector-index-messages.js +5 -0
  31. package/dist/cjs/core/vector-index-worker.d.ts +6 -0
  32. package/dist/cjs/core/vector-index-worker.js +314 -0
  33. package/dist/cjs/core/vector-index.d.ts +45 -10
  34. package/dist/cjs/core/vector-index.js +279 -218
  35. package/dist/cjs/factories/generator-factory.d.ts +88 -0
  36. package/dist/cjs/factories/generator-factory.js +151 -0
  37. package/dist/cjs/factories/index.d.ts +1 -0
  38. package/dist/cjs/factories/index.js +5 -0
  39. package/dist/cjs/factories/ingestion-factory.js +3 -7
  40. package/dist/cjs/factories/search-factory.js +11 -0
  41. package/dist/cjs/index-manager.d.ts +23 -3
  42. package/dist/cjs/index-manager.js +84 -15
  43. package/dist/cjs/index.d.ts +11 -1
  44. package/dist/cjs/index.js +19 -1
  45. package/dist/cjs/text/generators/causal-lm-generator.d.ts +65 -0
  46. package/dist/cjs/text/generators/causal-lm-generator.js +197 -0
  47. package/dist/cjs/text/generators/index.d.ts +10 -0
  48. package/dist/cjs/text/generators/index.js +10 -0
  49. package/dist/cjs/text/generators/instruct-generator.d.ts +62 -0
  50. package/dist/cjs/text/generators/instruct-generator.js +192 -0
  51. package/dist/esm/cli/indexer.js +73 -15
  52. package/dist/esm/cli/search.js +77 -2
  53. package/dist/esm/cli/ui-server.d.ts +5 -0
  54. package/dist/esm/cli/ui-server.js +152 -0
  55. package/dist/esm/cli.js +53 -7
  56. package/dist/esm/core/abstract-generator.d.ts +97 -0
  57. package/dist/esm/core/abstract-generator.js +222 -0
  58. package/dist/esm/core/binary-index-format.js +53 -10
  59. package/dist/esm/core/db.d.ts +56 -0
  60. package/dist/esm/core/db.js +105 -0
  61. package/dist/esm/core/generator-registry.d.ts +114 -0
  62. package/dist/esm/core/generator-registry.js +280 -0
  63. package/dist/esm/core/index.d.ts +4 -0
  64. package/dist/esm/core/index.js +11 -0
  65. package/dist/esm/core/ingestion.js +3 -0
  66. package/dist/esm/core/knowledge-base-manager.d.ts +109 -0
  67. package/dist/esm/core/knowledge-base-manager.js +256 -0
  68. package/dist/esm/core/lazy-dependency-loader.d.ts +43 -0
  69. package/dist/esm/core/lazy-dependency-loader.js +111 -2
  70. package/dist/esm/core/prompt-templates.d.ts +138 -0
  71. package/dist/esm/core/prompt-templates.js +225 -0
  72. package/dist/esm/core/response-generator.d.ts +132 -0
  73. package/dist/esm/core/response-generator.js +69 -0
  74. package/dist/esm/core/search-pipeline.js +1 -1
  75. package/dist/esm/core/search.d.ts +72 -1
  76. package/dist/esm/core/search.js +80 -7
  77. package/dist/esm/core/types.d.ts +1 -0
  78. package/dist/esm/core/vector-index-messages.d.ts +52 -0
  79. package/dist/esm/core/vector-index-messages.js +5 -0
  80. package/dist/esm/core/vector-index-worker.d.ts +6 -0
  81. package/dist/esm/core/vector-index-worker.js +314 -0
  82. package/dist/esm/core/vector-index.d.ts +45 -10
  83. package/dist/esm/core/vector-index.js +279 -218
  84. package/dist/esm/factories/generator-factory.d.ts +88 -0
  85. package/dist/esm/factories/generator-factory.js +151 -0
  86. package/dist/esm/factories/index.d.ts +1 -0
  87. package/dist/esm/factories/index.js +5 -0
  88. package/dist/esm/factories/ingestion-factory.js +3 -7
  89. package/dist/esm/factories/search-factory.js +11 -0
  90. package/dist/esm/index-manager.d.ts +23 -3
  91. package/dist/esm/index-manager.js +84 -15
  92. package/dist/esm/index.d.ts +11 -1
  93. package/dist/esm/index.js +19 -1
  94. package/dist/esm/text/generators/causal-lm-generator.d.ts +65 -0
  95. package/dist/esm/text/generators/causal-lm-generator.js +197 -0
  96. package/dist/esm/text/generators/index.d.ts +10 -0
  97. package/dist/esm/text/generators/index.js +10 -0
  98. package/dist/esm/text/generators/instruct-generator.d.ts +62 -0
  99. package/dist/esm/text/generators/instruct-generator.js +192 -0
  100. package/package.json +14 -7
@@ -0,0 +1,222 @@
1
+ /**
2
+ * CORE MODULE — Abstract Base Generator
3
+ *
4
+ * Provides model-agnostic base functionality for all generator implementations.
5
+ * This is an abstract base class, not a concrete implementation.
6
+ *
7
+ * ARCHITECTURAL NOTE:
8
+ * Similar to BaseUniversalEmbedder, this class provides shared infrastructure:
9
+ * - Model lifecycle management (loading, cleanup, disposal)
10
+ * - Token budget management
11
+ * - Error handling with helpful messages
12
+ * - Common utility methods
13
+ *
14
+ * IMPLEMENTATION LAYERS:
15
+ * - Text: InstructGenerator extends this class (SmolLM2-Instruct)
16
+ * - Text: CausalLMGenerator extends this class (DistilGPT2)
17
+ *
18
+ * @experimental This feature is experimental and may change in future versions.
19
+ */
20
+ import { GenerationError } from './response-generator.js';
21
+ import { GeneratorRegistry } from './generator-registry.js';
22
+ import { buildPrompt, getDefaultStopSequences } from './prompt-templates.js';
23
+ // =============================================================================
24
+ // BASE GENERATOR ABSTRACT CLASS
25
+ // =============================================================================
26
+ /**
27
+ * Abstract base class for response generators
28
+ * Provides common functionality and lifecycle management
29
+ */
30
+ export class BaseResponseGenerator {
31
+ modelName;
32
+ _isLoaded = false;
33
+ _modelInfo;
34
+ _options;
35
+ constructor(modelName, options = {}) {
36
+ this.modelName = modelName;
37
+ const modelInfo = GeneratorRegistry.getGeneratorInfo(modelName);
38
+ if (!modelInfo) {
39
+ throw new Error(`Generator model '${modelName}' is not supported. ` +
40
+ `Supported models: ${GeneratorRegistry.getSupportedGenerators().join(', ')}`);
41
+ }
42
+ this._modelInfo = modelInfo;
43
+ this._options = options;
44
+ }
45
+ // =============================================================================
46
+ // PUBLIC INTERFACE IMPLEMENTATION
47
+ // =============================================================================
48
+ get modelType() {
49
+ return this._modelInfo.type;
50
+ }
51
+ get maxContextLength() {
52
+ return this._modelInfo.capabilities.maxContextLength;
53
+ }
54
+ get maxOutputLength() {
55
+ return this._modelInfo.capabilities.defaultMaxOutputTokens;
56
+ }
57
+ isLoaded() {
58
+ return this._isLoaded;
59
+ }
60
+ getModelInfo() {
61
+ return { ...this._modelInfo }; // Return a copy to prevent mutation
62
+ }
63
+ // =============================================================================
64
+ // DEFAULT IMPLEMENTATION
65
+ // =============================================================================
66
+ /**
67
+ * Generate a response based on query and retrieved chunks
68
+ * This method orchestrates the generation pipeline
69
+ */
70
+ async generate(request) {
71
+ if (!this._isLoaded) {
72
+ await this.loadModel();
73
+ }
74
+ const startTime = Date.now();
75
+ try {
76
+ // Get generation parameters with defaults
77
+ const maxTokens = request.maxTokens ?? this._modelInfo.capabilities.defaultMaxOutputTokens;
78
+ const temperature = request.temperature ?? this._modelInfo.capabilities.recommendedTemperature;
79
+ const topP = request.topP ?? 0.9;
80
+ const topK = request.topK ?? 50;
81
+ const repetitionPenalty = request.repetitionPenalty ?? 1.1;
82
+ const stopSequences = request.stopSequences ?? getDefaultStopSequences(this.modelType);
83
+ // Get max chunks for context (configurable, with model-specific default)
84
+ const maxChunksForContext = request.maxChunksForContext ??
85
+ this._modelInfo.capabilities.defaultMaxChunksForContext;
86
+ // Limit chunks to maxChunksForContext (assumes chunks are already reranked)
87
+ const totalChunks = request.chunks.length;
88
+ const limitedChunks = request.chunks.slice(0, maxChunksForContext);
89
+ if (totalChunks > maxChunksForContext) {
90
+ console.log(`📊 Using top ${maxChunksForContext} of ${totalChunks} reranked chunks for generation`);
91
+ }
92
+ // Build the prompt with context
93
+ const builtPrompt = buildPrompt({
94
+ query: request.query,
95
+ chunks: limitedChunks,
96
+ modelType: this.modelType,
97
+ systemPrompt: request.systemPrompt,
98
+ maxContextLength: this.maxContextLength,
99
+ reservedOutputTokens: maxTokens,
100
+ includeSourceAttribution: request.includeSourceAttribution
101
+ });
102
+ // Log context info
103
+ if (builtPrompt.contextInfo.truncated) {
104
+ console.warn(`⚠️ Context truncated: Only ${builtPrompt.contextInfo.chunksIncluded} of ` +
105
+ `${builtPrompt.contextInfo.totalChunks} chunks fit in context window`);
106
+ }
107
+ // Generate response
108
+ const result = await this.generateText(builtPrompt.prompt, {
109
+ maxTokens,
110
+ temperature,
111
+ topP,
112
+ topK,
113
+ repetitionPenalty,
114
+ stopSequences
115
+ });
116
+ const generationTimeMs = Date.now() - startTime;
117
+ // Clean up the response text
118
+ const cleanedResponse = this.cleanResponseText(result.text);
119
+ return {
120
+ response: cleanedResponse,
121
+ tokensUsed: result.promptTokens + result.completionTokens,
122
+ truncated: builtPrompt.contextInfo.truncated,
123
+ modelName: this.modelName,
124
+ generationTimeMs,
125
+ metadata: {
126
+ promptTokens: result.promptTokens,
127
+ completionTokens: result.completionTokens,
128
+ chunksIncluded: builtPrompt.contextInfo.chunksIncluded,
129
+ totalChunks: totalChunks, // Report original total, not limited
130
+ finishReason: result.finishReason
131
+ }
132
+ };
133
+ }
134
+ catch (error) {
135
+ const generationTimeMs = Date.now() - startTime;
136
+ if (error instanceof GenerationError) {
137
+ throw error;
138
+ }
139
+ throw new GenerationError(this.modelName, 'generation', `Generation failed: ${error instanceof Error ? error.message : 'Unknown error'}`, error instanceof Error ? error : undefined);
140
+ }
141
+ }
142
+ // =============================================================================
143
+ // PROTECTED HELPER METHODS
144
+ // =============================================================================
145
+ /**
146
+ * Validate that the model is loaded before operations
147
+ */
148
+ ensureLoaded() {
149
+ if (!this._isLoaded) {
150
+ throw new GenerationError(this.modelName, 'generation', `Model '${this.modelName}' is not loaded. Call loadModel() first.`);
151
+ }
152
+ }
153
+ /**
154
+ * Clean up response text by removing artifacts
155
+ */
156
+ cleanResponseText(text) {
157
+ let cleaned = text.trim();
158
+ // Remove common artifacts
159
+ const artifactsToRemove = [
160
+ '<|im_end|>',
161
+ '<|im_start|>',
162
+ '<|endoftext|>',
163
+ '<|assistant|>',
164
+ '<|user|>',
165
+ '<|system|>'
166
+ ];
167
+ for (const artifact of artifactsToRemove) {
168
+ cleaned = cleaned.split(artifact)[0];
169
+ }
170
+ // Remove trailing incomplete sentences (if cut off at max tokens)
171
+ if (cleaned.length > 0 && !cleaned.match(/[.!?]$/)) {
172
+ const lastSentenceEnd = Math.max(cleaned.lastIndexOf('.'), cleaned.lastIndexOf('!'), cleaned.lastIndexOf('?'));
173
+ if (lastSentenceEnd > cleaned.length * 0.5) {
174
+ cleaned = cleaned.substring(0, lastSentenceEnd + 1);
175
+ }
176
+ }
177
+ return cleaned.trim();
178
+ }
179
+ /**
180
+ * Log model loading progress
181
+ */
182
+ logModelLoading(stage, details) {
183
+ const message = `[${this.modelName}] ${stage}`;
184
+ if (details) {
185
+ console.log(`${message}: ${details}`);
186
+ }
187
+ else {
188
+ console.log(message);
189
+ }
190
+ }
191
+ /**
192
+ * Handle model loading errors with helpful messages
193
+ */
194
+ handleLoadingError(error) {
195
+ const baseMessage = `Failed to load generator model '${this.modelName}': ${error.message}`;
196
+ // Provide specific guidance based on error type
197
+ if (error.message.includes('network') || error.message.includes('fetch')) {
198
+ return new GenerationError(this.modelName, 'loading', `${baseMessage}\n` +
199
+ `This appears to be a network error. Please check your internet connection ` +
200
+ `and ensure the model repository is accessible.`, error);
201
+ }
202
+ if (error.message.includes('memory') || error.message.includes('OOM')) {
203
+ return new GenerationError(this.modelName, 'loading', `${baseMessage}\n` +
204
+ `This appears to be a memory error. The model requires ` +
205
+ `${this._modelInfo.requirements.minimumMemory}MB. Try closing other applications ` +
206
+ `or using a smaller model like 'Xenova/distilgpt2'.`, error);
207
+ }
208
+ return new GenerationError(this.modelName, 'loading', baseMessage, error);
209
+ }
210
+ }
211
+ /**
212
+ * Create generator options with defaults
213
+ */
214
+ export function createGeneratorOptions(options = {}) {
215
+ return {
216
+ timeout: 60000, // 60 seconds
217
+ enableGPU: false,
218
+ logLevel: 'info',
219
+ ...options
220
+ };
221
+ }
222
+ //# sourceMappingURL=abstract-generator.js.map
@@ -27,10 +27,12 @@ export class BinaryIndexFormat {
27
27
  * @param data Index data to serialize
28
28
  */
29
29
  static async save(indexPath, data) {
30
- // Calculate total size
30
+ // Use actual vector count to ensure accurate file size
31
+ const actualVectorCount = data.vectors.length;
32
+ // Calculate total size based on actual vectors
31
33
  const headerSize = 24; // 6 uint32 fields
32
34
  const vectorSize = 4 + (data.dimensions * 4); // id + vector
33
- const totalSize = headerSize + (data.currentSize * vectorSize);
35
+ const totalSize = headerSize + (actualVectorCount * vectorSize);
34
36
  const buffer = new ArrayBuffer(totalSize);
35
37
  const view = new DataView(buffer);
36
38
  let offset = 0;
@@ -45,7 +47,8 @@ export class BinaryIndexFormat {
45
47
  offset += 4;
46
48
  view.setUint32(offset, data.seed, true);
47
49
  offset += 4;
48
- view.setUint32(offset, data.currentSize, true);
50
+ // Write actual vector count in header
51
+ view.setUint32(offset, actualVectorCount, true);
49
52
  offset += 4;
50
53
  // Write vectors
51
54
  for (const item of data.vectors) {
@@ -187,6 +190,9 @@ export class BinaryIndexFormat {
187
190
  const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
188
191
  let offset = 0;
189
192
  // Read basic header (24 bytes, all little-endian)
193
+ if (buffer.byteLength < 24) {
194
+ throw new Error(`Index file too small: expected at least 24 bytes, got ${buffer.byteLength}`);
195
+ }
190
196
  const dimensions = view.getUint32(offset, true);
191
197
  offset += 4;
192
198
  const maxElements = view.getUint32(offset, true);
@@ -199,10 +205,20 @@ export class BinaryIndexFormat {
199
205
  offset += 4;
200
206
  const currentSize = view.getUint32(offset, true);
201
207
  offset += 4;
202
- // Check if this is the extended grouped format (40+ bytes header)
203
- const hasGroups = buffer.byteLength >= 40 ? view.getUint32(offset, true) : 0;
204
- if (hasGroups === 1 && buffer.byteLength >= 40) {
205
- // Load grouped format
208
+ // Calculate expected size for original format
209
+ const vectorSize = 4 + (dimensions * 4); // id + vector
210
+ const expectedOriginalSize = 24 + (currentSize * vectorSize);
211
+ // Check if this is the extended grouped format (44 bytes header)
212
+ // Extended header has: 24 bytes basic + 4 bytes hasGroups + 16 bytes for offsets/counts = 44 bytes
213
+ // Only check for grouped format if file is larger than expected original format size
214
+ const hasGroups = buffer.byteLength > expectedOriginalSize && buffer.byteLength >= 44 && offset + 4 <= buffer.byteLength
215
+ ? view.getUint32(offset, true)
216
+ : 0;
217
+ if (hasGroups === 1 && buffer.byteLength >= 44) {
218
+ // Load grouped format - ensure we have enough bytes for extended header
219
+ if (offset + 20 > buffer.byteLength) {
220
+ throw new Error(`Index file too small for grouped format: expected at least ${offset + 20} bytes, got ${buffer.byteLength}`);
221
+ }
206
222
  const textOffset = view.getUint32(offset + 4, true);
207
223
  const textCount = view.getUint32(offset + 8, true);
208
224
  const imageOffset = view.getUint32(offset + 12, true);
@@ -215,14 +231,23 @@ export class BinaryIndexFormat {
215
231
  if (offset % 4 !== 0) {
216
232
  throw new Error(`Offset ${offset} is not 4-byte aligned`);
217
233
  }
234
+ // Check bounds before reading vector ID
235
+ if (offset + 4 > buffer.byteLength) {
236
+ throw new Error(`Text vector ID at offset ${offset} is outside the bounds of the DataView (buffer size: ${buffer.byteLength})`);
237
+ }
218
238
  // Read vector ID
219
239
  const id = view.getUint32(offset, true);
220
240
  offset += 4;
241
+ // Check bounds before reading vector data
242
+ const vectorDataSize = dimensions * 4;
243
+ if (offset + vectorDataSize > buffer.byteLength) {
244
+ throw new Error(`Text vector data at offset ${offset} would exceed buffer bounds (buffer size: ${buffer.byteLength}, required: ${offset + vectorDataSize})`);
245
+ }
221
246
  // Zero-copy Float32Array view
222
247
  const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
223
248
  // Copy to avoid buffer lifecycle issues
224
249
  const vector = new Float32Array(vectorView);
225
- offset += dimensions * 4;
250
+ offset += vectorDataSize;
226
251
  textVectors.push({ id, vector });
227
252
  }
228
253
  // Load image vectors
@@ -233,14 +258,23 @@ export class BinaryIndexFormat {
233
258
  if (offset % 4 !== 0) {
234
259
  throw new Error(`Offset ${offset} is not 4-byte aligned`);
235
260
  }
261
+ // Check bounds before reading vector ID
262
+ if (offset + 4 > buffer.byteLength) {
263
+ throw new Error(`Image vector ID at offset ${offset} is outside the bounds of the DataView (buffer size: ${buffer.byteLength})`);
264
+ }
236
265
  // Read vector ID
237
266
  const id = view.getUint32(offset, true);
238
267
  offset += 4;
268
+ // Check bounds before reading vector data
269
+ const vectorDataSize = dimensions * 4;
270
+ if (offset + vectorDataSize > buffer.byteLength) {
271
+ throw new Error(`Image vector data at offset ${offset} would exceed buffer bounds (buffer size: ${buffer.byteLength}, required: ${offset + vectorDataSize})`);
272
+ }
239
273
  // Zero-copy Float32Array view
240
274
  const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
241
275
  // Copy to avoid buffer lifecycle issues
242
276
  const vector = new Float32Array(vectorView);
243
- offset += dimensions * 4;
277
+ offset += vectorDataSize;
244
278
  imageVectors.push({ id, vector });
245
279
  }
246
280
  // Combine all vectors for backward compatibility
@@ -266,14 +300,23 @@ export class BinaryIndexFormat {
266
300
  if (offset % 4 !== 0) {
267
301
  throw new Error(`Offset ${offset} is not 4-byte aligned`);
268
302
  }
303
+ // Check bounds before reading vector ID
304
+ if (offset + 4 > buffer.byteLength) {
305
+ throw new Error(`Offset ${offset} is outside the bounds of the DataView (buffer size: ${buffer.byteLength})`);
306
+ }
269
307
  // Read vector ID
270
308
  const id = view.getUint32(offset, true);
271
309
  offset += 4;
310
+ // Check bounds before reading vector data
311
+ const vectorDataSize = dimensions * 4;
312
+ if (offset + vectorDataSize > buffer.byteLength) {
313
+ throw new Error(`Vector data at offset ${offset} would exceed buffer bounds (buffer size: ${buffer.byteLength}, required: ${offset + vectorDataSize})`);
314
+ }
272
315
  // Zero-copy Float32Array view (fast!)
273
316
  const vectorView = new Float32Array(buffer.buffer, buffer.byteOffset + offset, dimensions);
274
317
  // Copy to avoid buffer lifecycle issues
275
318
  const vector = new Float32Array(vectorView);
276
- offset += dimensions * 4;
319
+ offset += vectorDataSize;
277
320
  vectors.push({ id, vector });
278
321
  }
279
322
  return {
@@ -210,4 +210,60 @@ export declare function updateStorageStats(connection: DatabaseConnection, stats
210
210
  filesystemRefs?: number;
211
211
  lastCleanup?: Date;
212
212
  }): Promise<void>;
213
+ /**
214
+ * Result of a database reset operation
215
+ */
216
+ export interface DatabaseResetResult {
217
+ /** Whether the reset was successful */
218
+ success: boolean;
219
+ /** Number of documents deleted */
220
+ documentsDeleted: number;
221
+ /** Number of chunks deleted */
222
+ chunksDeleted: number;
223
+ /** Number of content metadata entries deleted */
224
+ contentMetadataDeleted: number;
225
+ /** Whether system_info was preserved or cleared */
226
+ systemInfoCleared: boolean;
227
+ /** Time taken for the reset operation in milliseconds */
228
+ resetTimeMs: number;
229
+ }
230
+ /**
231
+ * Options for database reset operation
232
+ */
233
+ export interface DatabaseResetOptions {
234
+ /** Whether to preserve system_info (mode, model configuration) - default: false */
235
+ preserveSystemInfo?: boolean;
236
+ /** Whether to run VACUUM after deletion to reclaim space - default: true */
237
+ runVacuum?: boolean;
238
+ }
239
+ /**
240
+ * Reset the database by deleting all data while keeping the schema intact.
241
+ * This is a safer alternative to file deletion that avoids file locking issues on Windows.
242
+ *
243
+ * This function:
244
+ * 1. Deletes all rows from chunks, documents, content_metadata tables
245
+ * 2. Optionally clears system_info (mode/model configuration)
246
+ * 3. Resets storage_stats counters
247
+ * 4. Optionally runs VACUUM to reclaim disk space
248
+ *
249
+ * @param connection - Database connection object
250
+ * @param options - Reset options
251
+ * @returns Promise resolving to reset result statistics
252
+ *
253
+ * @example
254
+ * ```typescript
255
+ * const db = await openDatabase('./db.sqlite');
256
+ * const result = await resetDatabase(db, { preserveSystemInfo: false });
257
+ * console.log(`Deleted ${result.documentsDeleted} documents and ${result.chunksDeleted} chunks`);
258
+ * ```
259
+ */
260
+ export declare function resetDatabase(connection: DatabaseConnection, options?: DatabaseResetOptions): Promise<DatabaseResetResult>;
261
+ /**
262
+ * Check if the database has any data (documents, chunks, or content)
263
+ * Useful for determining if a reset is needed
264
+ *
265
+ * @param connection - Database connection object
266
+ * @returns Promise resolving to true if database has data, false if empty
267
+ */
268
+ export declare function hasDatabaseData(connection: DatabaseConnection): Promise<boolean>;
213
269
  //# sourceMappingURL=db.d.ts.map
@@ -892,4 +892,109 @@ export async function updateStorageStats(connection, stats) {
892
892
  throw new Error(`Failed to update storage stats: ${error instanceof Error ? error.message : 'Unknown error'}`);
893
893
  }
894
894
  }
895
+ /**
896
+ * Reset the database by deleting all data while keeping the schema intact.
897
+ * This is a safer alternative to file deletion that avoids file locking issues on Windows.
898
+ *
899
+ * This function:
900
+ * 1. Deletes all rows from chunks, documents, content_metadata tables
901
+ * 2. Optionally clears system_info (mode/model configuration)
902
+ * 3. Resets storage_stats counters
903
+ * 4. Optionally runs VACUUM to reclaim disk space
904
+ *
905
+ * @param connection - Database connection object
906
+ * @param options - Reset options
907
+ * @returns Promise resolving to reset result statistics
908
+ *
909
+ * @example
910
+ * ```typescript
911
+ * const db = await openDatabase('./db.sqlite');
912
+ * const result = await resetDatabase(db, { preserveSystemInfo: false });
913
+ * console.log(`Deleted ${result.documentsDeleted} documents and ${result.chunksDeleted} chunks`);
914
+ * ```
915
+ */
916
+ export async function resetDatabase(connection, options = {}) {
917
+ const startTime = Date.now();
918
+ const { preserveSystemInfo = false, runVacuum = true } = options;
919
+ try {
920
+ console.log('🔄 Starting database reset...');
921
+ // Get counts before deletion for reporting
922
+ const docCountResult = await connection.get('SELECT COUNT(*) as count FROM documents');
923
+ const chunkCountResult = await connection.get('SELECT COUNT(*) as count FROM chunks');
924
+ const contentMetadataCountResult = await connection.get('SELECT COUNT(*) as count FROM content_metadata');
925
+ const documentsDeleted = docCountResult?.count || 0;
926
+ const chunksDeleted = chunkCountResult?.count || 0;
927
+ const contentMetadataDeleted = contentMetadataCountResult?.count || 0;
928
+ // Delete in order respecting foreign key constraints
929
+ // chunks → documents → content_metadata (chunks reference documents, documents reference content_metadata)
930
+ console.log(' Deleting chunks...');
931
+ await connection.run('DELETE FROM chunks');
932
+ console.log(' Deleting documents...');
933
+ await connection.run('DELETE FROM documents');
934
+ console.log(' Deleting content_metadata...');
935
+ await connection.run('DELETE FROM content_metadata');
936
+ // Reset storage_stats counters
937
+ console.log(' Resetting storage_stats...');
938
+ await connection.run(`
939
+ UPDATE storage_stats SET
940
+ content_dir_files = 0,
941
+ content_dir_size = 0,
942
+ filesystem_refs = 0,
943
+ updated_at = CURRENT_TIMESTAMP
944
+ WHERE id = 1
945
+ `);
946
+ // Optionally clear system_info
947
+ let systemInfoCleared = false;
948
+ if (!preserveSystemInfo) {
949
+ console.log(' Clearing system_info...');
950
+ await connection.run('DELETE FROM system_info WHERE id = 1');
951
+ systemInfoCleared = true;
952
+ }
953
+ else {
954
+ console.log(' Preserving system_info (mode/model configuration)');
955
+ }
956
+ // Run VACUUM to reclaim disk space
957
+ if (runVacuum) {
958
+ console.log(' Running VACUUM to reclaim disk space...');
959
+ await connection.run('VACUUM');
960
+ }
961
+ const resetTimeMs = Date.now() - startTime;
962
+ console.log(`✓ Database reset complete in ${resetTimeMs}ms`);
963
+ console.log(` Documents deleted: ${documentsDeleted}`);
964
+ console.log(` Chunks deleted: ${chunksDeleted}`);
965
+ console.log(` Content metadata deleted: ${contentMetadataDeleted}`);
966
+ console.log(` System info cleared: ${systemInfoCleared}`);
967
+ return {
968
+ success: true,
969
+ documentsDeleted,
970
+ chunksDeleted,
971
+ contentMetadataDeleted,
972
+ systemInfoCleared,
973
+ resetTimeMs
974
+ };
975
+ }
976
+ catch (error) {
977
+ const resetTimeMs = Date.now() - startTime;
978
+ console.error(`❌ Database reset failed after ${resetTimeMs}ms:`, error);
979
+ throw new Error(`Failed to reset database: ${error instanceof Error ? error.message : 'Unknown error'}`);
980
+ }
981
+ }
982
+ /**
983
+ * Check if the database has any data (documents, chunks, or content)
984
+ * Useful for determining if a reset is needed
985
+ *
986
+ * @param connection - Database connection object
987
+ * @returns Promise resolving to true if database has data, false if empty
988
+ */
989
+ export async function hasDatabaseData(connection) {
990
+ try {
991
+ const docCount = await connection.get('SELECT COUNT(*) as count FROM documents');
992
+ const chunkCount = await connection.get('SELECT COUNT(*) as count FROM chunks');
993
+ return (docCount?.count || 0) > 0 || (chunkCount?.count || 0) > 0;
994
+ }
995
+ catch (error) {
996
+ // If tables don't exist, consider it empty
997
+ return false;
998
+ }
999
+ }
895
1000
  //# sourceMappingURL=db.js.map
@@ -0,0 +1,114 @@
1
+ /**
2
+ * CORE MODULE — Generator Registry for RAG Response Generation
3
+ *
4
+ * Centralized registry of supported generator models with validation and
5
+ * compatibility checking. Follows the same patterns as model-registry.ts.
6
+ *
7
+ * SUPPORTED MODELS:
8
+ * - HuggingFaceTB/SmolLM2-135M-Instruct: Balanced instruct model (DEFAULT, 3 chunks)
9
+ * - HuggingFaceTB/SmolLM2-360M-Instruct: Higher quality instruct model (5 chunks)
10
+ *
11
+ * PREREQUISITES:
12
+ * - Reranking must be enabled for response generation to ensure quality context
13
+ *
14
+ * @experimental This feature is experimental and may change in future versions.
15
+ */
16
+ import type { GeneratorModelInfo, GeneratorModelType, GeneratorValidationResult, GeneratorCapabilities } from './response-generator.js';
17
+ /**
18
+ * Registry of supported generator models with their metadata and capabilities
19
+ */
20
+ export declare const SUPPORTED_GENERATORS: Record<string, GeneratorModelInfo>;
21
+ /** Default generator model name */
22
+ export declare const DEFAULT_GENERATOR_MODEL = "HuggingFaceTB/SmolLM2-135M-Instruct";
23
+ /**
24
+ * Generator registry class providing validation and model information services
25
+ */
26
+ export declare class GeneratorRegistry {
27
+ /**
28
+ * Gets generator model information for a given model name
29
+ * @param modelName - Name of the generator model
30
+ * @returns Generator model information or null if not supported
31
+ */
32
+ static getGeneratorInfo(modelName: string): GeneratorModelInfo | null;
33
+ /**
34
+ * Validates a generator model name and returns compatibility information
35
+ * @param modelName - Name of the model to validate
36
+ * @returns Validation result with errors, warnings, and suggestions
37
+ */
38
+ static validateGenerator(modelName: string): GeneratorValidationResult;
39
+ /**
40
+ * Lists all supported generator models
41
+ * @param modelType - Optional filter by model type
42
+ * @returns Array of supported generator model names
43
+ */
44
+ static getSupportedGenerators(modelType?: GeneratorModelType): string[];
45
+ /**
46
+ * Gets the default generator model name
47
+ * @returns Default generator model name
48
+ */
49
+ static getDefaultGenerator(): string;
50
+ /**
51
+ * Gets generators by type
52
+ * @param type - Generator type ('causal-lm' or 'instruct')
53
+ * @returns Array of model names matching the type
54
+ */
55
+ static getGeneratorsByType(type: GeneratorModelType): string[];
56
+ /**
57
+ * Checks if a generator model supports a specific capability
58
+ * @param modelName - Name of the model
59
+ * @param capability - Capability to check
60
+ * @returns True if the model supports the capability
61
+ */
62
+ static supportsCapability(modelName: string, capability: keyof GeneratorCapabilities): boolean;
63
+ /**
64
+ * Gets generators similar to the given model name (for suggestions)
65
+ * @private
66
+ */
67
+ private static getSimilarGenerators;
68
+ /**
69
+ * Validates system compatibility for a generator model
70
+ * @param modelName - Name of the model
71
+ * @param systemCapabilities - System capabilities to check against
72
+ * @returns Validation result with compatibility information
73
+ */
74
+ static validateSystemCompatibility(modelName: string, systemCapabilities: {
75
+ availableMemory?: number;
76
+ platform?: string;
77
+ transformersJsVersion?: string;
78
+ }): GeneratorValidationResult;
79
+ }
80
+ /**
81
+ * Gets the generator type for a given model name
82
+ * @param modelName - Name of the model
83
+ * @returns Generator type or null if model not supported
84
+ */
85
+ export declare function getGeneratorType(modelName: string): GeneratorModelType | null;
86
+ /**
87
+ * Checks if a model is an instruction-tuned model
88
+ * @param modelName - Name of the model
89
+ * @returns True if the model is instruction-tuned
90
+ */
91
+ export declare function isInstructionTunedModel(modelName: string): boolean;
92
+ /**
93
+ * Gets the maximum context length for a generator model
94
+ * @param modelName - Name of the model
95
+ * @returns Maximum context length or null if model not supported
96
+ */
97
+ export declare function getMaxContextLength(modelName: string): number | null;
98
+ /**
99
+ * Gets recommended generation settings for a model
100
+ * @param modelName - Name of the model
101
+ * @returns Recommended settings or null if model not supported
102
+ */
103
+ export declare function getRecommendedSettings(modelName: string): {
104
+ temperature: number;
105
+ maxTokens: number;
106
+ maxChunksForContext: number;
107
+ } | null;
108
+ /**
109
+ * Gets the default maximum chunks for context for a generator model
110
+ * @param modelName - Name of the model
111
+ * @returns Default max chunks for context or null if model not supported
112
+ */
113
+ export declare function getDefaultMaxChunksForContext(modelName: string): number | null;
114
+ //# sourceMappingURL=generator-registry.d.ts.map