rag-lite-ts 2.2.0 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -5
- package/dist/cjs/cli/indexer.js +73 -15
- package/dist/cjs/cli/search.js +77 -2
- package/dist/cjs/cli/ui-server.d.ts +5 -0
- package/dist/cjs/cli/ui-server.js +152 -0
- package/dist/cjs/cli.js +53 -7
- package/dist/cjs/core/abstract-generator.d.ts +97 -0
- package/dist/cjs/core/abstract-generator.js +222 -0
- package/dist/cjs/core/binary-index-format.js +53 -10
- package/dist/cjs/core/db.d.ts +56 -0
- package/dist/cjs/core/db.js +105 -0
- package/dist/cjs/core/generator-registry.d.ts +114 -0
- package/dist/cjs/core/generator-registry.js +280 -0
- package/dist/cjs/core/index.d.ts +4 -0
- package/dist/cjs/core/index.js +11 -0
- package/dist/cjs/core/ingestion.js +3 -0
- package/dist/cjs/core/knowledge-base-manager.d.ts +109 -0
- package/dist/cjs/core/knowledge-base-manager.js +256 -0
- package/dist/cjs/core/lazy-dependency-loader.d.ts +43 -0
- package/dist/cjs/core/lazy-dependency-loader.js +111 -2
- package/dist/cjs/core/prompt-templates.d.ts +138 -0
- package/dist/cjs/core/prompt-templates.js +225 -0
- package/dist/cjs/core/response-generator.d.ts +132 -0
- package/dist/cjs/core/response-generator.js +69 -0
- package/dist/cjs/core/search-pipeline.js +1 -1
- package/dist/cjs/core/search.d.ts +72 -1
- package/dist/cjs/core/search.js +80 -7
- package/dist/cjs/core/types.d.ts +1 -0
- package/dist/cjs/core/vector-index-messages.d.ts +52 -0
- package/dist/cjs/core/vector-index-messages.js +5 -0
- package/dist/cjs/core/vector-index-worker.d.ts +6 -0
- package/dist/cjs/core/vector-index-worker.js +314 -0
- package/dist/cjs/core/vector-index.d.ts +45 -10
- package/dist/cjs/core/vector-index.js +279 -218
- package/dist/cjs/factories/generator-factory.d.ts +88 -0
- package/dist/cjs/factories/generator-factory.js +151 -0
- package/dist/cjs/factories/index.d.ts +1 -0
- package/dist/cjs/factories/index.js +5 -0
- package/dist/cjs/factories/ingestion-factory.js +3 -7
- package/dist/cjs/factories/search-factory.js +11 -0
- package/dist/cjs/index-manager.d.ts +23 -3
- package/dist/cjs/index-manager.js +84 -15
- package/dist/cjs/index.d.ts +11 -1
- package/dist/cjs/index.js +19 -1
- package/dist/cjs/text/generators/causal-lm-generator.d.ts +65 -0
- package/dist/cjs/text/generators/causal-lm-generator.js +197 -0
- package/dist/cjs/text/generators/index.d.ts +10 -0
- package/dist/cjs/text/generators/index.js +10 -0
- package/dist/cjs/text/generators/instruct-generator.d.ts +62 -0
- package/dist/cjs/text/generators/instruct-generator.js +192 -0
- package/dist/esm/cli/indexer.js +73 -15
- package/dist/esm/cli/search.js +77 -2
- package/dist/esm/cli/ui-server.d.ts +5 -0
- package/dist/esm/cli/ui-server.js +152 -0
- package/dist/esm/cli.js +53 -7
- package/dist/esm/core/abstract-generator.d.ts +97 -0
- package/dist/esm/core/abstract-generator.js +222 -0
- package/dist/esm/core/binary-index-format.js +53 -10
- package/dist/esm/core/db.d.ts +56 -0
- package/dist/esm/core/db.js +105 -0
- package/dist/esm/core/generator-registry.d.ts +114 -0
- package/dist/esm/core/generator-registry.js +280 -0
- package/dist/esm/core/index.d.ts +4 -0
- package/dist/esm/core/index.js +11 -0
- package/dist/esm/core/ingestion.js +3 -0
- package/dist/esm/core/knowledge-base-manager.d.ts +109 -0
- package/dist/esm/core/knowledge-base-manager.js +256 -0
- package/dist/esm/core/lazy-dependency-loader.d.ts +43 -0
- package/dist/esm/core/lazy-dependency-loader.js +111 -2
- package/dist/esm/core/prompt-templates.d.ts +138 -0
- package/dist/esm/core/prompt-templates.js +225 -0
- package/dist/esm/core/response-generator.d.ts +132 -0
- package/dist/esm/core/response-generator.js +69 -0
- package/dist/esm/core/search-pipeline.js +1 -1
- package/dist/esm/core/search.d.ts +72 -1
- package/dist/esm/core/search.js +80 -7
- package/dist/esm/core/types.d.ts +1 -0
- package/dist/esm/core/vector-index-messages.d.ts +52 -0
- package/dist/esm/core/vector-index-messages.js +5 -0
- package/dist/esm/core/vector-index-worker.d.ts +6 -0
- package/dist/esm/core/vector-index-worker.js +314 -0
- package/dist/esm/core/vector-index.d.ts +45 -10
- package/dist/esm/core/vector-index.js +279 -218
- package/dist/esm/factories/generator-factory.d.ts +88 -0
- package/dist/esm/factories/generator-factory.js +151 -0
- package/dist/esm/factories/index.d.ts +1 -0
- package/dist/esm/factories/index.js +5 -0
- package/dist/esm/factories/ingestion-factory.js +3 -7
- package/dist/esm/factories/search-factory.js +11 -0
- package/dist/esm/index-manager.d.ts +23 -3
- package/dist/esm/index-manager.js +84 -15
- package/dist/esm/index.d.ts +11 -1
- package/dist/esm/index.js +19 -1
- package/dist/esm/text/generators/causal-lm-generator.d.ts +65 -0
- package/dist/esm/text/generators/causal-lm-generator.js +197 -0
- package/dist/esm/text/generators/index.d.ts +10 -0
- package/dist/esm/text/generators/index.js +10 -0
- package/dist/esm/text/generators/instruct-generator.d.ts +62 -0
- package/dist/esm/text/generators/instruct-generator.js +192 -0
- package/package.json +14 -7
package/dist/cjs/core/db.js
CHANGED
|
@@ -892,4 +892,109 @@ export async function updateStorageStats(connection, stats) {
|
|
|
892
892
|
throw new Error(`Failed to update storage stats: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
893
893
|
}
|
|
894
894
|
}
|
|
895
|
+
/**
|
|
896
|
+
* Reset the database by deleting all data while keeping the schema intact.
|
|
897
|
+
* This is a safer alternative to file deletion that avoids file locking issues on Windows.
|
|
898
|
+
*
|
|
899
|
+
* This function:
|
|
900
|
+
* 1. Deletes all rows from chunks, documents, content_metadata tables
|
|
901
|
+
* 2. Optionally clears system_info (mode/model configuration)
|
|
902
|
+
* 3. Resets storage_stats counters
|
|
903
|
+
* 4. Optionally runs VACUUM to reclaim disk space
|
|
904
|
+
*
|
|
905
|
+
* @param connection - Database connection object
|
|
906
|
+
* @param options - Reset options
|
|
907
|
+
* @returns Promise resolving to reset result statistics
|
|
908
|
+
*
|
|
909
|
+
* @example
|
|
910
|
+
* ```typescript
|
|
911
|
+
* const db = await openDatabase('./db.sqlite');
|
|
912
|
+
* const result = await resetDatabase(db, { preserveSystemInfo: false });
|
|
913
|
+
* console.log(`Deleted ${result.documentsDeleted} documents and ${result.chunksDeleted} chunks`);
|
|
914
|
+
* ```
|
|
915
|
+
*/
|
|
916
|
+
export async function resetDatabase(connection, options = {}) {
|
|
917
|
+
const startTime = Date.now();
|
|
918
|
+
const { preserveSystemInfo = false, runVacuum = true } = options;
|
|
919
|
+
try {
|
|
920
|
+
console.log('🔄 Starting database reset...');
|
|
921
|
+
// Get counts before deletion for reporting
|
|
922
|
+
const docCountResult = await connection.get('SELECT COUNT(*) as count FROM documents');
|
|
923
|
+
const chunkCountResult = await connection.get('SELECT COUNT(*) as count FROM chunks');
|
|
924
|
+
const contentMetadataCountResult = await connection.get('SELECT COUNT(*) as count FROM content_metadata');
|
|
925
|
+
const documentsDeleted = docCountResult?.count || 0;
|
|
926
|
+
const chunksDeleted = chunkCountResult?.count || 0;
|
|
927
|
+
const contentMetadataDeleted = contentMetadataCountResult?.count || 0;
|
|
928
|
+
// Delete in order respecting foreign key constraints
|
|
929
|
+
// chunks → documents → content_metadata (chunks reference documents, documents reference content_metadata)
|
|
930
|
+
console.log(' Deleting chunks...');
|
|
931
|
+
await connection.run('DELETE FROM chunks');
|
|
932
|
+
console.log(' Deleting documents...');
|
|
933
|
+
await connection.run('DELETE FROM documents');
|
|
934
|
+
console.log(' Deleting content_metadata...');
|
|
935
|
+
await connection.run('DELETE FROM content_metadata');
|
|
936
|
+
// Reset storage_stats counters
|
|
937
|
+
console.log(' Resetting storage_stats...');
|
|
938
|
+
await connection.run(`
|
|
939
|
+
UPDATE storage_stats SET
|
|
940
|
+
content_dir_files = 0,
|
|
941
|
+
content_dir_size = 0,
|
|
942
|
+
filesystem_refs = 0,
|
|
943
|
+
updated_at = CURRENT_TIMESTAMP
|
|
944
|
+
WHERE id = 1
|
|
945
|
+
`);
|
|
946
|
+
// Optionally clear system_info
|
|
947
|
+
let systemInfoCleared = false;
|
|
948
|
+
if (!preserveSystemInfo) {
|
|
949
|
+
console.log(' Clearing system_info...');
|
|
950
|
+
await connection.run('DELETE FROM system_info WHERE id = 1');
|
|
951
|
+
systemInfoCleared = true;
|
|
952
|
+
}
|
|
953
|
+
else {
|
|
954
|
+
console.log(' Preserving system_info (mode/model configuration)');
|
|
955
|
+
}
|
|
956
|
+
// Run VACUUM to reclaim disk space
|
|
957
|
+
if (runVacuum) {
|
|
958
|
+
console.log(' Running VACUUM to reclaim disk space...');
|
|
959
|
+
await connection.run('VACUUM');
|
|
960
|
+
}
|
|
961
|
+
const resetTimeMs = Date.now() - startTime;
|
|
962
|
+
console.log(`✓ Database reset complete in ${resetTimeMs}ms`);
|
|
963
|
+
console.log(` Documents deleted: ${documentsDeleted}`);
|
|
964
|
+
console.log(` Chunks deleted: ${chunksDeleted}`);
|
|
965
|
+
console.log(` Content metadata deleted: ${contentMetadataDeleted}`);
|
|
966
|
+
console.log(` System info cleared: ${systemInfoCleared}`);
|
|
967
|
+
return {
|
|
968
|
+
success: true,
|
|
969
|
+
documentsDeleted,
|
|
970
|
+
chunksDeleted,
|
|
971
|
+
contentMetadataDeleted,
|
|
972
|
+
systemInfoCleared,
|
|
973
|
+
resetTimeMs
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
catch (error) {
|
|
977
|
+
const resetTimeMs = Date.now() - startTime;
|
|
978
|
+
console.error(`❌ Database reset failed after ${resetTimeMs}ms:`, error);
|
|
979
|
+
throw new Error(`Failed to reset database: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
/**
|
|
983
|
+
* Check if the database has any data (documents, chunks, or content)
|
|
984
|
+
* Useful for determining if a reset is needed
|
|
985
|
+
*
|
|
986
|
+
* @param connection - Database connection object
|
|
987
|
+
* @returns Promise resolving to true if database has data, false if empty
|
|
988
|
+
*/
|
|
989
|
+
export async function hasDatabaseData(connection) {
|
|
990
|
+
try {
|
|
991
|
+
const docCount = await connection.get('SELECT COUNT(*) as count FROM documents');
|
|
992
|
+
const chunkCount = await connection.get('SELECT COUNT(*) as count FROM chunks');
|
|
993
|
+
return (docCount?.count || 0) > 0 || (chunkCount?.count || 0) > 0;
|
|
994
|
+
}
|
|
995
|
+
catch (error) {
|
|
996
|
+
// If tables don't exist, consider it empty
|
|
997
|
+
return false;
|
|
998
|
+
}
|
|
999
|
+
}
|
|
895
1000
|
//# sourceMappingURL=db.js.map
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CORE MODULE — Generator Registry for RAG Response Generation
|
|
3
|
+
*
|
|
4
|
+
* Centralized registry of supported generator models with validation and
|
|
5
|
+
* compatibility checking. Follows the same patterns as model-registry.ts.
|
|
6
|
+
*
|
|
7
|
+
* SUPPORTED MODELS:
|
|
8
|
+
* - HuggingFaceTB/SmolLM2-135M-Instruct: Balanced instruct model (DEFAULT, 3 chunks)
|
|
9
|
+
* - HuggingFaceTB/SmolLM2-360M-Instruct: Higher quality instruct model (5 chunks)
|
|
10
|
+
*
|
|
11
|
+
* PREREQUISITES:
|
|
12
|
+
* - Reranking must be enabled for response generation to ensure quality context
|
|
13
|
+
*
|
|
14
|
+
* @experimental This feature is experimental and may change in future versions.
|
|
15
|
+
*/
|
|
16
|
+
import type { GeneratorModelInfo, GeneratorModelType, GeneratorValidationResult, GeneratorCapabilities } from './response-generator.js';
|
|
17
|
+
/**
|
|
18
|
+
* Registry of supported generator models with their metadata and capabilities
|
|
19
|
+
*/
|
|
20
|
+
export declare const SUPPORTED_GENERATORS: Record<string, GeneratorModelInfo>;
|
|
21
|
+
/** Default generator model name */
|
|
22
|
+
export declare const DEFAULT_GENERATOR_MODEL = "HuggingFaceTB/SmolLM2-135M-Instruct";
|
|
23
|
+
/**
|
|
24
|
+
* Generator registry class providing validation and model information services
|
|
25
|
+
*/
|
|
26
|
+
export declare class GeneratorRegistry {
|
|
27
|
+
/**
|
|
28
|
+
* Gets generator model information for a given model name
|
|
29
|
+
* @param modelName - Name of the generator model
|
|
30
|
+
* @returns Generator model information or null if not supported
|
|
31
|
+
*/
|
|
32
|
+
static getGeneratorInfo(modelName: string): GeneratorModelInfo | null;
|
|
33
|
+
/**
|
|
34
|
+
* Validates a generator model name and returns compatibility information
|
|
35
|
+
* @param modelName - Name of the model to validate
|
|
36
|
+
* @returns Validation result with errors, warnings, and suggestions
|
|
37
|
+
*/
|
|
38
|
+
static validateGenerator(modelName: string): GeneratorValidationResult;
|
|
39
|
+
/**
|
|
40
|
+
* Lists all supported generator models
|
|
41
|
+
* @param modelType - Optional filter by model type
|
|
42
|
+
* @returns Array of supported generator model names
|
|
43
|
+
*/
|
|
44
|
+
static getSupportedGenerators(modelType?: GeneratorModelType): string[];
|
|
45
|
+
/**
|
|
46
|
+
* Gets the default generator model name
|
|
47
|
+
* @returns Default generator model name
|
|
48
|
+
*/
|
|
49
|
+
static getDefaultGenerator(): string;
|
|
50
|
+
/**
|
|
51
|
+
* Gets generators by type
|
|
52
|
+
* @param type - Generator type ('causal-lm' or 'instruct')
|
|
53
|
+
* @returns Array of model names matching the type
|
|
54
|
+
*/
|
|
55
|
+
static getGeneratorsByType(type: GeneratorModelType): string[];
|
|
56
|
+
/**
|
|
57
|
+
* Checks if a generator model supports a specific capability
|
|
58
|
+
* @param modelName - Name of the model
|
|
59
|
+
* @param capability - Capability to check
|
|
60
|
+
* @returns True if the model supports the capability
|
|
61
|
+
*/
|
|
62
|
+
static supportsCapability(modelName: string, capability: keyof GeneratorCapabilities): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* Gets generators similar to the given model name (for suggestions)
|
|
65
|
+
* @private
|
|
66
|
+
*/
|
|
67
|
+
private static getSimilarGenerators;
|
|
68
|
+
/**
|
|
69
|
+
* Validates system compatibility for a generator model
|
|
70
|
+
* @param modelName - Name of the model
|
|
71
|
+
* @param systemCapabilities - System capabilities to check against
|
|
72
|
+
* @returns Validation result with compatibility information
|
|
73
|
+
*/
|
|
74
|
+
static validateSystemCompatibility(modelName: string, systemCapabilities: {
|
|
75
|
+
availableMemory?: number;
|
|
76
|
+
platform?: string;
|
|
77
|
+
transformersJsVersion?: string;
|
|
78
|
+
}): GeneratorValidationResult;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Gets the generator type for a given model name
|
|
82
|
+
* @param modelName - Name of the model
|
|
83
|
+
* @returns Generator type or null if model not supported
|
|
84
|
+
*/
|
|
85
|
+
export declare function getGeneratorType(modelName: string): GeneratorModelType | null;
|
|
86
|
+
/**
|
|
87
|
+
* Checks if a model is an instruction-tuned model
|
|
88
|
+
* @param modelName - Name of the model
|
|
89
|
+
* @returns True if the model is instruction-tuned
|
|
90
|
+
*/
|
|
91
|
+
export declare function isInstructionTunedModel(modelName: string): boolean;
|
|
92
|
+
/**
|
|
93
|
+
* Gets the maximum context length for a generator model
|
|
94
|
+
* @param modelName - Name of the model
|
|
95
|
+
* @returns Maximum context length or null if model not supported
|
|
96
|
+
*/
|
|
97
|
+
export declare function getMaxContextLength(modelName: string): number | null;
|
|
98
|
+
/**
|
|
99
|
+
* Gets recommended generation settings for a model
|
|
100
|
+
* @param modelName - Name of the model
|
|
101
|
+
* @returns Recommended settings or null if model not supported
|
|
102
|
+
*/
|
|
103
|
+
export declare function getRecommendedSettings(modelName: string): {
|
|
104
|
+
temperature: number;
|
|
105
|
+
maxTokens: number;
|
|
106
|
+
maxChunksForContext: number;
|
|
107
|
+
} | null;
|
|
108
|
+
/**
|
|
109
|
+
* Gets the default maximum chunks for context for a generator model
|
|
110
|
+
* @param modelName - Name of the model
|
|
111
|
+
* @returns Default max chunks for context or null if model not supported
|
|
112
|
+
*/
|
|
113
|
+
export declare function getDefaultMaxChunksForContext(modelName: string): number | null;
|
|
114
|
+
//# sourceMappingURL=generator-registry.d.ts.map
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CORE MODULE — Generator Registry for RAG Response Generation
|
|
3
|
+
*
|
|
4
|
+
* Centralized registry of supported generator models with validation and
|
|
5
|
+
* compatibility checking. Follows the same patterns as model-registry.ts.
|
|
6
|
+
*
|
|
7
|
+
* SUPPORTED MODELS:
|
|
8
|
+
* - HuggingFaceTB/SmolLM2-135M-Instruct: Balanced instruct model (DEFAULT, 3 chunks)
|
|
9
|
+
* - HuggingFaceTB/SmolLM2-360M-Instruct: Higher quality instruct model (5 chunks)
|
|
10
|
+
*
|
|
11
|
+
* PREREQUISITES:
|
|
12
|
+
* - Reranking must be enabled for response generation to ensure quality context
|
|
13
|
+
*
|
|
14
|
+
* @experimental This feature is experimental and may change in future versions.
|
|
15
|
+
*/
|
|
16
|
+
// =============================================================================
|
|
17
|
+
// GENERATOR REGISTRY
|
|
18
|
+
// =============================================================================
|
|
19
|
+
/**
|
|
20
|
+
* Registry of supported generator models with their metadata and capabilities
|
|
21
|
+
*/
|
|
22
|
+
export const SUPPORTED_GENERATORS = {
|
|
23
|
+
// SmolLM2-135M-Instruct - Balanced instruction-tuned model (RECOMMENDED DEFAULT)
|
|
24
|
+
'HuggingFaceTB/SmolLM2-135M-Instruct': {
|
|
25
|
+
name: 'HuggingFaceTB/SmolLM2-135M-Instruct',
|
|
26
|
+
type: 'instruct',
|
|
27
|
+
version: '1.0.0',
|
|
28
|
+
description: 'Balanced instruction-tuned model with good quality and speed (uses top 3 chunks)',
|
|
29
|
+
capabilities: {
|
|
30
|
+
supportsStreaming: true,
|
|
31
|
+
supportsSystemPrompt: true, // Instruct models support system prompts
|
|
32
|
+
instructionTuned: true,
|
|
33
|
+
maxContextLength: 2048,
|
|
34
|
+
defaultMaxOutputTokens: 512,
|
|
35
|
+
recommendedTemperature: 0.1,
|
|
36
|
+
defaultMaxChunksForContext: 3 // Use top 3 reranked chunks for context
|
|
37
|
+
},
|
|
38
|
+
requirements: {
|
|
39
|
+
transformersJsVersion: '>=3.0.0',
|
|
40
|
+
minimumMemory: 768,
|
|
41
|
+
requiredFeatures: ['text-generation'],
|
|
42
|
+
platformSupport: ['node', 'browser']
|
|
43
|
+
},
|
|
44
|
+
isDefault: true // Recommended default model
|
|
45
|
+
},
|
|
46
|
+
// SmolLM2-360M-Instruct - Higher quality instruction-tuned model
|
|
47
|
+
'HuggingFaceTB/SmolLM2-360M-Instruct': {
|
|
48
|
+
name: 'HuggingFaceTB/SmolLM2-360M-Instruct',
|
|
49
|
+
type: 'instruct',
|
|
50
|
+
version: '1.0.0',
|
|
51
|
+
description: 'Higher quality instruction-tuned model, slower but more accurate (uses top 5 chunks)',
|
|
52
|
+
capabilities: {
|
|
53
|
+
supportsStreaming: true,
|
|
54
|
+
supportsSystemPrompt: true,
|
|
55
|
+
instructionTuned: true,
|
|
56
|
+
maxContextLength: 2048,
|
|
57
|
+
defaultMaxOutputTokens: 512,
|
|
58
|
+
recommendedTemperature: 0.1,
|
|
59
|
+
defaultMaxChunksForContext: 5 // Use top 5 reranked chunks for context
|
|
60
|
+
},
|
|
61
|
+
requirements: {
|
|
62
|
+
transformersJsVersion: '>=3.0.0',
|
|
63
|
+
minimumMemory: 1024,
|
|
64
|
+
requiredFeatures: ['text-generation'],
|
|
65
|
+
platformSupport: ['node', 'browser']
|
|
66
|
+
},
|
|
67
|
+
isDefault: false
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
// =============================================================================
|
|
71
|
+
// DEFAULT MODEL
|
|
72
|
+
// =============================================================================
|
|
73
|
+
/** Default generator model name */
|
|
74
|
+
export const DEFAULT_GENERATOR_MODEL = 'HuggingFaceTB/SmolLM2-135M-Instruct';
|
|
75
|
+
// =============================================================================
|
|
76
|
+
// GENERATOR REGISTRY CLASS
|
|
77
|
+
// =============================================================================
|
|
78
|
+
/**
|
|
79
|
+
* Generator registry class providing validation and model information services
|
|
80
|
+
*/
|
|
81
|
+
export class GeneratorRegistry {
|
|
82
|
+
/**
|
|
83
|
+
* Gets generator model information for a given model name
|
|
84
|
+
* @param modelName - Name of the generator model
|
|
85
|
+
* @returns Generator model information or null if not supported
|
|
86
|
+
*/
|
|
87
|
+
static getGeneratorInfo(modelName) {
|
|
88
|
+
return SUPPORTED_GENERATORS[modelName] || null;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Validates a generator model name and returns compatibility information
|
|
92
|
+
* @param modelName - Name of the model to validate
|
|
93
|
+
* @returns Validation result with errors, warnings, and suggestions
|
|
94
|
+
*/
|
|
95
|
+
static validateGenerator(modelName) {
|
|
96
|
+
const modelInfo = this.getGeneratorInfo(modelName);
|
|
97
|
+
if (!modelInfo) {
|
|
98
|
+
const suggestions = this.getSimilarGenerators(modelName);
|
|
99
|
+
return {
|
|
100
|
+
isValid: false,
|
|
101
|
+
errors: [`Generator model '${modelName}' is not supported`],
|
|
102
|
+
warnings: [],
|
|
103
|
+
suggestions: suggestions.length > 0
|
|
104
|
+
? [`Did you mean: ${suggestions.join(', ')}?`]
|
|
105
|
+
: [`Available generators: ${this.getSupportedGenerators().join(', ')}`]
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
const warnings = [];
|
|
109
|
+
const suggestions = [];
|
|
110
|
+
// Memory warnings
|
|
111
|
+
if (modelInfo.requirements.minimumMemory > 768) {
|
|
112
|
+
warnings.push(`Model requires ${modelInfo.requirements.minimumMemory}MB memory`);
|
|
113
|
+
}
|
|
114
|
+
return {
|
|
115
|
+
isValid: true,
|
|
116
|
+
errors: [],
|
|
117
|
+
warnings,
|
|
118
|
+
suggestions
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Lists all supported generator models
|
|
123
|
+
* @param modelType - Optional filter by model type
|
|
124
|
+
* @returns Array of supported generator model names
|
|
125
|
+
*/
|
|
126
|
+
static getSupportedGenerators(modelType) {
|
|
127
|
+
const allModels = Object.keys(SUPPORTED_GENERATORS);
|
|
128
|
+
if (!modelType) {
|
|
129
|
+
return allModels;
|
|
130
|
+
}
|
|
131
|
+
return allModels.filter(modelName => SUPPORTED_GENERATORS[modelName].type === modelType);
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Gets the default generator model name
|
|
135
|
+
* @returns Default generator model name
|
|
136
|
+
*/
|
|
137
|
+
static getDefaultGenerator() {
|
|
138
|
+
return DEFAULT_GENERATOR_MODEL;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Gets generators by type
|
|
142
|
+
* @param type - Generator type ('causal-lm' or 'instruct')
|
|
143
|
+
* @returns Array of model names matching the type
|
|
144
|
+
*/
|
|
145
|
+
static getGeneratorsByType(type) {
|
|
146
|
+
return Object.keys(SUPPORTED_GENERATORS).filter(modelName => SUPPORTED_GENERATORS[modelName].type === type);
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Checks if a generator model supports a specific capability
|
|
150
|
+
* @param modelName - Name of the model
|
|
151
|
+
* @param capability - Capability to check
|
|
152
|
+
* @returns True if the model supports the capability
|
|
153
|
+
*/
|
|
154
|
+
static supportsCapability(modelName, capability) {
|
|
155
|
+
const modelInfo = this.getGeneratorInfo(modelName);
|
|
156
|
+
if (!modelInfo)
|
|
157
|
+
return false;
|
|
158
|
+
const value = modelInfo.capabilities[capability];
|
|
159
|
+
return typeof value === 'boolean' ? value : value !== undefined;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Gets generators similar to the given model name (for suggestions)
|
|
163
|
+
* @private
|
|
164
|
+
*/
|
|
165
|
+
static getSimilarGenerators(modelName) {
|
|
166
|
+
const allModels = Object.keys(SUPPORTED_GENERATORS);
|
|
167
|
+
const lowerModelName = modelName.toLowerCase();
|
|
168
|
+
// Simple similarity check based on common substrings
|
|
169
|
+
const keywords = ['gpt', 'smol', 'lm', 'instruct', 'distil'];
|
|
170
|
+
const modelKeywords = keywords.filter(keyword => lowerModelName.includes(keyword));
|
|
171
|
+
return allModels.filter(supportedModel => {
|
|
172
|
+
const lowerSupported = supportedModel.toLowerCase();
|
|
173
|
+
return modelKeywords.some(keyword => lowerSupported.includes(keyword));
|
|
174
|
+
}).slice(0, 3);
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Validates system compatibility for a generator model
|
|
178
|
+
* @param modelName - Name of the model
|
|
179
|
+
* @param systemCapabilities - System capabilities to check against
|
|
180
|
+
* @returns Validation result with compatibility information
|
|
181
|
+
*/
|
|
182
|
+
static validateSystemCompatibility(modelName, systemCapabilities) {
|
|
183
|
+
const modelInfo = this.getGeneratorInfo(modelName);
|
|
184
|
+
if (!modelInfo) {
|
|
185
|
+
return {
|
|
186
|
+
isValid: false,
|
|
187
|
+
errors: [`Generator model '${modelName}' is not supported`],
|
|
188
|
+
warnings: [],
|
|
189
|
+
suggestions: []
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
const errors = [];
|
|
193
|
+
const warnings = [];
|
|
194
|
+
const suggestions = [];
|
|
195
|
+
// Check memory requirements
|
|
196
|
+
if (systemCapabilities.availableMemory !== undefined) {
|
|
197
|
+
if (systemCapabilities.availableMemory < modelInfo.requirements.minimumMemory) {
|
|
198
|
+
errors.push(`Insufficient memory: ${systemCapabilities.availableMemory}MB available, ` +
|
|
199
|
+
`${modelInfo.requirements.minimumMemory}MB required`);
|
|
200
|
+
// Suggest lighter models
|
|
201
|
+
const lighterModels = this.getSupportedGenerators().filter(name => {
|
|
202
|
+
const info = this.getGeneratorInfo(name);
|
|
203
|
+
return info &&
|
|
204
|
+
info.requirements.minimumMemory <= systemCapabilities.availableMemory;
|
|
205
|
+
});
|
|
206
|
+
if (lighterModels.length > 0) {
|
|
207
|
+
suggestions.push(`Consider lighter models: ${lighterModels.join(', ')}`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
// Check platform compatibility
|
|
212
|
+
if (systemCapabilities.platform) {
|
|
213
|
+
if (!modelInfo.requirements.platformSupport.includes(systemCapabilities.platform)) {
|
|
214
|
+
errors.push(`Platform '${systemCapabilities.platform}' not supported. ` +
|
|
215
|
+
`Supported platforms: ${modelInfo.requirements.platformSupport.join(', ')}`);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return {
|
|
219
|
+
isValid: errors.length === 0,
|
|
220
|
+
errors,
|
|
221
|
+
warnings,
|
|
222
|
+
suggestions
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
// =============================================================================
|
|
227
|
+
// UTILITY FUNCTIONS
|
|
228
|
+
// =============================================================================
|
|
229
|
+
/**
|
|
230
|
+
* Gets the generator type for a given model name
|
|
231
|
+
* @param modelName - Name of the model
|
|
232
|
+
* @returns Generator type or null if model not supported
|
|
233
|
+
*/
|
|
234
|
+
export function getGeneratorType(modelName) {
|
|
235
|
+
const modelInfo = GeneratorRegistry.getGeneratorInfo(modelName);
|
|
236
|
+
return modelInfo ? modelInfo.type : null;
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Checks if a model is an instruction-tuned model
|
|
240
|
+
* @param modelName - Name of the model
|
|
241
|
+
* @returns True if the model is instruction-tuned
|
|
242
|
+
*/
|
|
243
|
+
export function isInstructionTunedModel(modelName) {
|
|
244
|
+
const modelInfo = GeneratorRegistry.getGeneratorInfo(modelName);
|
|
245
|
+
return modelInfo ? modelInfo.capabilities.instructionTuned : false;
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Gets the maximum context length for a generator model
|
|
249
|
+
* @param modelName - Name of the model
|
|
250
|
+
* @returns Maximum context length or null if model not supported
|
|
251
|
+
*/
|
|
252
|
+
export function getMaxContextLength(modelName) {
|
|
253
|
+
const modelInfo = GeneratorRegistry.getGeneratorInfo(modelName);
|
|
254
|
+
return modelInfo ? modelInfo.capabilities.maxContextLength : null;
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Gets recommended generation settings for a model
|
|
258
|
+
* @param modelName - Name of the model
|
|
259
|
+
* @returns Recommended settings or null if model not supported
|
|
260
|
+
*/
|
|
261
|
+
export function getRecommendedSettings(modelName) {
|
|
262
|
+
const modelInfo = GeneratorRegistry.getGeneratorInfo(modelName);
|
|
263
|
+
if (!modelInfo)
|
|
264
|
+
return null;
|
|
265
|
+
return {
|
|
266
|
+
temperature: modelInfo.capabilities.recommendedTemperature,
|
|
267
|
+
maxTokens: modelInfo.capabilities.defaultMaxOutputTokens,
|
|
268
|
+
maxChunksForContext: modelInfo.capabilities.defaultMaxChunksForContext
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Gets the default maximum chunks for context for a generator model
|
|
273
|
+
* @param modelName - Name of the model
|
|
274
|
+
* @returns Default max chunks for context or null if model not supported
|
|
275
|
+
*/
|
|
276
|
+
export function getDefaultMaxChunksForContext(modelName) {
|
|
277
|
+
const modelInfo = GeneratorRegistry.getGeneratorInfo(modelName);
|
|
278
|
+
return modelInfo ? modelInfo.capabilities.defaultMaxChunksForContext : null;
|
|
279
|
+
}
|
|
280
|
+
//# sourceMappingURL=generator-registry.js.map
|
package/dist/cjs/core/index.d.ts
CHANGED
|
@@ -56,4 +56,8 @@ export * from './path-manager.js';
|
|
|
56
56
|
export { ContentManager, type MemoryContentMetadata, type ContentIngestionResult, type ContentManagerConfig } from './content-manager.js';
|
|
57
57
|
export { ContentResolver, type ContentRequest, type ContentResult } from './content-resolver.js';
|
|
58
58
|
export * from './error-handler.js';
|
|
59
|
+
export { type ResponseGenerator, type GeneratorModelType, type GenerationRequest, type GenerationResult, type GeneratorCapabilities, type GeneratorRequirements, type GeneratorModelInfo, type GeneratorValidationResult, type GeneratorCreationOptions, type GenerateFunction, type CreateGeneratorFunction, GeneratorValidationError, GenerationError, ContextWindowError, supportsStreaming, isInstructModel, createGenerateFunction } from './response-generator.js';
|
|
60
|
+
export { SUPPORTED_GENERATORS, DEFAULT_GENERATOR_MODEL, GeneratorRegistry, getGeneratorType, isInstructionTunedModel, getMaxContextLength, getRecommendedSettings, getDefaultMaxChunksForContext } from './generator-registry.js';
|
|
61
|
+
export { DEFAULT_SYSTEM_PROMPT, DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION, SMOLLM2_CHAT_TEMPLATE, formatContextChunks, buildPrompt, estimateTokenCount, calculateContextBudget, getDefaultStopSequences, type ContextFormattingOptions, type FormattedContext, type PromptBuildOptions, type BuiltPrompt } from './prompt-templates.js';
|
|
62
|
+
export { BaseResponseGenerator, createGeneratorOptions, type GeneratorOptions } from './abstract-generator.js';
|
|
59
63
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/cjs/core/index.js
CHANGED
|
@@ -66,4 +66,15 @@ export { ContentManager } from './content-manager.js';
|
|
|
66
66
|
export { ContentResolver } from './content-resolver.js';
|
|
67
67
|
// Error handling framework - supports implementation-specific error contexts
|
|
68
68
|
export * from './error-handler.js';
|
|
69
|
+
// =============================================================================
|
|
70
|
+
// EXPERIMENTAL: Response Generation
|
|
71
|
+
// =============================================================================
|
|
72
|
+
// Response generation types and interfaces (experimental)
|
|
73
|
+
export { GeneratorValidationError, GenerationError, ContextWindowError, supportsStreaming, isInstructModel, createGenerateFunction } from './response-generator.js';
|
|
74
|
+
// Generator registry (experimental)
|
|
75
|
+
export { SUPPORTED_GENERATORS, DEFAULT_GENERATOR_MODEL, GeneratorRegistry, getGeneratorType, isInstructionTunedModel, getMaxContextLength, getRecommendedSettings, getDefaultMaxChunksForContext } from './generator-registry.js';
|
|
76
|
+
// Prompt templates for generation (experimental)
|
|
77
|
+
export { DEFAULT_SYSTEM_PROMPT, DEFAULT_SYSTEM_PROMPT_WITH_ATTRIBUTION, SMOLLM2_CHAT_TEMPLATE, formatContextChunks, buildPrompt, estimateTokenCount, calculateContextBudget, getDefaultStopSequences } from './prompt-templates.js';
|
|
78
|
+
// Abstract base generator (experimental)
|
|
79
|
+
export { BaseResponseGenerator, createGeneratorOptions } from './abstract-generator.js';
|
|
69
80
|
//# sourceMappingURL=index.js.map
|
|
@@ -340,6 +340,9 @@ export class IngestionPipeline {
|
|
|
340
340
|
// Phase 5: Vector Index Updates
|
|
341
341
|
console.log('\n--- Phase 5: Vector Index Updates ---');
|
|
342
342
|
await this.updateVectorIndex(embeddingResult.embeddings);
|
|
343
|
+
// Final save to ensure all vectors are persisted
|
|
344
|
+
console.log('Performing final index save...');
|
|
345
|
+
await this.indexManager.saveIndex();
|
|
343
346
|
const endTime = Date.now();
|
|
344
347
|
const processingTimeMs = endTime - startTime;
|
|
345
348
|
const result = {
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Knowledge Base Manager
|
|
3
|
+
*
|
|
4
|
+
* Provides a unified API for managing the knowledge base (database + vector index).
|
|
5
|
+
* This module is designed to solve file locking issues on Windows by using
|
|
6
|
+
* in-place reset operations instead of file deletion.
|
|
7
|
+
*
|
|
8
|
+
* Key Features:
|
|
9
|
+
* - Reset database and index without file deletion (avoids EBUSY/EACCES errors)
|
|
10
|
+
* - Coordinated reset of both database and index in a single operation
|
|
11
|
+
* - Connection management to prevent orphaned handles
|
|
12
|
+
* - Cross-platform compatibility (especially Windows)
|
|
13
|
+
*
|
|
14
|
+
* @module knowledge-base-manager
|
|
15
|
+
*/
|
|
16
|
+
import { type DatabaseResetResult } from './db.js';
|
|
17
|
+
/**
|
|
18
|
+
* Result of a knowledge base reset operation
|
|
19
|
+
*/
|
|
20
|
+
export interface KnowledgeBaseResetResult {
|
|
21
|
+
/** Whether the overall reset was successful */
|
|
22
|
+
success: boolean;
|
|
23
|
+
/** Database reset result */
|
|
24
|
+
database: DatabaseResetResult;
|
|
25
|
+
/** Index reset statistics */
|
|
26
|
+
index: {
|
|
27
|
+
/** Number of vectors cleared */
|
|
28
|
+
vectorsCleared: number;
|
|
29
|
+
/** Time taken for index reset in milliseconds */
|
|
30
|
+
resetTimeMs: number;
|
|
31
|
+
};
|
|
32
|
+
/** Total time for the complete reset operation */
|
|
33
|
+
totalTimeMs: number;
|
|
34
|
+
/** Any warnings that occurred during reset */
|
|
35
|
+
warnings: string[];
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Options for knowledge base reset operation
|
|
39
|
+
*/
|
|
40
|
+
export interface KnowledgeBaseResetOptions {
|
|
41
|
+
/** Whether to preserve system_info (mode, model configuration) - default: false */
|
|
42
|
+
preserveSystemInfo?: boolean;
|
|
43
|
+
/** Whether to run VACUUM after database reset - default: true */
|
|
44
|
+
runVacuum?: boolean;
|
|
45
|
+
/** Model name to use for index recreation - default: from config */
|
|
46
|
+
modelName?: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Knowledge Base Manager
|
|
50
|
+
*
|
|
51
|
+
* Manages the complete knowledge base lifecycle including database and vector index.
|
|
52
|
+
* Provides safe reset operations that avoid file locking issues on Windows.
|
|
53
|
+
*
|
|
54
|
+
* @example
|
|
55
|
+
* ```typescript
|
|
56
|
+
* // Reset knowledge base for force rebuild
|
|
57
|
+
* const result = await KnowledgeBaseManager.reset('./db.sqlite', './index.bin');
|
|
58
|
+
* console.log(`Reset ${result.database.documentsDeleted} documents and ${result.index.vectorsCleared} vectors`);
|
|
59
|
+
*
|
|
60
|
+
* // Reset with options
|
|
61
|
+
* const result = await KnowledgeBaseManager.reset('./db.sqlite', './index.bin', {
|
|
62
|
+
* preserveSystemInfo: true, // Keep mode/model configuration
|
|
63
|
+
* modelName: 'all-MiniLM-L6-v2' // Specify model for index
|
|
64
|
+
* });
|
|
65
|
+
* ```
|
|
66
|
+
*/
|
|
67
|
+
export declare class KnowledgeBaseManager {
|
|
68
|
+
/**
|
|
69
|
+
* Reset the knowledge base by clearing all data while keeping files intact.
|
|
70
|
+
* This is a safer alternative to file deletion that avoids file locking issues on Windows.
|
|
71
|
+
*
|
|
72
|
+
* The reset operation:
|
|
73
|
+
* 1. Closes any existing connections via DatabaseConnectionManager
|
|
74
|
+
* 2. Opens a fresh connection to the database
|
|
75
|
+
* 3. Deletes all rows from documents, chunks, content_metadata tables
|
|
76
|
+
* 4. Optionally runs VACUUM to reclaim disk space
|
|
77
|
+
* 5. Reinitializes the vector index (clears all vectors)
|
|
78
|
+
* 6. Saves the empty index to disk (overwrites existing file content)
|
|
79
|
+
*
|
|
80
|
+
* This approach works because:
|
|
81
|
+
* - We don't delete files, so no EBUSY/EACCES errors
|
|
82
|
+
* - The same file handles can be reused or replaced safely
|
|
83
|
+
* - SQLite transactions ensure data integrity
|
|
84
|
+
* - Index overwrite uses standard file write operations
|
|
85
|
+
*
|
|
86
|
+
* @param dbPath - Path to the SQLite database file
|
|
87
|
+
* @param indexPath - Path to the vector index file
|
|
88
|
+
* @param options - Reset options
|
|
89
|
+
* @returns Promise resolving to reset result statistics
|
|
90
|
+
*
|
|
91
|
+
* @throws Error if database or index reset fails
|
|
92
|
+
*/
|
|
93
|
+
static reset(dbPath: string, indexPath: string, options?: KnowledgeBaseResetOptions): Promise<KnowledgeBaseResetResult>;
|
|
94
|
+
/**
|
|
95
|
+
* Check if the knowledge base has any data
|
|
96
|
+
*
|
|
97
|
+
* @param dbPath - Path to the SQLite database file
|
|
98
|
+
* @returns Promise resolving to true if database has data, false if empty
|
|
99
|
+
*/
|
|
100
|
+
static hasData(dbPath: string): Promise<boolean>;
|
|
101
|
+
/**
|
|
102
|
+
* Close all connections to the knowledge base
|
|
103
|
+
* Useful before operations that might conflict with open handles
|
|
104
|
+
*
|
|
105
|
+
* @param dbPath - Path to the SQLite database file
|
|
106
|
+
*/
|
|
107
|
+
static closeAllConnections(dbPath: string): Promise<void>;
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=knowledge-base-manager.d.ts.map
|