rag-lite-ts 1.0.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +606 -93
- package/dist/cli/indexer.js +192 -4
- package/dist/cli/search.js +50 -11
- package/dist/cli.js +183 -26
- package/dist/core/abstract-embedder.d.ts +125 -0
- package/dist/core/abstract-embedder.js +264 -0
- package/dist/core/actionable-error-messages.d.ts +60 -0
- package/dist/core/actionable-error-messages.js +397 -0
- package/dist/core/batch-processing-optimizer.d.ts +155 -0
- package/dist/core/batch-processing-optimizer.js +541 -0
- package/dist/core/chunker.d.ts +2 -0
- package/dist/core/cli-database-utils.d.ts +53 -0
- package/dist/core/cli-database-utils.js +239 -0
- package/dist/core/config.js +10 -3
- package/dist/core/content-errors.d.ts +111 -0
- package/dist/core/content-errors.js +362 -0
- package/dist/core/content-manager.d.ts +343 -0
- package/dist/core/content-manager.js +1504 -0
- package/dist/core/content-performance-optimizer.d.ts +150 -0
- package/dist/core/content-performance-optimizer.js +516 -0
- package/dist/core/content-resolver.d.ts +104 -0
- package/dist/core/content-resolver.js +285 -0
- package/dist/core/cross-modal-search.d.ts +164 -0
- package/dist/core/cross-modal-search.js +342 -0
- package/dist/core/database-connection-manager.d.ts +109 -0
- package/dist/core/database-connection-manager.js +304 -0
- package/dist/core/db.d.ts +141 -2
- package/dist/core/db.js +631 -89
- package/dist/core/embedder-factory.d.ts +176 -0
- package/dist/core/embedder-factory.js +338 -0
- package/dist/core/index.d.ts +3 -1
- package/dist/core/index.js +4 -1
- package/dist/core/ingestion.d.ts +85 -15
- package/dist/core/ingestion.js +510 -45
- package/dist/core/lazy-dependency-loader.d.ts +152 -0
- package/dist/core/lazy-dependency-loader.js +453 -0
- package/dist/core/mode-detection-service.d.ts +150 -0
- package/dist/core/mode-detection-service.js +565 -0
- package/dist/core/mode-model-validator.d.ts +92 -0
- package/dist/core/mode-model-validator.js +203 -0
- package/dist/core/model-registry.d.ts +120 -0
- package/dist/core/model-registry.js +415 -0
- package/dist/core/model-validator.d.ts +217 -0
- package/dist/core/model-validator.js +782 -0
- package/dist/core/polymorphic-search-factory.d.ts +154 -0
- package/dist/core/polymorphic-search-factory.js +344 -0
- package/dist/core/raglite-paths.d.ts +121 -0
- package/dist/core/raglite-paths.js +145 -0
- package/dist/core/reranking-config.d.ts +42 -0
- package/dist/core/reranking-config.js +156 -0
- package/dist/core/reranking-factory.d.ts +92 -0
- package/dist/core/reranking-factory.js +591 -0
- package/dist/core/reranking-strategies.d.ts +325 -0
- package/dist/core/reranking-strategies.js +720 -0
- package/dist/core/resource-cleanup.d.ts +163 -0
- package/dist/core/resource-cleanup.js +371 -0
- package/dist/core/resource-manager.d.ts +212 -0
- package/dist/core/resource-manager.js +564 -0
- package/dist/core/search.d.ts +28 -1
- package/dist/core/search.js +83 -5
- package/dist/core/streaming-operations.d.ts +145 -0
- package/dist/core/streaming-operations.js +409 -0
- package/dist/core/types.d.ts +3 -0
- package/dist/core/universal-embedder.d.ts +177 -0
- package/dist/core/universal-embedder.js +139 -0
- package/dist/core/validation-messages.d.ts +99 -0
- package/dist/core/validation-messages.js +334 -0
- package/dist/core/vector-index.js +7 -8
- package/dist/factories/index.d.ts +1 -1
- package/dist/factories/text-factory.d.ts +128 -34
- package/dist/factories/text-factory.js +346 -97
- package/dist/file-processor.d.ts +88 -2
- package/dist/file-processor.js +720 -17
- package/dist/index.d.ts +9 -0
- package/dist/index.js +11 -0
- package/dist/ingestion.d.ts +16 -0
- package/dist/ingestion.js +21 -0
- package/dist/mcp-server.d.ts +35 -3
- package/dist/mcp-server.js +1107 -31
- package/dist/multimodal/clip-embedder.d.ts +314 -0
- package/dist/multimodal/clip-embedder.js +945 -0
- package/dist/multimodal/index.d.ts +6 -0
- package/dist/multimodal/index.js +6 -0
- package/dist/run-error-recovery-tests.d.ts +7 -0
- package/dist/run-error-recovery-tests.js +101 -0
- package/dist/search.d.ts +26 -0
- package/dist/search.js +54 -1
- package/dist/test-utils.d.ts +8 -26
- package/dist/text/chunker.d.ts +1 -0
- package/dist/text/embedder.js +15 -8
- package/dist/text/index.d.ts +1 -0
- package/dist/text/index.js +1 -0
- package/dist/text/reranker.d.ts +1 -2
- package/dist/text/reranker.js +17 -47
- package/dist/text/sentence-transformer-embedder.d.ts +96 -0
- package/dist/text/sentence-transformer-embedder.js +340 -0
- package/dist/types.d.ts +39 -0
- package/dist/utils/vector-math.d.ts +31 -0
- package/dist/utils/vector-math.js +70 -0
- package/package.json +15 -3
- package/dist/api-errors.d.ts.map +0 -1
- package/dist/api-errors.js.map +0 -1
- package/dist/cli/indexer.d.ts.map +0 -1
- package/dist/cli/indexer.js.map +0 -1
- package/dist/cli/search.d.ts.map +0 -1
- package/dist/cli/search.js.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js.map +0 -1
- package/dist/core/adapters.d.ts.map +0 -1
- package/dist/core/adapters.js.map +0 -1
- package/dist/core/chunker.d.ts.map +0 -1
- package/dist/core/chunker.js.map +0 -1
- package/dist/core/config.d.ts.map +0 -1
- package/dist/core/config.js.map +0 -1
- package/dist/core/db.d.ts.map +0 -1
- package/dist/core/db.js.map +0 -1
- package/dist/core/error-handler.d.ts.map +0 -1
- package/dist/core/error-handler.js.map +0 -1
- package/dist/core/index.d.ts.map +0 -1
- package/dist/core/index.js.map +0 -1
- package/dist/core/ingestion.d.ts.map +0 -1
- package/dist/core/ingestion.js.map +0 -1
- package/dist/core/interfaces.d.ts.map +0 -1
- package/dist/core/interfaces.js.map +0 -1
- package/dist/core/path-manager.d.ts.map +0 -1
- package/dist/core/path-manager.js.map +0 -1
- package/dist/core/search-example.d.ts +0 -25
- package/dist/core/search-example.d.ts.map +0 -1
- package/dist/core/search-example.js +0 -138
- package/dist/core/search-example.js.map +0 -1
- package/dist/core/search-pipeline-example.d.ts +0 -21
- package/dist/core/search-pipeline-example.d.ts.map +0 -1
- package/dist/core/search-pipeline-example.js +0 -188
- package/dist/core/search-pipeline-example.js.map +0 -1
- package/dist/core/search-pipeline.d.ts.map +0 -1
- package/dist/core/search-pipeline.js.map +0 -1
- package/dist/core/search.d.ts.map +0 -1
- package/dist/core/search.js.map +0 -1
- package/dist/core/types.d.ts.map +0 -1
- package/dist/core/types.js.map +0 -1
- package/dist/core/vector-index.d.ts.map +0 -1
- package/dist/core/vector-index.js.map +0 -1
- package/dist/dom-polyfills.d.ts.map +0 -1
- package/dist/dom-polyfills.js.map +0 -1
- package/dist/examples/clean-api-examples.d.ts +0 -44
- package/dist/examples/clean-api-examples.d.ts.map +0 -1
- package/dist/examples/clean-api-examples.js +0 -206
- package/dist/examples/clean-api-examples.js.map +0 -1
- package/dist/factories/index.d.ts.map +0 -1
- package/dist/factories/index.js.map +0 -1
- package/dist/factories/text-factory.d.ts.map +0 -1
- package/dist/factories/text-factory.js.map +0 -1
- package/dist/file-processor.d.ts.map +0 -1
- package/dist/file-processor.js.map +0 -1
- package/dist/index-manager.d.ts.map +0 -1
- package/dist/index-manager.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/indexer.d.ts.map +0 -1
- package/dist/indexer.js.map +0 -1
- package/dist/ingestion.d.ts.map +0 -1
- package/dist/ingestion.js.map +0 -1
- package/dist/mcp-server.d.ts.map +0 -1
- package/dist/mcp-server.js.map +0 -1
- package/dist/preprocess.d.ts.map +0 -1
- package/dist/preprocess.js.map +0 -1
- package/dist/preprocessors/index.d.ts.map +0 -1
- package/dist/preprocessors/index.js.map +0 -1
- package/dist/preprocessors/mdx.d.ts.map +0 -1
- package/dist/preprocessors/mdx.js.map +0 -1
- package/dist/preprocessors/mermaid.d.ts.map +0 -1
- package/dist/preprocessors/mermaid.js.map +0 -1
- package/dist/preprocessors/registry.d.ts.map +0 -1
- package/dist/preprocessors/registry.js.map +0 -1
- package/dist/search-standalone.d.ts.map +0 -1
- package/dist/search-standalone.js.map +0 -1
- package/dist/search.d.ts.map +0 -1
- package/dist/search.js.map +0 -1
- package/dist/test-utils.d.ts.map +0 -1
- package/dist/test-utils.js.map +0 -1
- package/dist/text/chunker.d.ts.map +0 -1
- package/dist/text/chunker.js.map +0 -1
- package/dist/text/embedder.d.ts.map +0 -1
- package/dist/text/embedder.js.map +0 -1
- package/dist/text/index.d.ts.map +0 -1
- package/dist/text/index.js.map +0 -1
- package/dist/text/preprocessors/index.d.ts.map +0 -1
- package/dist/text/preprocessors/index.js.map +0 -1
- package/dist/text/preprocessors/mdx.d.ts.map +0 -1
- package/dist/text/preprocessors/mdx.js.map +0 -1
- package/dist/text/preprocessors/mermaid.d.ts.map +0 -1
- package/dist/text/preprocessors/mermaid.js.map +0 -1
- package/dist/text/preprocessors/registry.d.ts.map +0 -1
- package/dist/text/preprocessors/registry.js.map +0 -1
- package/dist/text/reranker.d.ts.map +0 -1
- package/dist/text/reranker.js.map +0 -1
- package/dist/text/tokenizer.d.ts.map +0 -1
- package/dist/text/tokenizer.js.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js.map +0 -1
package/dist/file-processor.d.ts
CHANGED
|
@@ -13,6 +13,10 @@ export interface FileProcessorOptions {
|
|
|
13
13
|
* Default options for file processing
|
|
14
14
|
*/
|
|
15
15
|
export declare const DEFAULT_FILE_PROCESSOR_OPTIONS: FileProcessorOptions;
|
|
16
|
+
/**
|
|
17
|
+
* Default options for image-to-text processing
|
|
18
|
+
*/
|
|
19
|
+
export declare const DEFAULT_IMAGE_TO_TEXT_OPTIONS: ImageToTextOptions;
|
|
16
20
|
/**
|
|
17
21
|
* Result of file discovery operation
|
|
18
22
|
*/
|
|
@@ -37,6 +41,54 @@ export interface DocumentProcessingResult {
|
|
|
37
41
|
error: string;
|
|
38
42
|
}>;
|
|
39
43
|
}
|
|
44
|
+
/**
|
|
45
|
+
* Image description generation result
|
|
46
|
+
*/
|
|
47
|
+
export interface ImageDescriptionResult {
|
|
48
|
+
/** Generated text description */
|
|
49
|
+
description: string;
|
|
50
|
+
/** Confidence score (0-1) */
|
|
51
|
+
confidence?: number;
|
|
52
|
+
/** Model used for generation */
|
|
53
|
+
model: string;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Options for image-to-text processing
|
|
57
|
+
*/
|
|
58
|
+
export interface ImageToTextOptions {
|
|
59
|
+
/** Model to use for image captioning */
|
|
60
|
+
model?: string;
|
|
61
|
+
/** Maximum description length */
|
|
62
|
+
maxLength?: number;
|
|
63
|
+
/** Batch size for processing multiple images */
|
|
64
|
+
batchSize?: number;
|
|
65
|
+
/** Whether to include confidence scores */
|
|
66
|
+
includeConfidence?: boolean;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Image metadata extracted from image files
|
|
70
|
+
*/
|
|
71
|
+
export interface ImageMetadata {
|
|
72
|
+
/** Original file path */
|
|
73
|
+
originalPath: string;
|
|
74
|
+
/** Image dimensions */
|
|
75
|
+
dimensions: {
|
|
76
|
+
width: number;
|
|
77
|
+
height: number;
|
|
78
|
+
};
|
|
79
|
+
/** File size in bytes */
|
|
80
|
+
fileSize: number;
|
|
81
|
+
/** Image format (jpeg, png, gif, webp, bmp) */
|
|
82
|
+
format: string;
|
|
83
|
+
/** File creation date */
|
|
84
|
+
createdAt?: Date;
|
|
85
|
+
/** Generated text description (if available) */
|
|
86
|
+
description?: string;
|
|
87
|
+
/** Model used for description generation */
|
|
88
|
+
descriptionModel?: string;
|
|
89
|
+
/** Confidence score for description */
|
|
90
|
+
descriptionConfidence?: number;
|
|
91
|
+
}
|
|
40
92
|
/**
|
|
41
93
|
* Discover files for ingestion
|
|
42
94
|
* Supports both single files and directories (with optional recursion)
|
|
@@ -46,14 +98,48 @@ export declare function discoverFiles(path: string, options?: FileProcessorOptio
|
|
|
46
98
|
* Process multiple files into Documents
|
|
47
99
|
* Handles errors gracefully by skipping problematic files
|
|
48
100
|
*/
|
|
49
|
-
export declare function processFiles(filePaths: string[], pathManager: DocumentPathManager): Promise<DocumentProcessingResult>;
|
|
101
|
+
export declare function processFiles(filePaths: string[], pathManager: DocumentPathManager, imageToTextOptions?: ImageToTextOptions): Promise<DocumentProcessingResult>;
|
|
50
102
|
/**
|
|
51
103
|
* Complete file discovery and processing pipeline
|
|
52
104
|
* Discovers files and processes them into Documents
|
|
53
105
|
*/
|
|
54
|
-
export declare function discoverAndProcessFiles(path: string, options?: FileProcessorOptions, pathManager?: DocumentPathManager): Promise<{
|
|
106
|
+
export declare function discoverAndProcessFiles(path: string, options?: FileProcessorOptions, pathManager?: DocumentPathManager, imageToTextOptions?: ImageToTextOptions): Promise<{
|
|
55
107
|
documents: Document[];
|
|
56
108
|
discoveryResult: FileDiscoveryResult;
|
|
57
109
|
processingResult: DocumentProcessingResult;
|
|
58
110
|
}>;
|
|
111
|
+
/**
|
|
112
|
+
* Clean up image processing resources
|
|
113
|
+
* Call this when shutting down the application to free memory
|
|
114
|
+
*/
|
|
115
|
+
export declare function cleanupImageProcessingResources(): Promise<void>;
|
|
116
|
+
/**
|
|
117
|
+
* Clean up image-to-text pipeline resources (legacy function for backward compatibility)
|
|
118
|
+
* @deprecated Use cleanupImageProcessingResources() instead
|
|
119
|
+
*/
|
|
120
|
+
export declare function cleanupImageToTextPipeline(): Promise<void>;
|
|
121
|
+
/**
|
|
122
|
+
* Generate description for a single image (exported for external use)
|
|
123
|
+
*/
|
|
124
|
+
export declare function generateImageDescriptionForFile(imagePath: string, options?: ImageToTextOptions): Promise<ImageDescriptionResult>;
|
|
125
|
+
/**
|
|
126
|
+
* Generate descriptions for multiple images (exported for external use)
|
|
127
|
+
*/
|
|
128
|
+
export declare function generateImageDescriptionsForFiles(imagePaths: string[], options?: ImageToTextOptions): Promise<Array<{
|
|
129
|
+
path: string;
|
|
130
|
+
result?: ImageDescriptionResult;
|
|
131
|
+
error?: string;
|
|
132
|
+
}>>;
|
|
133
|
+
/**
|
|
134
|
+
* Extract metadata from a single image file (exported for external use)
|
|
135
|
+
*/
|
|
136
|
+
export declare function extractImageMetadataForFile(imagePath: string): Promise<ImageMetadata>;
|
|
137
|
+
/**
|
|
138
|
+
* Extract metadata from multiple image files (exported for external use)
|
|
139
|
+
*/
|
|
140
|
+
export declare function extractImageMetadataForFiles(imagePaths: string[]): Promise<Array<{
|
|
141
|
+
path: string;
|
|
142
|
+
metadata?: ImageMetadata;
|
|
143
|
+
error?: string;
|
|
144
|
+
}>>;
|
|
59
145
|
//# sourceMappingURL=file-processor.d.ts.map
|