rag-lite-ts 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{cli → cjs/cli}/indexer.js +1 -1
- package/dist/{cli → cjs/cli}/search.js +5 -10
- package/dist/{core → cjs/core}/binary-index-format.d.ts +28 -2
- package/dist/cjs/core/binary-index-format.js +291 -0
- package/dist/{core → cjs/core}/ingestion.d.ts +5 -1
- package/dist/{core → cjs/core}/ingestion.js +76 -9
- package/dist/{core → cjs/core}/model-validator.js +1 -1
- package/dist/{core → cjs/core}/reranking-strategies.js +4 -5
- package/dist/{core → cjs/core}/search.js +2 -1
- package/dist/{core → cjs/core}/types.d.ts +1 -1
- package/dist/{core → cjs/core}/vector-index.d.ts +4 -0
- package/dist/{core → cjs/core}/vector-index.js +10 -2
- package/dist/{file-processor.d.ts → cjs/file-processor.d.ts} +2 -0
- package/dist/{file-processor.js → cjs/file-processor.js} +20 -0
- package/dist/{index-manager.d.ts → cjs/index-manager.d.ts} +17 -1
- package/dist/{index-manager.js → cjs/index-manager.js} +148 -7
- package/dist/{multimodal → cjs/multimodal}/clip-embedder.js +71 -66
- package/dist/esm/api-errors.d.ts +90 -0
- package/dist/esm/api-errors.js +320 -0
- package/dist/esm/cli/indexer.d.ts +11 -0
- package/dist/esm/cli/indexer.js +471 -0
- package/dist/esm/cli/search.d.ts +7 -0
- package/dist/esm/cli/search.js +332 -0
- package/dist/esm/cli.d.ts +3 -0
- package/dist/esm/cli.js +529 -0
- package/dist/esm/config.d.ts +51 -0
- package/dist/esm/config.js +79 -0
- package/dist/esm/core/abstract-embedder.d.ts +125 -0
- package/dist/esm/core/abstract-embedder.js +264 -0
- package/dist/esm/core/actionable-error-messages.d.ts +60 -0
- package/dist/esm/core/actionable-error-messages.js +397 -0
- package/dist/esm/core/adapters.d.ts +93 -0
- package/dist/esm/core/adapters.js +139 -0
- package/dist/esm/core/batch-processing-optimizer.d.ts +155 -0
- package/dist/esm/core/batch-processing-optimizer.js +536 -0
- package/dist/esm/core/binary-index-format.d.ts +78 -0
- package/dist/esm/core/binary-index-format.js +291 -0
- package/dist/esm/core/chunker.d.ts +119 -0
- package/dist/esm/core/chunker.js +73 -0
- package/dist/esm/core/cli-database-utils.d.ts +53 -0
- package/dist/esm/core/cli-database-utils.js +239 -0
- package/dist/esm/core/config.d.ts +102 -0
- package/dist/esm/core/config.js +247 -0
- package/dist/esm/core/content-errors.d.ts +111 -0
- package/dist/esm/core/content-errors.js +362 -0
- package/dist/esm/core/content-manager.d.ts +335 -0
- package/dist/esm/core/content-manager.js +1476 -0
- package/dist/esm/core/content-performance-optimizer.d.ts +150 -0
- package/dist/esm/core/content-performance-optimizer.js +516 -0
- package/dist/esm/core/content-resolver.d.ts +104 -0
- package/dist/esm/core/content-resolver.js +285 -0
- package/dist/esm/core/cross-modal-search.d.ts +164 -0
- package/dist/esm/core/cross-modal-search.js +342 -0
- package/dist/esm/core/database-connection-manager.d.ts +109 -0
- package/dist/esm/core/database-connection-manager.js +310 -0
- package/dist/esm/core/db.d.ts +213 -0
- package/dist/esm/core/db.js +895 -0
- package/dist/esm/core/embedder-factory.d.ts +154 -0
- package/dist/esm/core/embedder-factory.js +311 -0
- package/dist/esm/core/error-handler.d.ts +112 -0
- package/dist/esm/core/error-handler.js +239 -0
- package/dist/esm/core/index.d.ts +59 -0
- package/dist/esm/core/index.js +69 -0
- package/dist/esm/core/ingestion.d.ts +202 -0
- package/dist/esm/core/ingestion.js +901 -0
- package/dist/esm/core/interfaces.d.ts +408 -0
- package/dist/esm/core/interfaces.js +106 -0
- package/dist/esm/core/lazy-dependency-loader.d.ts +147 -0
- package/dist/esm/core/lazy-dependency-loader.js +435 -0
- package/dist/esm/core/mode-detection-service.d.ts +150 -0
- package/dist/esm/core/mode-detection-service.js +565 -0
- package/dist/esm/core/mode-model-validator.d.ts +92 -0
- package/dist/esm/core/mode-model-validator.js +203 -0
- package/dist/esm/core/model-registry.d.ts +116 -0
- package/dist/esm/core/model-registry.js +411 -0
- package/dist/esm/core/model-validator.d.ts +217 -0
- package/dist/esm/core/model-validator.js +782 -0
- package/dist/esm/core/path-manager.d.ts +47 -0
- package/dist/esm/core/path-manager.js +71 -0
- package/dist/esm/core/raglite-paths.d.ts +121 -0
- package/dist/esm/core/raglite-paths.js +145 -0
- package/dist/esm/core/reranking-config.d.ts +42 -0
- package/dist/esm/core/reranking-config.js +147 -0
- package/dist/esm/core/reranking-factory.d.ts +92 -0
- package/dist/esm/core/reranking-factory.js +410 -0
- package/dist/esm/core/reranking-strategies.d.ts +310 -0
- package/dist/esm/core/reranking-strategies.js +650 -0
- package/dist/esm/core/resource-cleanup.d.ts +163 -0
- package/dist/esm/core/resource-cleanup.js +371 -0
- package/dist/esm/core/resource-manager.d.ts +212 -0
- package/dist/esm/core/resource-manager.js +564 -0
- package/dist/esm/core/search-pipeline.d.ts +111 -0
- package/dist/esm/core/search-pipeline.js +287 -0
- package/dist/esm/core/search.d.ts +141 -0
- package/dist/esm/core/search.js +320 -0
- package/dist/esm/core/streaming-operations.d.ts +145 -0
- package/dist/esm/core/streaming-operations.js +409 -0
- package/dist/esm/core/types.d.ts +66 -0
- package/dist/esm/core/types.js +6 -0
- package/dist/esm/core/universal-embedder.d.ts +177 -0
- package/dist/esm/core/universal-embedder.js +139 -0
- package/dist/esm/core/validation-messages.d.ts +99 -0
- package/dist/esm/core/validation-messages.js +334 -0
- package/dist/esm/core/vector-index.d.ts +72 -0
- package/dist/esm/core/vector-index.js +333 -0
- package/dist/esm/dom-polyfills.d.ts +6 -0
- package/dist/esm/dom-polyfills.js +37 -0
- package/dist/esm/factories/index.d.ts +27 -0
- package/dist/esm/factories/index.js +29 -0
- package/dist/esm/factories/ingestion-factory.d.ts +200 -0
- package/dist/esm/factories/ingestion-factory.js +477 -0
- package/dist/esm/factories/search-factory.d.ts +154 -0
- package/dist/esm/factories/search-factory.js +344 -0
- package/dist/esm/file-processor.d.ts +147 -0
- package/dist/esm/file-processor.js +963 -0
- package/dist/esm/index-manager.d.ts +116 -0
- package/dist/esm/index-manager.js +598 -0
- package/dist/esm/index.d.ts +75 -0
- package/dist/esm/index.js +110 -0
- package/dist/esm/indexer.d.ts +7 -0
- package/dist/esm/indexer.js +54 -0
- package/dist/esm/ingestion.d.ts +63 -0
- package/dist/esm/ingestion.js +124 -0
- package/dist/esm/mcp-server.d.ts +46 -0
- package/dist/esm/mcp-server.js +1820 -0
- package/dist/esm/multimodal/clip-embedder.d.ts +327 -0
- package/dist/esm/multimodal/clip-embedder.js +996 -0
- package/dist/esm/multimodal/index.d.ts +6 -0
- package/dist/esm/multimodal/index.js +6 -0
- package/dist/esm/preprocess.d.ts +19 -0
- package/dist/esm/preprocess.js +203 -0
- package/dist/esm/preprocessors/index.d.ts +17 -0
- package/dist/esm/preprocessors/index.js +38 -0
- package/dist/esm/preprocessors/mdx.d.ts +25 -0
- package/dist/esm/preprocessors/mdx.js +101 -0
- package/dist/esm/preprocessors/mermaid.d.ts +68 -0
- package/dist/esm/preprocessors/mermaid.js +329 -0
- package/dist/esm/preprocessors/registry.d.ts +56 -0
- package/dist/esm/preprocessors/registry.js +179 -0
- package/dist/esm/run-error-recovery-tests.d.ts +7 -0
- package/dist/esm/run-error-recovery-tests.js +101 -0
- package/dist/esm/search-standalone.d.ts +7 -0
- package/dist/esm/search-standalone.js +117 -0
- package/dist/esm/search.d.ts +99 -0
- package/dist/esm/search.js +177 -0
- package/dist/esm/test-utils.d.ts +18 -0
- package/dist/esm/test-utils.js +27 -0
- package/dist/esm/text/chunker.d.ts +33 -0
- package/dist/esm/text/chunker.js +279 -0
- package/dist/esm/text/embedder.d.ts +111 -0
- package/dist/esm/text/embedder.js +386 -0
- package/dist/esm/text/index.d.ts +8 -0
- package/dist/esm/text/index.js +9 -0
- package/dist/esm/text/preprocessors/index.d.ts +17 -0
- package/dist/esm/text/preprocessors/index.js +38 -0
- package/dist/esm/text/preprocessors/mdx.d.ts +25 -0
- package/dist/esm/text/preprocessors/mdx.js +101 -0
- package/dist/esm/text/preprocessors/mermaid.d.ts +68 -0
- package/dist/esm/text/preprocessors/mermaid.js +330 -0
- package/dist/esm/text/preprocessors/registry.d.ts +56 -0
- package/dist/esm/text/preprocessors/registry.js +180 -0
- package/dist/esm/text/reranker.d.ts +49 -0
- package/dist/esm/text/reranker.js +274 -0
- package/dist/esm/text/sentence-transformer-embedder.d.ts +96 -0
- package/dist/esm/text/sentence-transformer-embedder.js +340 -0
- package/dist/esm/text/tokenizer.d.ts +22 -0
- package/dist/esm/text/tokenizer.js +64 -0
- package/dist/esm/types.d.ts +83 -0
- package/dist/esm/types.js +3 -0
- package/dist/esm/utils/vector-math.d.ts +31 -0
- package/dist/esm/utils/vector-math.js +70 -0
- package/package.json +30 -12
- package/dist/core/binary-index-format.js +0 -122
- /package/dist/{api-errors.d.ts → cjs/api-errors.d.ts} +0 -0
- /package/dist/{api-errors.js → cjs/api-errors.js} +0 -0
- /package/dist/{cli → cjs/cli}/indexer.d.ts +0 -0
- /package/dist/{cli → cjs/cli}/search.d.ts +0 -0
- /package/dist/{cli.d.ts → cjs/cli.d.ts} +0 -0
- /package/dist/{cli.js → cjs/cli.js} +0 -0
- /package/dist/{config.d.ts → cjs/config.d.ts} +0 -0
- /package/dist/{config.js → cjs/config.js} +0 -0
- /package/dist/{core → cjs/core}/abstract-embedder.d.ts +0 -0
- /package/dist/{core → cjs/core}/abstract-embedder.js +0 -0
- /package/dist/{core → cjs/core}/actionable-error-messages.d.ts +0 -0
- /package/dist/{core → cjs/core}/actionable-error-messages.js +0 -0
- /package/dist/{core → cjs/core}/adapters.d.ts +0 -0
- /package/dist/{core → cjs/core}/adapters.js +0 -0
- /package/dist/{core → cjs/core}/batch-processing-optimizer.d.ts +0 -0
- /package/dist/{core → cjs/core}/batch-processing-optimizer.js +0 -0
- /package/dist/{core → cjs/core}/chunker.d.ts +0 -0
- /package/dist/{core → cjs/core}/chunker.js +0 -0
- /package/dist/{core → cjs/core}/cli-database-utils.d.ts +0 -0
- /package/dist/{core → cjs/core}/cli-database-utils.js +0 -0
- /package/dist/{core → cjs/core}/config.d.ts +0 -0
- /package/dist/{core → cjs/core}/config.js +0 -0
- /package/dist/{core → cjs/core}/content-errors.d.ts +0 -0
- /package/dist/{core → cjs/core}/content-errors.js +0 -0
- /package/dist/{core → cjs/core}/content-manager.d.ts +0 -0
- /package/dist/{core → cjs/core}/content-manager.js +0 -0
- /package/dist/{core → cjs/core}/content-performance-optimizer.d.ts +0 -0
- /package/dist/{core → cjs/core}/content-performance-optimizer.js +0 -0
- /package/dist/{core → cjs/core}/content-resolver.d.ts +0 -0
- /package/dist/{core → cjs/core}/content-resolver.js +0 -0
- /package/dist/{core → cjs/core}/cross-modal-search.d.ts +0 -0
- /package/dist/{core → cjs/core}/cross-modal-search.js +0 -0
- /package/dist/{core → cjs/core}/database-connection-manager.d.ts +0 -0
- /package/dist/{core → cjs/core}/database-connection-manager.js +0 -0
- /package/dist/{core → cjs/core}/db.d.ts +0 -0
- /package/dist/{core → cjs/core}/db.js +0 -0
- /package/dist/{core → cjs/core}/embedder-factory.d.ts +0 -0
- /package/dist/{core → cjs/core}/embedder-factory.js +0 -0
- /package/dist/{core → cjs/core}/error-handler.d.ts +0 -0
- /package/dist/{core → cjs/core}/error-handler.js +0 -0
- /package/dist/{core → cjs/core}/index.d.ts +0 -0
- /package/dist/{core → cjs/core}/index.js +0 -0
- /package/dist/{core → cjs/core}/interfaces.d.ts +0 -0
- /package/dist/{core → cjs/core}/interfaces.js +0 -0
- /package/dist/{core → cjs/core}/lazy-dependency-loader.d.ts +0 -0
- /package/dist/{core → cjs/core}/lazy-dependency-loader.js +0 -0
- /package/dist/{core → cjs/core}/mode-detection-service.d.ts +0 -0
- /package/dist/{core → cjs/core}/mode-detection-service.js +0 -0
- /package/dist/{core → cjs/core}/mode-model-validator.d.ts +0 -0
- /package/dist/{core → cjs/core}/mode-model-validator.js +0 -0
- /package/dist/{core → cjs/core}/model-registry.d.ts +0 -0
- /package/dist/{core → cjs/core}/model-registry.js +0 -0
- /package/dist/{core → cjs/core}/model-validator.d.ts +0 -0
- /package/dist/{core → cjs/core}/path-manager.d.ts +0 -0
- /package/dist/{core → cjs/core}/path-manager.js +0 -0
- /package/dist/{core → cjs/core}/raglite-paths.d.ts +0 -0
- /package/dist/{core → cjs/core}/raglite-paths.js +0 -0
- /package/dist/{core → cjs/core}/reranking-config.d.ts +0 -0
- /package/dist/{core → cjs/core}/reranking-config.js +0 -0
- /package/dist/{core → cjs/core}/reranking-factory.d.ts +0 -0
- /package/dist/{core → cjs/core}/reranking-factory.js +0 -0
- /package/dist/{core → cjs/core}/reranking-strategies.d.ts +0 -0
- /package/dist/{core → cjs/core}/resource-cleanup.d.ts +0 -0
- /package/dist/{core → cjs/core}/resource-cleanup.js +0 -0
- /package/dist/{core → cjs/core}/resource-manager.d.ts +0 -0
- /package/dist/{core → cjs/core}/resource-manager.js +0 -0
- /package/dist/{core → cjs/core}/search-pipeline.d.ts +0 -0
- /package/dist/{core → cjs/core}/search-pipeline.js +0 -0
- /package/dist/{core → cjs/core}/search.d.ts +0 -0
- /package/dist/{core → cjs/core}/streaming-operations.d.ts +0 -0
- /package/dist/{core → cjs/core}/streaming-operations.js +0 -0
- /package/dist/{core → cjs/core}/types.js +0 -0
- /package/dist/{core → cjs/core}/universal-embedder.d.ts +0 -0
- /package/dist/{core → cjs/core}/universal-embedder.js +0 -0
- /package/dist/{core → cjs/core}/validation-messages.d.ts +0 -0
- /package/dist/{core → cjs/core}/validation-messages.js +0 -0
- /package/dist/{dom-polyfills.d.ts → cjs/dom-polyfills.d.ts} +0 -0
- /package/dist/{dom-polyfills.js → cjs/dom-polyfills.js} +0 -0
- /package/dist/{factories → cjs/factories}/index.d.ts +0 -0
- /package/dist/{factories → cjs/factories}/index.js +0 -0
- /package/dist/{factories → cjs/factories}/ingestion-factory.d.ts +0 -0
- /package/dist/{factories → cjs/factories}/ingestion-factory.js +0 -0
- /package/dist/{factories → cjs/factories}/search-factory.d.ts +0 -0
- /package/dist/{factories → cjs/factories}/search-factory.js +0 -0
- /package/dist/{index.d.ts → cjs/index.d.ts} +0 -0
- /package/dist/{index.js → cjs/index.js} +0 -0
- /package/dist/{indexer.d.ts → cjs/indexer.d.ts} +0 -0
- /package/dist/{indexer.js → cjs/indexer.js} +0 -0
- /package/dist/{ingestion.d.ts → cjs/ingestion.d.ts} +0 -0
- /package/dist/{ingestion.js → cjs/ingestion.js} +0 -0
- /package/dist/{mcp-server.d.ts → cjs/mcp-server.d.ts} +0 -0
- /package/dist/{mcp-server.js → cjs/mcp-server.js} +0 -0
- /package/dist/{multimodal → cjs/multimodal}/clip-embedder.d.ts +0 -0
- /package/dist/{multimodal → cjs/multimodal}/index.d.ts +0 -0
- /package/dist/{multimodal → cjs/multimodal}/index.js +0 -0
- /package/dist/{preprocess.d.ts → cjs/preprocess.d.ts} +0 -0
- /package/dist/{preprocess.js → cjs/preprocess.js} +0 -0
- /package/dist/{preprocessors → cjs/preprocessors}/index.d.ts +0 -0
- /package/dist/{preprocessors → cjs/preprocessors}/index.js +0 -0
- /package/dist/{preprocessors → cjs/preprocessors}/mdx.d.ts +0 -0
- /package/dist/{preprocessors → cjs/preprocessors}/mdx.js +0 -0
- /package/dist/{preprocessors → cjs/preprocessors}/mermaid.d.ts +0 -0
- /package/dist/{preprocessors → cjs/preprocessors}/mermaid.js +0 -0
- /package/dist/{preprocessors → cjs/preprocessors}/registry.d.ts +0 -0
- /package/dist/{preprocessors → cjs/preprocessors}/registry.js +0 -0
- /package/dist/{run-error-recovery-tests.d.ts → cjs/run-error-recovery-tests.d.ts} +0 -0
- /package/dist/{run-error-recovery-tests.js → cjs/run-error-recovery-tests.js} +0 -0
- /package/dist/{search-standalone.d.ts → cjs/search-standalone.d.ts} +0 -0
- /package/dist/{search-standalone.js → cjs/search-standalone.js} +0 -0
- /package/dist/{search.d.ts → cjs/search.d.ts} +0 -0
- /package/dist/{search.js → cjs/search.js} +0 -0
- /package/dist/{test-utils.d.ts → cjs/test-utils.d.ts} +0 -0
- /package/dist/{test-utils.js → cjs/test-utils.js} +0 -0
- /package/dist/{text → cjs/text}/chunker.d.ts +0 -0
- /package/dist/{text → cjs/text}/chunker.js +0 -0
- /package/dist/{text → cjs/text}/embedder.d.ts +0 -0
- /package/dist/{text → cjs/text}/embedder.js +0 -0
- /package/dist/{text → cjs/text}/index.d.ts +0 -0
- /package/dist/{text → cjs/text}/index.js +0 -0
- /package/dist/{text → cjs/text}/preprocessors/index.d.ts +0 -0
- /package/dist/{text → cjs/text}/preprocessors/index.js +0 -0
- /package/dist/{text → cjs/text}/preprocessors/mdx.d.ts +0 -0
- /package/dist/{text → cjs/text}/preprocessors/mdx.js +0 -0
- /package/dist/{text → cjs/text}/preprocessors/mermaid.d.ts +0 -0
- /package/dist/{text → cjs/text}/preprocessors/mermaid.js +0 -0
- /package/dist/{text → cjs/text}/preprocessors/registry.d.ts +0 -0
- /package/dist/{text → cjs/text}/preprocessors/registry.js +0 -0
- /package/dist/{text → cjs/text}/reranker.d.ts +0 -0
- /package/dist/{text → cjs/text}/reranker.js +0 -0
- /package/dist/{text → cjs/text}/sentence-transformer-embedder.d.ts +0 -0
- /package/dist/{text → cjs/text}/sentence-transformer-embedder.js +0 -0
- /package/dist/{text → cjs/text}/tokenizer.d.ts +0 -0
- /package/dist/{text → cjs/text}/tokenizer.js +0 -0
- /package/dist/{types.d.ts → cjs/types.d.ts} +0 -0
- /package/dist/{types.js → cjs/types.js} +0 -0
- /package/dist/{utils → cjs/utils}/vector-math.d.ts +0 -0
- /package/dist/{utils → cjs/utils}/vector-math.js +0 -0
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Factory functions for creating text-specific search and ingestion instances
|
|
3
|
+
* Handles complex initialization logic while providing clean API for common use cases
|
|
4
|
+
*
|
|
5
|
+
* FACTORY PATTERN BENEFITS:
|
|
6
|
+
* - Abstracts complex initialization (model loading, database setup, index initialization)
|
|
7
|
+
* - Provides simple API for common use cases while preserving access to dependency injection
|
|
8
|
+
* - Clear validation and error handling without fallback mechanisms
|
|
9
|
+
* - Supports different embedding models and configurations
|
|
10
|
+
* - Enables clean separation between simple usage and advanced customization
|
|
11
|
+
*
|
|
12
|
+
* MODE SELECTION GUIDE:
|
|
13
|
+
* - Text Mode (default): Optimized for text-only content
|
|
14
|
+
* - Uses sentence-transformer models (fast, accurate for text)
|
|
15
|
+
* - Images converted to text descriptions
|
|
16
|
+
* - Best for: document search, text clustering, semantic similarity
|
|
17
|
+
*
|
|
18
|
+
* - Multimodal Mode: Optimized for mixed text/image content
|
|
19
|
+
* - Uses CLIP models (unified embedding space)
|
|
20
|
+
* - True cross-modal search (text finds images, images find text)
|
|
21
|
+
* - Best for: image search, visual QA, multimodal retrieval
|
|
22
|
+
*
|
|
23
|
+
* USAGE PATTERNS:
|
|
24
|
+
*
|
|
25
|
+
* 1. Mode Selection:
|
|
26
|
+
* ```typescript
|
|
27
|
+
* // Text mode (default) - optimized for text-only content
|
|
28
|
+
* const textIngestion = await IngestionFactory.create('./db.sqlite', './index.bin', {
|
|
29
|
+
* mode: 'text',
|
|
30
|
+
* embeddingModel: 'sentence-transformers/all-MiniLM-L6-v2'
|
|
31
|
+
* });
|
|
32
|
+
*
|
|
33
|
+
* // Multimodal mode - enables cross-modal search
|
|
34
|
+
* const multimodalIngestion = await IngestionFactory.create('./db.sqlite', './index.bin', {
|
|
35
|
+
* mode: 'multimodal',
|
|
36
|
+
* embeddingModel: 'Xenova/clip-vit-base-patch32',
|
|
37
|
+
* rerankingStrategy: 'text-derived'
|
|
38
|
+
* });
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
41
|
+
import { IngestionPipeline } from '../core/ingestion.js';
|
|
42
|
+
import { IndexManager } from '../index-manager.js';
|
|
43
|
+
import { openDatabase } from '../core/db.js';
|
|
44
|
+
import { createTextEmbedFunction } from '../text/embedder.js';
|
|
45
|
+
import { config, getModelDefaults } from '../core/config.js';
|
|
46
|
+
import { existsSync } from 'fs';
|
|
47
|
+
import { dirname } from 'path';
|
|
48
|
+
import { mkdirSync } from 'fs';
|
|
49
|
+
import { ContentManager } from '../core/content-manager.js';
|
|
50
|
+
import { validateModeModelCompatibilityOrThrow } from '../core/mode-model-validator.js';
|
|
51
|
+
import { createInvalidPathError, createFactoryCreationError, createModeMismatchError } from '../core/actionable-error-messages.js';
|
|
52
|
+
/**
|
|
53
|
+
* Factory for creating text-based IngestionPipeline instances
|
|
54
|
+
* Handles model loading, database initialization, and index setup
|
|
55
|
+
*
|
|
56
|
+
* This factory abstracts the complex initialization process required for text ingestion:
|
|
57
|
+
* 1. Creates necessary directories if they don't exist
|
|
58
|
+
* 2. Validates mode-model compatibility (no fallback mechanisms)
|
|
59
|
+
* 3. Loads and validates embedding models with clear error reporting
|
|
60
|
+
* 4. Establishes database connections and initializes schema
|
|
61
|
+
* 5. Stores mode configuration in database for automatic detection
|
|
62
|
+
* 6. Creates or loads vector indexes with proper configuration
|
|
63
|
+
* 7. Creates IngestionPipeline with proper dependency injection
|
|
64
|
+
*
|
|
65
|
+
* Mode Configuration:
|
|
66
|
+
* - Text Mode (default): Uses sentence-transformer models for text-only content
|
|
67
|
+
* - Multimodal Mode: Uses CLIP models for mixed text/image content
|
|
68
|
+
* - Mode is stored in database and auto-detected during search
|
|
69
|
+
* - Clear validation prevents mode-model mismatches
|
|
70
|
+
*
|
|
71
|
+
* @example
|
|
72
|
+
* ```typescript
|
|
73
|
+
* // Basic usage
|
|
74
|
+
* const ingestion = await IngestionFactory.create('./db.sqlite', './index.bin');
|
|
75
|
+
* await ingestion.ingestDirectory('./documents');
|
|
76
|
+
*
|
|
77
|
+
* // With custom configuration
|
|
78
|
+
* const ingestion = await IngestionFactory.create('./db.sqlite', './index.bin', {
|
|
79
|
+
* embeddingModel: 'all-MiniLM-L6-v2',
|
|
80
|
+
* chunkSize: 512,
|
|
81
|
+
* chunkOverlap: 50,
|
|
82
|
+
* forceRebuild: true
|
|
83
|
+
* });
|
|
84
|
+
*
|
|
85
|
+
* // With defaults
|
|
86
|
+
* const ingestion = await IngestionFactory.createWithDefaults({
|
|
87
|
+
* batchSize: 32 // Faster processing
|
|
88
|
+
* });
|
|
89
|
+
* ```
|
|
90
|
+
*/
|
|
91
|
+
export class IngestionFactory {
|
|
92
|
+
/**
|
|
93
|
+
* Create an IngestionPipeline configured for text ingestion
|
|
94
|
+
*
|
|
95
|
+
* This method handles the complete initialization process:
|
|
96
|
+
* - Creates necessary directories if they don't exist
|
|
97
|
+
* - Loads text embedding model (with lazy initialization)
|
|
98
|
+
* - Opens database connection and initializes schema
|
|
99
|
+
* - Creates or loads vector index (with force rebuild option)
|
|
100
|
+
* - Creates IngestionPipeline with dependency injection
|
|
101
|
+
* - Validates the complete setup
|
|
102
|
+
*
|
|
103
|
+
* @param dbPath - Path to the SQLite database file (will be created if doesn't exist)
|
|
104
|
+
* @param indexPath - Path to the vector index file (will be created if doesn't exist)
|
|
105
|
+
* @param options - Optional configuration overrides
|
|
106
|
+
* @param options.embeddingModel - Override embedding model (default: from config)
|
|
107
|
+
* @param options.batchSize - Override embedding batch size (default: from config)
|
|
108
|
+
* @param options.chunkSize - Override chunk size (default: from config)
|
|
109
|
+
* @param options.chunkOverlap - Override chunk overlap (default: from config)
|
|
110
|
+
* @param options.forceRebuild - Force rebuild of existing index (default: false)
|
|
111
|
+
* @param options.contentSystemConfig - Content system configuration options
|
|
112
|
+
* @param options.contentSystemConfig.contentDir - Content directory path (default: '.raglite/content')
|
|
113
|
+
* @param options.contentSystemConfig.maxFileSize - Maximum file size in bytes (default: 50MB)
|
|
114
|
+
* @param options.contentSystemConfig.maxContentDirSize - Maximum content directory size (default: 2GB)
|
|
115
|
+
* @param options.contentSystemConfig.enableDeduplication - Enable content deduplication (default: true)
|
|
116
|
+
* @param options.contentSystemConfig.enableStorageTracking - Enable storage tracking (default: true)
|
|
117
|
+
* @returns Promise resolving to configured IngestionPipeline
|
|
118
|
+
* @throws {Error} If initialization fails
|
|
119
|
+
*
|
|
120
|
+
* @example
|
|
121
|
+
* ```typescript
|
|
122
|
+
* // Create ingestion pipeline with default content system
|
|
123
|
+
* const ingestion = await IngestionFactory.create('./my-db.sqlite', './my-index.bin');
|
|
124
|
+
*
|
|
125
|
+
* // Create with custom content system configuration
|
|
126
|
+
* const ingestion = await IngestionFactory.create('./my-db.sqlite', './my-index.bin', {
|
|
127
|
+
* contentSystemConfig: {
|
|
128
|
+
* contentDir: './custom-content',
|
|
129
|
+
* maxFileSize: 100 * 1024 * 1024, // 100MB
|
|
130
|
+
* maxContentDirSize: 5 * 1024 * 1024 * 1024, // 5GB
|
|
131
|
+
* enableDeduplication: true
|
|
132
|
+
* }
|
|
133
|
+
* });
|
|
134
|
+
*
|
|
135
|
+
* // Ingest documents from directory
|
|
136
|
+
* const result = await ingestion.ingestDirectory('./documents');
|
|
137
|
+
* console.log(`Processed ${result.documentsProcessed} documents`);
|
|
138
|
+
*
|
|
139
|
+
* // Ingest content from memory (MCP integration)
|
|
140
|
+
* const contentId = await ingestion.ingestFromMemory(buffer, {
|
|
141
|
+
* displayName: 'uploaded-file.pdf',
|
|
142
|
+
* contentType: 'application/pdf'
|
|
143
|
+
* });
|
|
144
|
+
*
|
|
145
|
+
* // Clean up when done
|
|
146
|
+
* await ingestion.cleanup();
|
|
147
|
+
* ```
|
|
148
|
+
*/
|
|
149
|
+
static async create(dbPath, indexPath, options = {}) {
|
|
150
|
+
try {
|
|
151
|
+
console.log('🏭 IngestionFactory: Initializing text ingestion pipeline...');
|
|
152
|
+
// Validate input paths
|
|
153
|
+
if (!dbPath || !indexPath) {
|
|
154
|
+
throw createInvalidPathError([
|
|
155
|
+
{ name: 'dbPath', value: dbPath },
|
|
156
|
+
{ name: 'indexPath', value: indexPath }
|
|
157
|
+
], { operationContext: 'IngestionFactory.create' });
|
|
158
|
+
}
|
|
159
|
+
// Ensure directories exist
|
|
160
|
+
const dbDir = dirname(dbPath);
|
|
161
|
+
const indexDir = dirname(indexPath);
|
|
162
|
+
if (!existsSync(dbDir)) {
|
|
163
|
+
console.log(`📁 Creating database directory: ${dbDir}`);
|
|
164
|
+
mkdirSync(dbDir, { recursive: true });
|
|
165
|
+
}
|
|
166
|
+
if (!existsSync(indexDir)) {
|
|
167
|
+
console.log(`📁 Creating index directory: ${indexDir}`);
|
|
168
|
+
mkdirSync(indexDir, { recursive: true });
|
|
169
|
+
}
|
|
170
|
+
// Step 1: Determine effective mode and select appropriate default model
|
|
171
|
+
const effectiveMode = options.mode || 'text';
|
|
172
|
+
// Step 1.5: Select model based on mode if not explicitly provided
|
|
173
|
+
let effectiveModel;
|
|
174
|
+
if (options.embeddingModel) {
|
|
175
|
+
// Use explicitly provided model
|
|
176
|
+
effectiveModel = options.embeddingModel;
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
// Select default model based on mode
|
|
180
|
+
if (effectiveMode === 'multimodal') {
|
|
181
|
+
const { DEFAULT_MODELS } = await import('../core/model-registry.js');
|
|
182
|
+
effectiveModel = DEFAULT_MODELS['clip'];
|
|
183
|
+
console.log(`📊 No model specified for multimodal mode, using default: ${effectiveModel}`);
|
|
184
|
+
}
|
|
185
|
+
else {
|
|
186
|
+
effectiveModel = config.embedding_model;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
// Step 2: Get model-specific defaults and merge with options
|
|
190
|
+
const modelDefaults = getModelDefaults(effectiveModel);
|
|
191
|
+
const effectiveBatchSize = options.batchSize ?? modelDefaults.batch_size;
|
|
192
|
+
const effectiveChunkSize = options.chunkSize ?? modelDefaults.chunk_size;
|
|
193
|
+
const effectiveChunkOverlap = options.chunkOverlap ?? modelDefaults.chunk_overlap;
|
|
194
|
+
// Step 3: Validate mode-model compatibility at creation time
|
|
195
|
+
console.log('🔍 Validating mode-model compatibility...');
|
|
196
|
+
validateModeModelCompatibilityOrThrow(effectiveMode, effectiveModel);
|
|
197
|
+
console.log('✓ Mode-model compatibility validated');
|
|
198
|
+
// Step 4: Initialize embedding function based on mode
|
|
199
|
+
let embedFn;
|
|
200
|
+
if (effectiveMode === 'multimodal') {
|
|
201
|
+
console.log('📊 Loading CLIP embedding model for multimodal mode...');
|
|
202
|
+
const { createEmbedder } = await import('../core/embedder-factory.js');
|
|
203
|
+
const clipEmbedder = await createEmbedder(effectiveModel);
|
|
204
|
+
// Wrap CLIP embedder to match EmbedFunction signature
|
|
205
|
+
embedFn = async (content, contentType) => {
|
|
206
|
+
if (contentType === 'image') {
|
|
207
|
+
// Use CLIP image embedding for image content
|
|
208
|
+
return await clipEmbedder.embedImage(content);
|
|
209
|
+
}
|
|
210
|
+
// Use CLIP text embedding for text content
|
|
211
|
+
return await clipEmbedder.embedText(content);
|
|
212
|
+
};
|
|
213
|
+
console.log('✓ CLIP embedder created for multimodal mode');
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
// Text mode: use sentence-transformer embedder (existing behavior)
|
|
217
|
+
console.log('📊 Loading text embedding model...');
|
|
218
|
+
embedFn = createTextEmbedFunction(options.embeddingModel, effectiveBatchSize);
|
|
219
|
+
console.log('✓ Text embedding function created successfully');
|
|
220
|
+
}
|
|
221
|
+
// Step 3: Initialize database connection
|
|
222
|
+
console.log('💾 Opening database connection...');
|
|
223
|
+
const db = await openDatabase(dbPath);
|
|
224
|
+
// Initialize database schema if needed
|
|
225
|
+
const { initializeSchema } = await import('../core/db.js');
|
|
226
|
+
await initializeSchema(db);
|
|
227
|
+
console.log('✓ Database connection established');
|
|
228
|
+
// Step 3.1: Handle mode storage during ingestion
|
|
229
|
+
await this.handleModeStorage(db, options, modelDefaults, effectiveModel);
|
|
230
|
+
// Step 5: Initialize index manager
|
|
231
|
+
console.log('📇 Initializing vector index...');
|
|
232
|
+
const indexManager = new IndexManager(indexPath, dbPath, modelDefaults.dimensions, effectiveModel);
|
|
233
|
+
// Check if we need to force recreation due to model change
|
|
234
|
+
let forceRecreate = false;
|
|
235
|
+
if (options.forceRebuild && existsSync(indexPath) && existsSync(dbPath)) {
|
|
236
|
+
// When forceRebuild is true, always force recreation to handle any model/dimension mismatches
|
|
237
|
+
forceRecreate = true;
|
|
238
|
+
// Check if model has changed during rebuild for logging purposes
|
|
239
|
+
const { getSystemInfo } = await import('../core/db.js');
|
|
240
|
+
const tempDb = await openDatabase(dbPath);
|
|
241
|
+
try {
|
|
242
|
+
const systemInfo = await getSystemInfo(tempDb);
|
|
243
|
+
if (systemInfo && systemInfo.modelName && systemInfo.modelName !== effectiveModel) {
|
|
244
|
+
console.log(`🔄 Model change detected: ${systemInfo.modelName} → ${effectiveModel}`);
|
|
245
|
+
console.log(`🔄 Dimensions change: ${systemInfo.modelDimensions} → ${modelDefaults.dimensions}`);
|
|
246
|
+
}
|
|
247
|
+
else if (systemInfo && systemInfo.modelDimensions && systemInfo.modelDimensions !== modelDefaults.dimensions) {
|
|
248
|
+
console.log(`🔄 Dimension mismatch detected: ${systemInfo.modelDimensions} → ${modelDefaults.dimensions}`);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
finally {
|
|
252
|
+
await tempDb.close();
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
// Handle force rebuild or create new index
|
|
256
|
+
if (options.forceRebuild || !existsSync(indexPath)) {
|
|
257
|
+
if (options.forceRebuild && existsSync(indexPath)) {
|
|
258
|
+
console.log('🔄 Force rebuild requested, recreating index...');
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
261
|
+
console.log('📇 Creating new vector index...');
|
|
262
|
+
}
|
|
263
|
+
// Initialize with skipModelCheck and forceRecreate for rebuilds
|
|
264
|
+
await indexManager.initialize(options.forceRebuild, forceRecreate);
|
|
265
|
+
// Update stored model info when rebuilding or creating new index
|
|
266
|
+
if (options.forceRebuild || forceRecreate) {
|
|
267
|
+
const { setSystemInfo } = await import('../core/db.js');
|
|
268
|
+
await setSystemInfo(db, {
|
|
269
|
+
modelName: effectiveModel,
|
|
270
|
+
modelDimensions: modelDefaults.dimensions
|
|
271
|
+
});
|
|
272
|
+
console.log(`✓ Updated stored model info: ${effectiveModel} (${modelDefaults.dimensions} dimensions)`);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
else {
|
|
276
|
+
// Load existing index
|
|
277
|
+
await indexManager.initialize();
|
|
278
|
+
}
|
|
279
|
+
console.log('✓ Vector index ready');
|
|
280
|
+
// Step 5: Create ContentManager for unified content system
|
|
281
|
+
console.log('📁 Initializing content management system...');
|
|
282
|
+
const contentSystemConfig = await this.validateAndPrepareContentSystemConfig(options.contentSystemConfig);
|
|
283
|
+
const contentManager = new ContentManager(db, contentSystemConfig);
|
|
284
|
+
console.log('✓ Content management system ready');
|
|
285
|
+
// Step 6: Create IngestionPipeline with dependency injection and chunk configuration
|
|
286
|
+
const chunkConfig = {
|
|
287
|
+
chunkSize: effectiveChunkSize,
|
|
288
|
+
chunkOverlap: effectiveChunkOverlap
|
|
289
|
+
};
|
|
290
|
+
const ingestionPipeline = new IngestionPipeline(embedFn, indexManager, db, chunkConfig, contentManager);
|
|
291
|
+
console.log('🎉 IngestionFactory: Ingestion pipeline initialized successfully');
|
|
292
|
+
return ingestionPipeline;
|
|
293
|
+
}
|
|
294
|
+
catch (error) {
|
|
295
|
+
console.error('❌ IngestionFactory: Failed to create ingestion pipeline');
|
|
296
|
+
// Preserve custom error messages for model mismatch and mode mismatch
|
|
297
|
+
if (error instanceof Error && (error.message.includes('Model mismatch') ||
|
|
298
|
+
error.message.includes('Mode mismatch') ||
|
|
299
|
+
error.message.includes('--force-rebuild') ||
|
|
300
|
+
error.message.includes('--rebuild-if-needed'))) {
|
|
301
|
+
throw error; // Re-throw custom validation errors as-is
|
|
302
|
+
}
|
|
303
|
+
throw createFactoryCreationError('IngestionFactory', error instanceof Error ? error.message : 'Unknown error', { operationContext: 'ingestion pipeline creation' });
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Create an IngestionPipeline with automatic path resolution
|
|
308
|
+
* Uses default paths based on current working directory
|
|
309
|
+
* @param options - Optional configuration overrides
|
|
310
|
+
* @returns Promise resolving to configured IngestionPipeline
|
|
311
|
+
*/
|
|
312
|
+
static async createWithDefaults(options = {}) {
|
|
313
|
+
const dbPath = config.db_file || './database.sqlite';
|
|
314
|
+
const indexPath = config.index_file || './index.bin';
|
|
315
|
+
return this.create(dbPath, indexPath, options);
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Handles mode storage during ingestion
|
|
319
|
+
* Creates or validates system info based on the provided mode and options
|
|
320
|
+
* @private
|
|
321
|
+
*/
|
|
322
|
+
static async handleModeStorage(db, options, modelDefaults, effectiveModel) {
|
|
323
|
+
const { getSystemInfo, setSystemInfo } = await import('../core/db.js');
|
|
324
|
+
// Determine the effective mode and reranking strategy
|
|
325
|
+
const effectiveMode = options.mode || 'text';
|
|
326
|
+
// Phase 1: Fix mode-specific reranking strategy defaults
|
|
327
|
+
const effectiveRerankingStrategy = options.rerankingStrategy ||
|
|
328
|
+
(effectiveMode === 'multimodal' ? 'text-derived' : 'cross-encoder');
|
|
329
|
+
// Determine model type based on model name
|
|
330
|
+
let modelType;
|
|
331
|
+
if (effectiveModel.includes('clip')) {
|
|
332
|
+
modelType = 'clip';
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
modelType = 'sentence-transformer';
|
|
336
|
+
}
|
|
337
|
+
// Determine supported content types based on mode
|
|
338
|
+
const supportedContentTypes = effectiveMode === 'multimodal' ? ['text', 'image'] : ['text'];
|
|
339
|
+
try {
|
|
340
|
+
// Check if system info already exists
|
|
341
|
+
const existingSystemInfo = await getSystemInfo(db);
|
|
342
|
+
if (existingSystemInfo) {
|
|
343
|
+
// Validate mode consistency for subsequent ingestions
|
|
344
|
+
if (existingSystemInfo.mode !== effectiveMode) {
|
|
345
|
+
console.warn(`⚠️ Mode mismatch detected!`);
|
|
346
|
+
console.warn(` Database mode: ${existingSystemInfo.mode}`);
|
|
347
|
+
console.warn(` Requested mode: ${effectiveMode}`);
|
|
348
|
+
if (options.forceRebuild) {
|
|
349
|
+
console.log('🔄 Force rebuild enabled, updating mode configuration...');
|
|
350
|
+
await this.updateSystemInfo(db, effectiveMode, effectiveModel, modelType, modelDefaults, effectiveRerankingStrategy, supportedContentTypes);
|
|
351
|
+
}
|
|
352
|
+
else {
|
|
353
|
+
throw createModeMismatchError(existingSystemInfo.mode, effectiveMode, { operationContext: 'IngestionFactory.create' });
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
else if (existingSystemInfo.modelName !== effectiveModel) {
|
|
357
|
+
// Model change within the same mode
|
|
358
|
+
console.log(`🔄 Model change detected: ${existingSystemInfo.modelName} → ${effectiveModel}`);
|
|
359
|
+
if (options.forceRebuild) {
|
|
360
|
+
console.log('🔄 Force rebuild enabled, updating model configuration...');
|
|
361
|
+
await this.updateSystemInfo(db, effectiveMode, effectiveModel, modelType, modelDefaults, effectiveRerankingStrategy, supportedContentTypes);
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
// Create a specific error message for model mismatch with rebuild suggestions
|
|
365
|
+
const errorMessage = [
|
|
366
|
+
`❌ Model mismatch: Database is configured for '${existingSystemInfo.modelName}', but '${effectiveModel}' was requested.`,
|
|
367
|
+
'',
|
|
368
|
+
'🛠️ How to fix this:',
|
|
369
|
+
' 1. Use --force-rebuild to change models:',
|
|
370
|
+
' raglite ingest <path> --model ' + effectiveModel + ' --force-rebuild',
|
|
371
|
+
'',
|
|
372
|
+
' 2. Or use --rebuild-if-needed for automatic handling:',
|
|
373
|
+
' raglite ingest <path> --model ' + effectiveModel + ' --rebuild-if-needed',
|
|
374
|
+
'',
|
|
375
|
+
' 3. Or continue using the existing model:',
|
|
376
|
+
' raglite ingest <path> # Uses ' + existingSystemInfo.modelName,
|
|
377
|
+
'',
|
|
378
|
+
'🔍 Model switching requires rebuilding the vector index because different models',
|
|
379
|
+
' produce embeddings with different dimensions and characteristics.'
|
|
380
|
+
].join('\n');
|
|
381
|
+
throw new Error(errorMessage);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
else {
|
|
385
|
+
console.log(`✅ Mode consistency validated: ${effectiveMode} mode with ${effectiveModel}`);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
else {
|
|
389
|
+
// First ingestion - create system info
|
|
390
|
+
console.log(`🔧 First ingestion detected, storing system configuration...`);
|
|
391
|
+
console.log(` Mode: ${effectiveMode}`);
|
|
392
|
+
console.log(` Model: ${effectiveModel} (${modelType})`);
|
|
393
|
+
console.log(` Dimensions: ${modelDefaults.dimensions}`);
|
|
394
|
+
console.log(` Reranking: ${effectiveRerankingStrategy}`);
|
|
395
|
+
console.log(` Content types: ${supportedContentTypes.join(', ')}`);
|
|
396
|
+
await this.updateSystemInfo(db, effectiveMode, effectiveModel, modelType, modelDefaults, effectiveRerankingStrategy, supportedContentTypes);
|
|
397
|
+
console.log('✅ System configuration stored successfully');
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
catch (error) {
|
|
401
|
+
if (error instanceof Error && (error.message.includes('Mode mismatch') || error.message.includes('Model mismatch'))) {
|
|
402
|
+
throw error; // Re-throw validation errors with custom messages
|
|
403
|
+
}
|
|
404
|
+
console.error('❌ Failed to handle mode storage:', error);
|
|
405
|
+
throw new Error(`Mode storage failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Updates system info in the database
|
|
410
|
+
* @private
|
|
411
|
+
*/
|
|
412
|
+
static async updateSystemInfo(db, mode, modelName, modelType, modelDefaults, rerankingStrategy, supportedContentTypes) {
|
|
413
|
+
const { setSystemInfo } = await import('../core/db.js');
|
|
414
|
+
await setSystemInfo(db, {
|
|
415
|
+
mode,
|
|
416
|
+
modelName,
|
|
417
|
+
modelType,
|
|
418
|
+
modelDimensions: modelDefaults.dimensions,
|
|
419
|
+
modelVersion: '1.0.0', // TODO: Get actual version from model
|
|
420
|
+
supportedContentTypes,
|
|
421
|
+
rerankingStrategy: rerankingStrategy,
|
|
422
|
+
rerankingModel: undefined,
|
|
423
|
+
rerankingConfig: undefined
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Validates and prepares content system configuration
|
|
428
|
+
* @private
|
|
429
|
+
*/
|
|
430
|
+
static async validateAndPrepareContentSystemConfig(userConfig) {
|
|
431
|
+
// Default configuration
|
|
432
|
+
const defaultConfig = {
|
|
433
|
+
contentDir: '.raglite/content',
|
|
434
|
+
maxFileSize: 50 * 1024 * 1024, // 50MB
|
|
435
|
+
maxContentDirSize: 2 * 1024 * 1024 * 1024, // 2GB
|
|
436
|
+
enableDeduplication: true,
|
|
437
|
+
enableStorageTracking: true
|
|
438
|
+
};
|
|
439
|
+
// Merge with user configuration
|
|
440
|
+
const config = { ...defaultConfig, ...userConfig };
|
|
441
|
+
// Validate content directory path
|
|
442
|
+
if (!config.contentDir || typeof config.contentDir !== 'string') {
|
|
443
|
+
throw new Error('Content directory path must be a non-empty string');
|
|
444
|
+
}
|
|
445
|
+
// Validate file size limits
|
|
446
|
+
if (config.maxFileSize && (typeof config.maxFileSize !== 'number' || config.maxFileSize <= 0)) {
|
|
447
|
+
throw new Error('Maximum file size must be a positive number');
|
|
448
|
+
}
|
|
449
|
+
if (config.maxContentDirSize && (typeof config.maxContentDirSize !== 'number' || config.maxContentDirSize <= 0)) {
|
|
450
|
+
throw new Error('Maximum content directory size must be a positive number');
|
|
451
|
+
}
|
|
452
|
+
// Validate that maxFileSize is not larger than maxContentDirSize
|
|
453
|
+
if (config.maxFileSize && config.maxContentDirSize && config.maxFileSize > config.maxContentDirSize) {
|
|
454
|
+
throw new Error('Maximum file size cannot be larger than maximum content directory size');
|
|
455
|
+
}
|
|
456
|
+
// Validate boolean options
|
|
457
|
+
if (config.enableDeduplication !== undefined && typeof config.enableDeduplication !== 'boolean') {
|
|
458
|
+
throw new Error('enableDeduplication must be a boolean value');
|
|
459
|
+
}
|
|
460
|
+
if (config.enableStorageTracking !== undefined && typeof config.enableStorageTracking !== 'boolean') {
|
|
461
|
+
throw new Error('enableStorageTracking must be a boolean value');
|
|
462
|
+
}
|
|
463
|
+
// Create content directory if it doesn't exist
|
|
464
|
+
try {
|
|
465
|
+
const { promises: fs } = await import('fs');
|
|
466
|
+
await fs.mkdir(config.contentDir, { recursive: true });
|
|
467
|
+
// Verify directory is writable
|
|
468
|
+
await fs.access(config.contentDir, (await import('fs')).constants.W_OK);
|
|
469
|
+
console.log(`✓ Content directory validated: ${config.contentDir}`);
|
|
470
|
+
}
|
|
471
|
+
catch (error) {
|
|
472
|
+
throw new Error(`Failed to create or access content directory '${config.contentDir}': ${error instanceof Error ? error.message : 'Unknown error'}. Please check permissions and path validity.`);
|
|
473
|
+
}
|
|
474
|
+
return config;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
//# sourceMappingURL=ingestion-factory.js.map
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CORE MODULE — Polymorphic Search Factory for Chameleon Architecture
|
|
3
|
+
*
|
|
4
|
+
* Automatically detects mode from database and creates the appropriate search engine.
|
|
5
|
+
* No fallback mechanisms - each mode uses its optimal implementation reliably.
|
|
6
|
+
*
|
|
7
|
+
* Mode Behavior:
|
|
8
|
+
* - Text Mode: Uses sentence-transformer models for fast text-only search
|
|
9
|
+
* - Multimodal Mode: Uses CLIP models for cross-modal text/image search
|
|
10
|
+
*
|
|
11
|
+
* The mode is determined during ingestion and stored in the database, then
|
|
12
|
+
* automatically detected during search - no manual configuration needed.
|
|
13
|
+
*/
|
|
14
|
+
import '../dom-polyfills.js';
|
|
15
|
+
import { SearchEngine } from '../core/search.js';
|
|
16
|
+
import type { SystemInfo, ModeType } from '../types.js';
|
|
17
|
+
/**
|
|
18
|
+
* Factory for creating search engines with automatic mode detection
|
|
19
|
+
*
|
|
20
|
+
* Detects the mode from database configuration and creates the appropriate
|
|
21
|
+
* search engine without fallback mechanisms. Each mode uses its optimal
|
|
22
|
+
* implementation for predictable, reliable behavior.
|
|
23
|
+
*
|
|
24
|
+
* Mode Selection (configured during ingestion):
|
|
25
|
+
* - Text Mode: Optimized for text-only content
|
|
26
|
+
* - Uses sentence-transformer models
|
|
27
|
+
* - Fast text similarity search
|
|
28
|
+
* - Images converted to text descriptions
|
|
29
|
+
*
|
|
30
|
+
* - Multimodal Mode: Optimized for mixed text/image content
|
|
31
|
+
* - Uses CLIP models
|
|
32
|
+
* - Unified embedding space for text and images
|
|
33
|
+
* - True cross-modal search capabilities
|
|
34
|
+
* - Text queries find images, image queries find text
|
|
35
|
+
*/
|
|
36
|
+
export declare class SearchFactory {
|
|
37
|
+
/**
|
|
38
|
+
* Create a SearchEngine with automatic mode detection and configuration
|
|
39
|
+
*
|
|
40
|
+
* Automatically detects the mode from database configuration and creates
|
|
41
|
+
* the appropriate search engine. No fallback mechanisms - each mode works
|
|
42
|
+
* reliably with its optimal implementation.
|
|
43
|
+
*
|
|
44
|
+
* Process:
|
|
45
|
+
* 1. Detects mode from database (text or multimodal)
|
|
46
|
+
* 2. Validates mode-model compatibility
|
|
47
|
+
* 3. Creates appropriate embedder (sentence-transformer or CLIP)
|
|
48
|
+
* 4. Creates appropriate reranker based on mode and strategy
|
|
49
|
+
* 5. Initializes SearchEngine with proper dependency injection
|
|
50
|
+
*
|
|
51
|
+
* Mode Behavior:
|
|
52
|
+
* - Text Mode: Fast text-only search with sentence-transformers
|
|
53
|
+
* - Optimized for text similarity
|
|
54
|
+
* - Optional cross-encoder reranking
|
|
55
|
+
*
|
|
56
|
+
* - Multimodal Mode: Cross-modal search with CLIP
|
|
57
|
+
* - Unified embedding space for text and images
|
|
58
|
+
* - Text queries find images, image queries find text
|
|
59
|
+
* - Optional text-derived or metadata reranking
|
|
60
|
+
*
|
|
61
|
+
* @param indexPath - Path to the vector index file (must exist)
|
|
62
|
+
* @param dbPath - Path to the SQLite database file (must exist)
|
|
63
|
+
* @returns Promise resolving to configured SearchEngine
|
|
64
|
+
* @throws {Error} If files don't exist, mode-model incompatible, or initialization fails
|
|
65
|
+
*
|
|
66
|
+
* @example
|
|
67
|
+
* ```typescript
|
|
68
|
+
* // Automatic mode detection and engine creation
|
|
69
|
+
* const search = await SearchFactory.create('./index.bin', './db.sqlite');
|
|
70
|
+
*
|
|
71
|
+
* // Search works based on detected mode:
|
|
72
|
+
* // Text mode: fast text similarity search
|
|
73
|
+
* const textResults = await search.search('machine learning');
|
|
74
|
+
*
|
|
75
|
+
* // Multimodal mode: cross-modal search
|
|
76
|
+
* const imageResults = await search.search('red sports car'); // Finds images
|
|
77
|
+
* ```
|
|
78
|
+
*/
|
|
79
|
+
static create(indexPath: string, dbPath: string): Promise<SearchEngine>;
|
|
80
|
+
/**
|
|
81
|
+
* Create a search engine configured for text mode
|
|
82
|
+
* @private
|
|
83
|
+
*/
|
|
84
|
+
private static createTextSearchEngine;
|
|
85
|
+
/**
|
|
86
|
+
* Create a search engine configured for multimodal mode
|
|
87
|
+
* @private
|
|
88
|
+
*/
|
|
89
|
+
private static createMultimodalSearchEngine;
|
|
90
|
+
/**
|
|
91
|
+
* Create reranker for text mode using lazy loading
|
|
92
|
+
* @private
|
|
93
|
+
*/
|
|
94
|
+
private static createTextReranker;
|
|
95
|
+
/**
|
|
96
|
+
* Create reranker for multimodal mode using lazy loading
|
|
97
|
+
* @private
|
|
98
|
+
*/
|
|
99
|
+
private static createMultimodalReranker;
|
|
100
|
+
/**
|
|
101
|
+
* Validate that required files exist
|
|
102
|
+
* @private
|
|
103
|
+
*/
|
|
104
|
+
private static validateRequiredFiles;
|
|
105
|
+
/**
|
|
106
|
+
* Enhance creation errors with helpful context
|
|
107
|
+
* @private
|
|
108
|
+
*/
|
|
109
|
+
private static enhanceCreationError;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Quick function to create a search engine with automatic mode detection
|
|
113
|
+
* Convenience wrapper around SearchFactory.create
|
|
114
|
+
*
|
|
115
|
+
* @param indexPath - Path to the vector index file
|
|
116
|
+
* @param dbPath - Path to the database file
|
|
117
|
+
* @returns Promise resolving to configured SearchEngine
|
|
118
|
+
*
|
|
119
|
+
* @example
|
|
120
|
+
* ```typescript
|
|
121
|
+
* const search = await createSearchEngine('./index.bin', './db.sqlite');
|
|
122
|
+
* const results = await search.search('query');
|
|
123
|
+
* ```
|
|
124
|
+
*/
|
|
125
|
+
export declare function createSearchEngine(indexPath: string, dbPath: string): Promise<SearchEngine>;
|
|
126
|
+
/**
|
|
127
|
+
* Check what mode a database is configured for
|
|
128
|
+
* Convenience function for inspecting database configuration
|
|
129
|
+
*
|
|
130
|
+
* @param dbPath - Path to the database file
|
|
131
|
+
* @returns Promise resolving to the detected mode
|
|
132
|
+
*
|
|
133
|
+
* @example
|
|
134
|
+
* ```typescript
|
|
135
|
+
* const mode = await detectSearchEngineMode('./db.sqlite');
|
|
136
|
+
* console.log(`Database is configured for ${mode} mode`);
|
|
137
|
+
* ```
|
|
138
|
+
*/
|
|
139
|
+
export declare function detectSearchEngineMode(dbPath: string): Promise<ModeType>;
|
|
140
|
+
/**
|
|
141
|
+
* Get system information for a database
|
|
142
|
+
* Convenience function for inspecting complete database configuration
|
|
143
|
+
*
|
|
144
|
+
* @param dbPath - Path to the database file
|
|
145
|
+
* @returns Promise resolving to complete SystemInfo
|
|
146
|
+
*
|
|
147
|
+
* @example
|
|
148
|
+
* ```typescript
|
|
149
|
+
* const info = await getSearchEngineInfo('./db.sqlite');
|
|
150
|
+
* console.log(`Mode: ${info.mode}, Model: ${info.modelName}, Dimensions: ${info.modelDimensions}`);
|
|
151
|
+
* ```
|
|
152
|
+
*/
|
|
153
|
+
export declare function getSearchEngineInfo(dbPath: string): Promise<SystemInfo>;
|
|
154
|
+
//# sourceMappingURL=search-factory.d.ts.map
|