vectra 0.12.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +92 -100
- package/bin/vectra.js +3 -0
- package/lib/BrowserWebFetcher.d.ts +75 -0
- package/lib/BrowserWebFetcher.d.ts.map +1 -0
- package/lib/BrowserWebFetcher.js +290 -0
- package/lib/BrowserWebFetcher.js.map +1 -0
- package/lib/FileFetcher.d.ts +5 -0
- package/lib/FileFetcher.d.ts.map +1 -0
- package/lib/FileFetcher.js +89 -0
- package/lib/FileFetcher.js.map +1 -0
- package/lib/FileFetcher.spec.d.ts +2 -0
- package/lib/FileFetcher.spec.d.ts.map +1 -0
- package/lib/FileFetcher.spec.js +244 -0
- package/lib/FileFetcher.spec.js.map +1 -0
- package/lib/FolderWatcher.d.ts +91 -0
- package/lib/FolderWatcher.d.ts.map +1 -0
- package/lib/FolderWatcher.js +304 -0
- package/lib/FolderWatcher.js.map +1 -0
- package/lib/FolderWatcher.spec.d.ts +2 -0
- package/lib/FolderWatcher.spec.d.ts.map +1 -0
- package/lib/FolderWatcher.spec.js +308 -0
- package/lib/FolderWatcher.spec.js.map +1 -0
- package/lib/GPT3Tokenizer.d.ts +9 -0
- package/lib/GPT3Tokenizer.spec.d.ts +2 -0
- package/lib/GPT3Tokenizer.spec.d.ts.map +1 -0
- package/lib/GPT3Tokenizer.spec.js +45 -0
- package/lib/GPT3Tokenizer.spec.js.map +1 -0
- package/lib/ItemSelector.d.ts +41 -0
- package/lib/ItemSelector.d.ts.map +1 -0
- package/lib/ItemSelector.js +179 -0
- package/lib/ItemSelector.js.map +1 -0
- package/lib/ItemSelector.spec.d.ts +2 -0
- package/lib/ItemSelector.spec.d.ts.map +1 -0
- package/lib/ItemSelector.spec.js +204 -0
- package/lib/ItemSelector.spec.js.map +1 -0
- package/lib/LocalDocument.d.ts +54 -0
- package/lib/LocalDocument.d.ts.map +1 -1
- package/lib/LocalDocument.js +116 -0
- package/lib/LocalDocument.js.map +1 -0
- package/lib/LocalDocument.spec.d.ts +2 -0
- package/lib/LocalDocument.spec.d.ts.map +1 -0
- package/lib/LocalDocument.spec.js +214 -0
- package/lib/LocalDocument.spec.js.map +1 -0
- package/lib/LocalDocumentIndex.d.ts +152 -0
- package/lib/LocalDocumentIndex.d.ts.map +1 -1
- package/lib/LocalDocumentIndex.js +420 -0
- package/lib/LocalDocumentIndex.js.map +1 -0
- package/lib/LocalDocumentIndex.spec.d.ts +2 -0
- package/lib/LocalDocumentIndex.spec.d.ts.map +1 -0
- package/lib/LocalDocumentIndex.spec.js +494 -0
- package/lib/LocalDocumentIndex.spec.js.map +1 -0
- package/lib/LocalDocumentResult.d.ts +66 -0
- package/lib/LocalDocumentResult.d.ts.map +1 -1
- package/lib/LocalDocumentResult.js +376 -0
- package/lib/LocalDocumentResult.js.map +1 -0
- package/lib/LocalDocumentResult.spec.d.ts +2 -0
- package/lib/LocalDocumentResult.spec.d.ts.map +1 -0
- package/lib/LocalDocumentResult.spec.js +373 -0
- package/lib/LocalDocumentResult.spec.js.map +1 -0
- package/lib/LocalEmbeddings.d.ts +59 -0
- package/lib/LocalEmbeddings.d.ts.map +1 -0
- package/lib/LocalEmbeddings.js +101 -0
- package/lib/LocalEmbeddings.js.map +1 -0
- package/lib/LocalEmbeddings.spec.d.ts +2 -0
- package/lib/LocalEmbeddings.spec.d.ts.map +1 -0
- package/lib/LocalEmbeddings.spec.js +155 -0
- package/lib/LocalEmbeddings.spec.js.map +1 -0
- package/lib/LocalIndex.d.ts +159 -0
- package/lib/LocalIndex.d.ts.map +1 -1
- package/lib/LocalIndex.js +519 -0
- package/lib/LocalIndex.js.map +1 -0
- package/lib/LocalIndex.spec.d.ts +2 -0
- package/lib/LocalIndex.spec.js +611 -9
- package/lib/LocalIndex.spec.js.map +1 -1
- package/lib/OpenAIEmbeddings.d.ts +124 -0
- package/lib/OpenAIEmbeddings.d.ts.map +1 -0
- package/lib/OpenAIEmbeddings.js +166 -0
- package/lib/OpenAIEmbeddings.js.map +1 -0
- package/lib/OpenAIEmbeddings.spec.d.ts +2 -0
- package/lib/OpenAIEmbeddings.spec.d.ts.map +1 -0
- package/lib/OpenAIEmbeddings.spec.js +298 -0
- package/lib/OpenAIEmbeddings.spec.js.map +1 -0
- package/lib/TextSplitter.d.ts +21 -0
- package/lib/TextSplitter.d.ts.map +1 -1
- package/lib/TextSplitter.js +500 -0
- package/lib/TextSplitter.js.map +1 -0
- package/lib/TextSplitter.spec.d.ts +2 -0
- package/lib/TextSplitter.spec.d.ts.map +1 -0
- package/lib/TextSplitter.spec.js +337 -0
- package/lib/TextSplitter.spec.js.map +1 -0
- package/lib/TransformersEmbeddings.d.ts +121 -0
- package/lib/TransformersEmbeddings.d.ts.map +1 -0
- package/lib/TransformersEmbeddings.js +176 -0
- package/lib/TransformersEmbeddings.js.map +1 -0
- package/lib/TransformersEmbeddings.spec.d.ts +2 -0
- package/lib/TransformersEmbeddings.spec.d.ts.map +1 -0
- package/lib/TransformersEmbeddings.spec.js +198 -0
- package/lib/TransformersEmbeddings.spec.js.map +1 -0
- package/lib/TransformersTokenizer.d.ts +33 -0
- package/lib/TransformersTokenizer.d.ts.map +1 -0
- package/lib/TransformersTokenizer.js +44 -0
- package/lib/TransformersTokenizer.js.map +1 -0
- package/lib/TransformersTokenizer.spec.d.ts +2 -0
- package/lib/TransformersTokenizer.spec.d.ts.map +1 -0
- package/lib/TransformersTokenizer.spec.js +112 -0
- package/lib/TransformersTokenizer.spec.js.map +1 -0
- package/lib/WebFetcher.d.ts +14 -0
- package/lib/WebFetcher.d.ts.map +1 -0
- package/lib/WebFetcher.js +238 -0
- package/lib/WebFetcher.js.map +1 -0
- package/lib/WebFetcher.spec.d.ts +2 -0
- package/lib/WebFetcher.spec.d.ts.map +1 -0
- package/lib/WebFetcher.spec.js +263 -0
- package/lib/WebFetcher.spec.js.map +1 -0
- package/lib/browser.d.ts +30 -0
- package/lib/browser.d.ts.map +1 -0
- package/lib/browser.js +52 -0
- package/lib/browser.js.map +1 -0
- package/lib/codecs/IndexCodec.d.ts +37 -0
- package/lib/codecs/IndexCodec.d.ts.map +1 -0
- package/lib/codecs/IndexCodec.js +3 -0
- package/lib/codecs/IndexCodec.js.map +1 -0
- package/lib/codecs/JsonCodec.d.ts +19 -0
- package/lib/codecs/JsonCodec.d.ts.map +1 -0
- package/lib/codecs/JsonCodec.js +35 -0
- package/lib/codecs/JsonCodec.js.map +1 -0
- package/lib/codecs/JsonCodec.spec.d.ts +2 -0
- package/lib/codecs/JsonCodec.spec.d.ts.map +1 -0
- package/lib/codecs/JsonCodec.spec.js +66 -0
- package/lib/codecs/JsonCodec.spec.js.map +1 -0
- package/lib/codecs/LocalIndex.protobuf.spec.d.ts +2 -0
- package/lib/codecs/LocalIndex.protobuf.spec.d.ts.map +1 -0
- package/lib/codecs/LocalIndex.protobuf.spec.js +108 -0
- package/lib/codecs/LocalIndex.protobuf.spec.js.map +1 -0
- package/lib/codecs/ProtobufCodec.d.ts +20 -0
- package/lib/codecs/ProtobufCodec.d.ts.map +1 -0
- package/lib/codecs/ProtobufCodec.js +225 -0
- package/lib/codecs/ProtobufCodec.js.map +1 -0
- package/lib/codecs/ProtobufCodec.spec.d.ts +2 -0
- package/lib/codecs/ProtobufCodec.spec.d.ts.map +1 -0
- package/lib/codecs/ProtobufCodec.spec.js +155 -0
- package/lib/codecs/ProtobufCodec.spec.js.map +1 -0
- package/lib/codecs/index.d.ts +5 -0
- package/lib/codecs/index.d.ts.map +1 -0
- package/lib/codecs/index.js +21 -0
- package/lib/codecs/index.js.map +1 -0
- package/lib/codecs/migrateIndex.d.ts +24 -0
- package/lib/codecs/migrateIndex.d.ts.map +1 -0
- package/lib/codecs/migrateIndex.js +119 -0
- package/lib/codecs/migrateIndex.js.map +1 -0
- package/lib/codecs/migrateIndex.spec.d.ts +2 -0
- package/lib/codecs/migrateIndex.spec.d.ts.map +1 -0
- package/lib/codecs/migrateIndex.spec.js +151 -0
- package/lib/codecs/migrateIndex.spec.js.map +1 -0
- package/lib/codecs/schemas/index.proto +34 -0
- package/lib/index.d.ts +20 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +36 -0
- package/lib/index.js.map +1 -0
- package/lib/internals/Colorize.d.ts +14 -0
- package/lib/internals/Colorize.d.ts.map +1 -0
- package/lib/internals/Colorize.js +69 -0
- package/lib/internals/Colorize.js.map +1 -0
- package/lib/internals/index.d.ts +3 -0
- package/lib/internals/index.d.ts.map +1 -0
- package/lib/internals/index.js +19 -0
- package/lib/internals/index.js.map +1 -0
- package/lib/internals/types.d.ts +43 -0
- package/lib/internals/types.d.ts.map +1 -0
- package/lib/internals/types.js +3 -0
- package/lib/internals/types.js.map +1 -0
- package/lib/server/IndexManager.d.ts +78 -0
- package/lib/server/IndexManager.d.ts.map +1 -0
- package/lib/server/IndexManager.js +259 -0
- package/lib/server/IndexManager.js.map +1 -0
- package/lib/server/VectraServer.d.ts +40 -0
- package/lib/server/VectraServer.d.ts.map +1 -0
- package/lib/server/VectraServer.js +151 -0
- package/lib/server/VectraServer.js.map +1 -0
- package/lib/server/VectraServer.spec.d.ts +2 -0
- package/lib/server/VectraServer.spec.d.ts.map +1 -0
- package/lib/server/VectraServer.spec.js +322 -0
- package/lib/server/VectraServer.spec.js.map +1 -0
- package/lib/server/handlers/documentHandlers.d.ts +15 -0
- package/lib/server/handlers/documentHandlers.d.ts.map +1 -0
- package/lib/server/handlers/documentHandlers.js +95 -0
- package/lib/server/handlers/documentHandlers.js.map +1 -0
- package/lib/server/handlers/helpers.d.ts +23 -0
- package/lib/server/handlers/helpers.d.ts.map +1 -0
- package/lib/server/handlers/helpers.js +138 -0
- package/lib/server/handlers/helpers.js.map +1 -0
- package/lib/server/handlers/index.d.ts +8 -0
- package/lib/server/handlers/index.d.ts.map +1 -0
- package/lib/server/handlers/index.js +22 -0
- package/lib/server/handlers/index.js.map +1 -0
- package/lib/server/handlers/indexHandlers.d.ts +14 -0
- package/lib/server/handlers/indexHandlers.d.ts.map +1 -0
- package/lib/server/handlers/indexHandlers.js +85 -0
- package/lib/server/handlers/indexHandlers.js.map +1 -0
- package/lib/server/handlers/itemHandlers.d.ts +34 -0
- package/lib/server/handlers/itemHandlers.d.ts.map +1 -0
- package/lib/server/handlers/itemHandlers.js +166 -0
- package/lib/server/handlers/itemHandlers.js.map +1 -0
- package/lib/server/handlers/lifecycleHandlers.d.ts +11 -0
- package/lib/server/handlers/lifecycleHandlers.d.ts.map +1 -0
- package/lib/server/handlers/lifecycleHandlers.js +31 -0
- package/lib/server/handlers/lifecycleHandlers.js.map +1 -0
- package/lib/server/handlers/queryHandlers.d.ts +27 -0
- package/lib/server/handlers/queryHandlers.d.ts.map +1 -0
- package/lib/server/handlers/queryHandlers.js +135 -0
- package/lib/server/handlers/queryHandlers.js.map +1 -0
- package/lib/server/handlers/statsHandlers.d.ts +17 -0
- package/lib/server/handlers/statsHandlers.d.ts.map +1 -0
- package/lib/server/handlers/statsHandlers.js +81 -0
- package/lib/server/handlers/statsHandlers.js.map +1 -0
- package/lib/server/index.d.ts +4 -0
- package/lib/server/index.d.ts.map +1 -0
- package/lib/server/index.js +23 -0
- package/lib/server/index.js.map +1 -0
- package/lib/storage/FileStorage.d.ts +92 -0
- package/lib/storage/FileStorage.d.ts.map +1 -0
- package/lib/storage/FileStorage.js +3 -0
- package/lib/storage/FileStorage.js.map +1 -0
- package/lib/storage/FileStorageUtilities.d.ts +36 -0
- package/lib/storage/FileStorageUtilities.d.ts.map +1 -0
- package/lib/storage/FileStorageUtilities.js +91 -0
- package/lib/storage/FileStorageUtilities.js.map +1 -0
- package/lib/storage/FileStorageUtilities.spec.d.ts +2 -0
- package/lib/storage/FileStorageUtilities.spec.d.ts.map +1 -0
- package/lib/storage/FileStorageUtilities.spec.js +98 -0
- package/lib/storage/FileStorageUtilities.spec.js.map +1 -0
- package/lib/storage/FileType.d.ts +29 -0
- package/lib/storage/FileType.d.ts.map +1 -0
- package/lib/storage/FileType.js +38 -0
- package/lib/storage/FileType.js.map +1 -0
- package/lib/storage/IndexedDBStorage.d.ts +47 -0
- package/lib/storage/IndexedDBStorage.d.ts.map +1 -0
- package/lib/storage/IndexedDBStorage.js +347 -0
- package/lib/storage/IndexedDBStorage.js.map +1 -0
- package/lib/storage/LocalFileStorage.browser.d.ts +19 -0
- package/lib/storage/LocalFileStorage.browser.d.ts.map +1 -0
- package/lib/storage/LocalFileStorage.browser.js +43 -0
- package/lib/storage/LocalFileStorage.browser.js.map +1 -0
- package/lib/storage/LocalFileStorage.d.ts +23 -0
- package/lib/storage/LocalFileStorage.d.ts.map +1 -0
- package/lib/storage/LocalFileStorage.js +152 -0
- package/lib/storage/LocalFileStorage.js.map +1 -0
- package/lib/storage/LocalFileStorage.spec.d.ts +2 -0
- package/lib/storage/LocalFileStorage.spec.d.ts.map +1 -0
- package/lib/storage/LocalFileStorage.spec.js +249 -0
- package/lib/storage/LocalFileStorage.spec.js.map +1 -0
- package/lib/storage/VirtualFileStorage.d.ts +18 -0
- package/lib/storage/VirtualFileStorage.d.ts.map +1 -0
- package/lib/storage/VirtualFileStorage.js +178 -0
- package/lib/storage/VirtualFileStorage.js.map +1 -0
- package/lib/storage/VirtualFileStorage.spec.d.ts +2 -0
- package/lib/storage/VirtualFileStorage.spec.d.ts.map +1 -0
- package/lib/storage/VirtualFileStorage.spec.js +302 -0
- package/lib/storage/VirtualFileStorage.spec.js.map +1 -0
- package/lib/storage/index.d.ts +6 -0
- package/lib/storage/index.d.ts.map +1 -0
- package/lib/storage/index.js +22 -0
- package/lib/storage/index.js.map +1 -0
- package/lib/templates/templates/csharp/README.md +48 -0
- package/lib/templates/templates/csharp/VectraClient.cs +234 -0
- package/lib/templates/templates/go/README.md +71 -0
- package/lib/templates/templates/go/vectra_client.go +322 -0
- package/lib/templates/templates/java/README.md +81 -0
- package/lib/templates/templates/java/VectraClient.java +232 -0
- package/lib/templates/templates/python/README.md +37 -0
- package/lib/templates/templates/python/vectra_client.py +279 -0
- package/lib/templates/templates/rust/Cargo.toml +14 -0
- package/lib/templates/templates/rust/README.md +39 -0
- package/lib/templates/templates/rust/build.rs +4 -0
- package/lib/templates/templates/rust/lib.rs +284 -0
- package/lib/templates/templates/typescript/README.md +96 -0
- package/lib/templates/templates/typescript/VectraClient.ts +374 -0
- package/lib/templates/typescript/VectraClient.d.ts +114 -0
- package/lib/templates/typescript/VectraClient.d.ts.map +1 -0
- package/lib/templates/typescript/VectraClient.js +328 -0
- package/lib/templates/typescript/VectraClient.js.map +1 -0
- package/lib/types.d.ts +153 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +3 -0
- package/lib/types.js.map +1 -0
- package/lib/utils/index.d.ts +2 -0
- package/lib/utils/index.d.ts.map +1 -0
- package/lib/utils/index.js +18 -0
- package/lib/utils/index.js.map +1 -0
- package/lib/utils/pathUtils.d.ts +40 -0
- package/lib/utils/pathUtils.d.ts.map +1 -0
- package/lib/utils/pathUtils.js +98 -0
- package/lib/utils/pathUtils.js.map +1 -0
- package/lib/vectra-cli.d.ts +2 -0
- package/lib/vectra-cli.d.ts.map +1 -1
- package/lib/vectra-cli.generate.spec.d.ts +2 -0
- package/lib/vectra-cli.generate.spec.d.ts.map +1 -0
- package/lib/vectra-cli.generate.spec.js +112 -0
- package/lib/vectra-cli.generate.spec.js.map +1 -0
- package/lib/vectra-cli.js +760 -0
- package/lib/vectra-cli.js.map +1 -0
- package/lib/vectra-cli.spec.d.ts +1 -0
- package/lib/vectra-cli.spec.d.ts.map +1 -0
- package/lib/vectra-cli.spec.js +2 -0
- package/lib/vectra-cli.spec.js.map +1 -0
- package/package.json +91 -16
- package/proto/vectra_service.proto +276 -0
- package/src/BrowserWebFetcher.ts +345 -0
- package/src/FileFetcher.spec.ts +234 -0
- package/src/FileFetcher.ts +37 -25
- package/src/FolderWatcher.spec.ts +288 -0
- package/src/FolderWatcher.ts +304 -0
- package/src/GPT3Tokenizer.spec.ts +50 -0
- package/src/ItemSelector.spec.ts +252 -0
- package/src/ItemSelector.ts +163 -150
- package/src/LocalDocument.spec.ts +211 -0
- package/src/LocalDocument.ts +88 -94
- package/src/LocalDocumentIndex.spec.ts +481 -0
- package/src/LocalDocumentIndex.ts +39 -40
- package/src/LocalDocumentResult.spec.ts +373 -0
- package/src/LocalDocumentResult.ts +489 -319
- package/src/LocalEmbeddings.spec.ts +138 -0
- package/src/LocalEmbeddings.ts +120 -0
- package/src/LocalIndex.spec.ts +808 -66
- package/src/LocalIndex.ts +479 -429
- package/src/OpenAIEmbeddings.spec.ts +354 -0
- package/src/OpenAIEmbeddings.ts +26 -27
- package/src/TextSplitter.spec.ts +342 -0
- package/src/TextSplitter.ts +517 -532
- package/src/TransformersEmbeddings.spec.ts +188 -0
- package/src/TransformersEmbeddings.ts +232 -0
- package/src/TransformersTokenizer.spec.ts +143 -0
- package/src/TransformersTokenizer.ts +45 -0
- package/src/WebFetcher.spec.ts +288 -0
- package/src/WebFetcher.ts +184 -186
- package/src/browser.ts +69 -0
- package/src/codecs/IndexCodec.ts +40 -0
- package/src/codecs/JsonCodec.spec.ts +70 -0
- package/src/codecs/JsonCodec.ts +37 -0
- package/src/codecs/LocalIndex.protobuf.spec.ts +115 -0
- package/src/codecs/ProtobufCodec.spec.ts +166 -0
- package/src/codecs/ProtobufCodec.ts +193 -0
- package/src/codecs/index.ts +4 -0
- package/src/codecs/migrateIndex.spec.ts +176 -0
- package/src/codecs/migrateIndex.ts +125 -0
- package/src/codecs/schemas/index.proto +34 -0
- package/src/index.ts +9 -1
- package/src/internals/Colorize.ts +19 -16
- package/src/server/IndexManager.ts +243 -0
- package/src/server/VectraServer.spec.ts +303 -0
- package/src/server/VectraServer.ts +156 -0
- package/src/server/handlers/documentHandlers.ts +59 -0
- package/src/server/handlers/helpers.ts +93 -0
- package/src/server/handlers/index.ts +7 -0
- package/src/server/handlers/indexHandlers.ts +44 -0
- package/src/server/handlers/itemHandlers.ts +140 -0
- package/src/server/handlers/lifecycleHandlers.ts +26 -0
- package/src/server/handlers/queryHandlers.ts +96 -0
- package/src/server/handlers/statsHandlers.ts +38 -0
- package/src/server/index.ts +3 -0
- package/src/storage/FileStorage.ts +105 -0
- package/src/storage/FileStorageUtilities.spec.ts +106 -0
- package/src/storage/FileStorageUtilities.ts +77 -0
- package/src/storage/FileType.ts +61 -0
- package/src/storage/IndexedDBStorage.ts +365 -0
- package/src/storage/LocalFileStorage.browser.ts +52 -0
- package/src/storage/LocalFileStorage.spec.ts +292 -0
- package/src/storage/LocalFileStorage.ts +98 -0
- package/src/storage/VirtualFileStorage.spec.ts +307 -0
- package/src/storage/VirtualFileStorage.ts +169 -0
- package/src/storage/index.ts +5 -0
- package/src/templates/csharp/README.md +48 -0
- package/src/templates/csharp/VectraClient.cs +234 -0
- package/src/templates/go/README.md +71 -0
- package/src/templates/go/vectra_client.go +322 -0
- package/src/templates/java/README.md +81 -0
- package/src/templates/java/VectraClient.java +232 -0
- package/src/templates/python/README.md +37 -0
- package/src/templates/python/vectra_client.py +279 -0
- package/src/templates/rust/Cargo.toml +14 -0
- package/src/templates/rust/README.md +39 -0
- package/src/templates/rust/build.rs +4 -0
- package/src/templates/rust/lib.rs +284 -0
- package/src/templates/typescript/README.md +96 -0
- package/src/templates/typescript/VectraClient.ts +374 -0
- package/src/types.ts +131 -123
- package/src/utils/index.ts +1 -0
- package/src/utils/pathUtils.ts +106 -0
- package/src/vectra-cli.generate.spec.ts +72 -0
- package/src/vectra-cli.spec.ts +0 -0
- package/src/vectra-cli.ts +687 -246
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.LocalDocument = void 0;
|
|
13
|
+
const pathUtils_1 = require("./utils/pathUtils");
|
|
14
|
+
/**
|
|
15
|
+
* Represents an indexed document stored on disk.
|
|
16
|
+
*/
|
|
17
|
+
class LocalDocument {
|
|
18
|
+
/**
|
|
19
|
+
* Creates a new `LocalDocument` instance.
|
|
20
|
+
* @param index Parent index that contains the document.
|
|
21
|
+
* @param id ID of the document.
|
|
22
|
+
* @param uri URI of the document.
|
|
23
|
+
*/
|
|
24
|
+
constructor(index, id, uri) {
|
|
25
|
+
this._index = index;
|
|
26
|
+
this._id = id;
|
|
27
|
+
this._uri = uri;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Returns the folder path where the document is stored.
|
|
31
|
+
*/
|
|
32
|
+
get folderPath() {
|
|
33
|
+
return this._index.folderPath;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Returns the ID of the document.
|
|
37
|
+
*/
|
|
38
|
+
get id() {
|
|
39
|
+
return this._id;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Returns the URI of the document.
|
|
43
|
+
*/
|
|
44
|
+
get uri() {
|
|
45
|
+
return this._uri;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Returns the length of the document in tokens.
|
|
49
|
+
* @remarks
|
|
50
|
+
* This value will be estimated for documents longer then 40k bytes.
|
|
51
|
+
* @returns Length of the document in tokens.
|
|
52
|
+
*/
|
|
53
|
+
getLength() {
|
|
54
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
55
|
+
const text = yield this.loadText();
|
|
56
|
+
if (text.length <= 40000) {
|
|
57
|
+
return this._index.tokenizer.encode(text).length;
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
return Math.ceil(text.length / 4);
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Determines if the document has additional metadata stored on disk.
|
|
66
|
+
* @returns True if the document has metadata; otherwise, false.
|
|
67
|
+
*/
|
|
68
|
+
hasMetadata() {
|
|
69
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
70
|
+
return this._index.storage.pathExists(pathUtils_1.pathUtils.join(this.folderPath, `${this.id}.json`));
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Loads the metadata for the document from disk.
|
|
75
|
+
* @returns Metadata for the document.
|
|
76
|
+
*/
|
|
77
|
+
loadMetadata() {
|
|
78
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
79
|
+
if (this._metadata == undefined) {
|
|
80
|
+
let json;
|
|
81
|
+
try {
|
|
82
|
+
json = (yield this._index.storage.readFile(pathUtils_1.pathUtils.join(this.folderPath, `${this.id}.json`))).toString('utf-8');
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
throw new Error(`Error reading metadata for document "${this.uri}": ${err.toString()}`);
|
|
86
|
+
}
|
|
87
|
+
try {
|
|
88
|
+
this._metadata = JSON.parse(json);
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
throw new Error(`Error parsing metadata for document "${this.uri}": ${err.toString()}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return this._metadata;
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Loads the text for the document from disk.
|
|
99
|
+
* @returns Text for the document.
|
|
100
|
+
*/
|
|
101
|
+
loadText() {
|
|
102
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
103
|
+
if (this._text == undefined) {
|
|
104
|
+
try {
|
|
105
|
+
this._text = (yield this._index.storage.readFile(pathUtils_1.pathUtils.join(this.folderPath, `${this.id}.txt`))).toString('utf-8');
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
throw new Error(`Error reading text file for document "${this.uri}": ${err.toString()}`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return this._text;
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
exports.LocalDocument = LocalDocument;
|
|
116
|
+
//# sourceMappingURL=LocalDocument.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LocalDocument.js","sourceRoot":"","sources":["../src/LocalDocument.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iDAAsD;AAItD;;GAEG;AACH,MAAa,aAAa;IAOxB;;;;;OAKG;IACH,YAAmB,KAAyB,EAAE,EAAU,EAAE,GAAW;QACnE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAW,UAAU;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,GAAG,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAW,GAAG;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACU,SAAS;;YACpB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YACnD,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;KAAA;IAED;;;OAGG;IACU,WAAW;;YACtB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACvF,CAAC;KAAA;IAED;;;OAGG;IACU,YAAY;;YACvB,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;gBAChC,IAAI,IAAY,CAAC;gBACjB,IAAI,CAAC;oBACH,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAC/G,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACtB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBACnG,CAAC;gBAED,IAAI,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpC,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACtB,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBACnG,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC,SAAU,CAAC;QACzB,CAAC;KAAA;IAED;;;OAGG;IACU,QAAQ;;YACnB,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACH,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,qBAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpH,CAAC;gBAAC,OAAO,GAAY,EAAE,CAAC;oBACtB,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAI,CAAC,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBACpG,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;KAAA;CACF;AArGD,sCAqGC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LocalDocument.spec.d.ts","sourceRoot":"","sources":["../src/LocalDocument.spec.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
45
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
46
|
+
};
|
|
47
|
+
var _a;
|
|
48
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
49
|
+
const node_assert_1 = require("node:assert");
|
|
50
|
+
const pathUtils_1 = __importDefault(require("./utils/pathUtils"));
|
|
51
|
+
const mocha_1 = require("mocha");
|
|
52
|
+
const LocalDocumentModule = __importStar(require("../src/LocalDocument"));
|
|
53
|
+
const LocalDocument = (_a = LocalDocumentModule.LocalDocument) !== null && _a !== void 0 ? _a : LocalDocumentModule.default;
|
|
54
|
+
if (!LocalDocument) {
|
|
55
|
+
throw new Error('Unable to import LocalDocument (neither named "LocalDocument" nor default export found).');
|
|
56
|
+
}
|
|
57
|
+
(0, mocha_1.describe)("LocalDocument", () => {
|
|
58
|
+
let calls;
|
|
59
|
+
let indexStub;
|
|
60
|
+
let storageStub;
|
|
61
|
+
let tokenizerStub;
|
|
62
|
+
let doc;
|
|
63
|
+
const folderPath = "/folder";
|
|
64
|
+
const id = "doc1";
|
|
65
|
+
const uri = "file:///doc1";
|
|
66
|
+
(0, mocha_1.beforeEach)(() => {
|
|
67
|
+
calls = [];
|
|
68
|
+
storageStub = {
|
|
69
|
+
readFile: (filePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
70
|
+
calls.push({ method: "readFile", args: [filePath] });
|
|
71
|
+
if (filePath.endsWith(".txt")) {
|
|
72
|
+
return Buffer.from("hello world", "utf8");
|
|
73
|
+
}
|
|
74
|
+
if (filePath.endsWith(".json")) {
|
|
75
|
+
return Buffer.from(JSON.stringify({ key: "value" }), "utf8");
|
|
76
|
+
}
|
|
77
|
+
throw new Error("File not found");
|
|
78
|
+
}),
|
|
79
|
+
pathExists: (filePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
80
|
+
calls.push({ method: "pathExists", args: [filePath] });
|
|
81
|
+
return filePath.endsWith(".json");
|
|
82
|
+
}),
|
|
83
|
+
};
|
|
84
|
+
tokenizerStub = {
|
|
85
|
+
encode: (text) => {
|
|
86
|
+
calls.push({ method: "encode", args: [text] });
|
|
87
|
+
return Array.from(text);
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
indexStub = {
|
|
91
|
+
folderPath,
|
|
92
|
+
storage: storageStub,
|
|
93
|
+
tokenizer: tokenizerStub,
|
|
94
|
+
};
|
|
95
|
+
doc = new LocalDocument(indexStub, id, uri);
|
|
96
|
+
});
|
|
97
|
+
(0, mocha_1.it)("constructor and getters", () => {
|
|
98
|
+
node_assert_1.strict.equal(doc.id, id);
|
|
99
|
+
node_assert_1.strict.equal(doc.uri, uri);
|
|
100
|
+
node_assert_1.strict.equal(doc.folderPath, folderPath);
|
|
101
|
+
});
|
|
102
|
+
(0, mocha_1.describe)("loadText", () => {
|
|
103
|
+
(0, mocha_1.it)("loads and caches text", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
104
|
+
const text1 = yield doc.loadText();
|
|
105
|
+
const text2 = yield doc.loadText();
|
|
106
|
+
node_assert_1.strict.equal(text1, "hello world");
|
|
107
|
+
node_assert_1.strict.equal(text2, "hello world");
|
|
108
|
+
const readFileCalls = calls.filter((c) => c.method === "readFile");
|
|
109
|
+
node_assert_1.strict.equal(readFileCalls.length, 1);
|
|
110
|
+
node_assert_1.strict.equal(readFileCalls[0].args[0], pathUtils_1.default.join(folderPath, `${id}.txt`));
|
|
111
|
+
}));
|
|
112
|
+
(0, mocha_1.it)("throws on read error with uri in message", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
113
|
+
storageStub.readFile = (filePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
114
|
+
calls.push({ method: "readFile", args: [filePath] });
|
|
115
|
+
throw new Error("read error");
|
|
116
|
+
});
|
|
117
|
+
const doc2 = new LocalDocument(indexStub, id, uri);
|
|
118
|
+
yield node_assert_1.strict.rejects(() => doc2.loadText(), (err) => String(err.message).includes(`Error reading text file for document "${uri}":`));
|
|
119
|
+
}));
|
|
120
|
+
});
|
|
121
|
+
(0, mocha_1.describe)("hasMetadata", () => {
|
|
122
|
+
(0, mocha_1.it)("returns true if metadata exists, using correct path", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
123
|
+
const result = yield doc.hasMetadata();
|
|
124
|
+
node_assert_1.strict.equal(result, true);
|
|
125
|
+
const pe = calls.find((c) => c.method === "pathExists");
|
|
126
|
+
node_assert_1.strict.ok(pe);
|
|
127
|
+
node_assert_1.strict.equal(pe.args[0], pathUtils_1.default.join(folderPath, `${id}.json`));
|
|
128
|
+
}));
|
|
129
|
+
(0, mocha_1.it)("returns false if metadata does not exist", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
130
|
+
storageStub.pathExists = (filePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
131
|
+
calls.push({ method: "pathExists", args: [filePath] });
|
|
132
|
+
return false;
|
|
133
|
+
});
|
|
134
|
+
const doc2 = new LocalDocument(indexStub, id, uri);
|
|
135
|
+
const result = yield doc2.hasMetadata();
|
|
136
|
+
node_assert_1.strict.equal(result, false);
|
|
137
|
+
}));
|
|
138
|
+
});
|
|
139
|
+
(0, mocha_1.describe)("loadMetadata", () => {
|
|
140
|
+
(0, mocha_1.it)("loads and caches metadata, using correct path", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
141
|
+
const meta1 = yield doc.loadMetadata();
|
|
142
|
+
const meta2 = yield doc.loadMetadata();
|
|
143
|
+
node_assert_1.strict.deepEqual(meta1, { key: "value" });
|
|
144
|
+
node_assert_1.strict.equal(meta1, meta2);
|
|
145
|
+
const jsonCalls = calls.filter((c) => c.method === "readFile" && c.args[0].endsWith(".json"));
|
|
146
|
+
node_assert_1.strict.equal(jsonCalls.length, 1);
|
|
147
|
+
node_assert_1.strict.equal(jsonCalls[0].args[0], pathUtils_1.default.join(folderPath, `${id}.json`));
|
|
148
|
+
}));
|
|
149
|
+
(0, mocha_1.it)("throws on read error with uri in message", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
150
|
+
storageStub.readFile = (filePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
151
|
+
calls.push({ method: "readFile", args: [filePath] });
|
|
152
|
+
if (filePath.endsWith(".json"))
|
|
153
|
+
throw new Error("read error");
|
|
154
|
+
return Buffer.from("ok", "utf8");
|
|
155
|
+
});
|
|
156
|
+
const doc2 = new LocalDocument(indexStub, id, uri);
|
|
157
|
+
yield node_assert_1.strict.rejects(() => doc2.loadMetadata(), (err) => String(err.message).includes(`Error reading metadata for document "${uri}":`));
|
|
158
|
+
}));
|
|
159
|
+
(0, mocha_1.it)("throws on parse error with uri in message", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
160
|
+
storageStub.readFile = (filePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
161
|
+
calls.push({ method: "readFile", args: [filePath] });
|
|
162
|
+
if (filePath.endsWith(".json"))
|
|
163
|
+
return Buffer.from("{", "utf8");
|
|
164
|
+
return Buffer.from("ok", "utf8");
|
|
165
|
+
});
|
|
166
|
+
const doc2 = new LocalDocument(indexStub, id, uri);
|
|
167
|
+
yield node_assert_1.strict.rejects(() => doc2.loadMetadata(), (err) => String(err.message).includes(`Error parsing metadata for document "${uri}":`));
|
|
168
|
+
}));
|
|
169
|
+
});
|
|
170
|
+
(0, mocha_1.describe)("getLength", () => {
|
|
171
|
+
(0, mocha_1.it)("uses tokenizer.encode for small text (≤ 40,000 chars)", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
172
|
+
tokenizerStub.encode = (text) => {
|
|
173
|
+
calls.push({ method: "encode", args: [text] });
|
|
174
|
+
return new Array(5);
|
|
175
|
+
};
|
|
176
|
+
storageStub.readFile = (filePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
177
|
+
calls.push({ method: "readFile", args: [filePath] });
|
|
178
|
+
return Buffer.from("small text", "utf8");
|
|
179
|
+
});
|
|
180
|
+
const doc2 = new LocalDocument(indexStub, id, uri);
|
|
181
|
+
const length = yield doc2.getLength();
|
|
182
|
+
node_assert_1.strict.equal(length, 5);
|
|
183
|
+
node_assert_1.strict.ok(calls.some((c) => c.method === "encode"));
|
|
184
|
+
}));
|
|
185
|
+
(0, mocha_1.it)("estimates length for large text (> 40,000 chars) without encoding", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
186
|
+
const largeText = "a".repeat(40001);
|
|
187
|
+
storageStub.readFile = (filePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
188
|
+
calls.push({ method: "readFile", args: [filePath] });
|
|
189
|
+
return Buffer.from(largeText, "utf8");
|
|
190
|
+
});
|
|
191
|
+
tokenizerStub.encode = () => {
|
|
192
|
+
throw new Error("encode should not be called for large text");
|
|
193
|
+
};
|
|
194
|
+
const doc2 = new LocalDocument(indexStub, id, uri);
|
|
195
|
+
const length = yield doc2.getLength();
|
|
196
|
+
node_assert_1.strict.equal(length, Math.ceil(40001 / 4));
|
|
197
|
+
}));
|
|
198
|
+
(0, mocha_1.it)("reuses cached text across calls (no extra reads)", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
199
|
+
tokenizerStub.encode = (text) => new Array(text.length);
|
|
200
|
+
storageStub.readFile = (filePath) => __awaiter(void 0, void 0, void 0, function* () {
|
|
201
|
+
calls.push({ method: "readFile", args: [filePath] });
|
|
202
|
+
return Buffer.from("small text", "utf8");
|
|
203
|
+
});
|
|
204
|
+
const doc2 = new LocalDocument(indexStub, id, uri);
|
|
205
|
+
const length1 = yield doc2.getLength();
|
|
206
|
+
const length2 = yield doc2.getLength();
|
|
207
|
+
node_assert_1.strict.equal(length1, length2);
|
|
208
|
+
const readFileCalls = calls.filter((c) => c.method === "readFile");
|
|
209
|
+
node_assert_1.strict.equal(readFileCalls.length, 1);
|
|
210
|
+
node_assert_1.strict.equal(readFileCalls[0].args[0], pathUtils_1.default.join(folderPath, `${id}.txt`));
|
|
211
|
+
}));
|
|
212
|
+
});
|
|
213
|
+
});
|
|
214
|
+
//# sourceMappingURL=LocalDocument.spec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LocalDocument.spec.js","sourceRoot":"","sources":["../src/LocalDocument.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA+C;AAC/C,kEAAqC;AACrC,iCAAiD;AACjD,0EAA4D;AAI5D,MAAM,aAAa,GACjB,MAAC,mBAA2B,CAAC,aAAa,mCACzC,mBAA2B,CAAC,OAAO,CAAC;AAEvC,IAAI,CAAC,aAAa,EAAE,CAAC;IACnB,MAAM,IAAI,KAAK,CACb,0FAA0F,CAC3F,CAAC;AACJ,CAAC;AAED,IAAA,gBAAQ,EAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,IAAI,KAAwC,CAAC;IAC7C,IAAI,SAAc,CAAC;IACnB,IAAI,WAAgB,CAAC;IACrB,IAAI,aAAkB,CAAC;IACvB,IAAI,GAAoC,CAAC;IACzC,MAAM,UAAU,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,GAAG,MAAM,CAAC;IAClB,MAAM,GAAG,GAAG,cAAc,CAAC;IAE3B,IAAA,kBAAU,EAAC,GAAG,EAAE;QACd,KAAK,GAAG,EAAE,CAAC;QACX,WAAW,GAAG;YACZ,QAAQ,EAAE,CAAO,QAAgB,EAAE,EAAE;gBACnC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrD,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC9B,OAAO,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;gBAC5C,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC/B,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;gBAC/D,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;YACpC,CAAC,CAAA;YACD,UAAU,EAAE,CAAO,QAAgB,EAAE,EAAE;gBACrC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACvD,OAAO,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC,CAAA;SACF,CAAC;QACF,aAAa,GAAG;YACd,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC/C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;SACF,CAAC;QACF,SAAS,GAAG;YACV,UAAU;YACV,OAAO,EAAE,WAAW;YACpB,SAAS,EAAE,aAAa;SACzB,CAAC;QACF,GAAG,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,oBAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACzB,oBAAM,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC3B,oBAAM,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,IAAA,gBAAQ,EAAC,UAAU,EAAE,GAAG,EAAE;QACxB,IAAA,UAAE,EAAC,uBAAuB,EAAE,GAAS,EAAE;YACrC,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC;YACnC,oBAAM,CAAC,KAAK,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YACnC,oBAAM,CAAC,KAAK,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YAEnC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;YACnE,oBAAM,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACtC,oBAAM,CAAC,KAAK,CACV,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EACxB,mBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,CAAC,CACnC,CAAC;QACJ,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,0CAA0C,EAAE,GAAS,EAAE;YACxD,WAAW,CAAC,QAAQ,GAAG,CAAO,QAAgB,EAAE,EAAE;gBAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrD,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;YAChC,CAAC,CAAA,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACnD,MAAM,oBAAM,CAAC,OAAO,CAClB,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,EACrB,CAAC,GAAQ,EAAE,EAAE,CACX,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAC1B,yCAAyC,GAAG,IAAI,CACjD,CACJ,CAAC;QACJ,CAAC,CAAA,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAA,gBAAQ,EAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,IAAA,UAAE,EAAC,qDAAqD,EAAE,GAAS,EAAE;YACnE,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;YACvC,oBAAM,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAC3B,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC;YACxD,oBAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACd,oBAAM,CAAC,KAAK,CAAC,EAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,mBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,0CAA0C,EAAE,GAAS,EAAE;YACxD,WAAW,CAAC,UAAU,GAAG,CAAO,QAAgB,EAAE,EAAE;gBAClD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACvD,OAAO,KAAK,CAAC;YACf,CAAC,CAAA,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,oBAAM,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC,CAAA,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAA,gBAAQ,EAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,IAAA,UAAE,EAAC,+CAA+C,EAAE,GAAS,EAAE;YAC7D,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,YAAY,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,YAAY,EAAE,CAAC;YACvC,oBAAM,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;YAC1C,oBAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAE3B,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAC5B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAC9D,CAAC;YACF,oBAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAClC,oBAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,mBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,0CAA0C,EAAE,GAAS,EAAE;YACxD,WAAW,CAAC,QAAQ,GAAG,CAAO,QAAgB,EAAE,EAAE;gBAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrD,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;gBAC9D,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC,CAAA,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACnD,MAAM,oBAAM,CAAC,OAAO,CAClB,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,EACzB,CAAC,GAAQ,EAAE,EAAE,CACX,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAC1B,wCAAwC,GAAG,IAAI,CAChD,CACJ,CAAC;QACJ,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,2CAA2C,EAAE,GAAS,EAAE;YACzD,WAAW,CAAC,QAAQ,GAAG,CAAO,QAAgB,EAAE,EAAE;gBAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrD,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAChE,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC,CAAA,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACnD,MAAM,oBAAM,CAAC,OAAO,CAClB,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,EACzB,CAAC,GAAQ,EAAE,EAAE,CACX,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAC1B,wCAAwC,GAAG,IAAI,CAChD,CACJ,CAAC;QACJ,CAAC,CAAA,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAA,gBAAQ,EAAC,WAAW,EAAE,GAAG,EAAE;QACzB,IAAA,UAAE,EAAC,uDAAuD,EAAE,GAAS,EAAE;YACrE,aAAa,CAAC,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE;gBACtC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC/C,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC,CAAC;YACF,WAAW,CAAC,QAAQ,GAAG,CAAO,QAAgB,EAAE,EAAE;gBAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrD,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YAC3C,CAAC,CAAA,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,oBAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxB,oBAAM,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC;QACtD,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,mEAAmE,EAAE,GAAS,EAAE;YACjF,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpC,WAAW,CAAC,QAAQ,GAAG,CAAO,QAAgB,EAAE,EAAE;gBAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrD,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACxC,CAAC,CAAA,CAAC;YACF,aAAa,CAAC,MAAM,GAAG,GAAG,EAAE;gBAC1B,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAChE,CAAC,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,oBAAM,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,kDAAkD,EAAE,GAAS,EAAE;YAChE,aAAa,CAAC,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChE,WAAW,CAAC,QAAQ,GAAG,CAAO,QAAgB,EAAE,EAAE;gBAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrD,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YAC3C,CAAC,CAAA,CAAC;YACF,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YAEnD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACvC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACvC,oBAAM,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE/B,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;YACnE,oBAAM,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACtC,oBAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,mBAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;QAC7E,CAAC,CAAA,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { CreateIndexConfig, LocalIndex } from "./LocalIndex";
|
|
2
|
+
import { TextSplitterConfig } from "./TextSplitter";
|
|
3
|
+
import { MetadataFilter, EmbeddingsModel, Tokenizer, MetadataTypes, DocumentChunkMetadata, DocumentCatalogStats } from "./types";
|
|
4
|
+
import { LocalDocumentResult } from './LocalDocumentResult';
|
|
5
|
+
import { LocalDocument } from './LocalDocument';
|
|
6
|
+
import { FileStorage } from './storage';
|
|
7
|
+
import { IndexCodec } from './codecs';
|
|
8
|
+
/**
|
|
9
|
+
* Options for querying documents in the index.
|
|
10
|
+
*/
|
|
11
|
+
export interface DocumentQueryOptions {
|
|
12
|
+
/**
|
|
13
|
+
* Optional. Maximum number of documents to return.
|
|
14
|
+
* @remarks
|
|
15
|
+
* Default is 10.
|
|
16
|
+
*/
|
|
17
|
+
maxDocuments?: number;
|
|
18
|
+
/**
|
|
19
|
+
* Maximum number of chunks to return per document.
|
|
20
|
+
* @remarks
|
|
21
|
+
* Default is 50.
|
|
22
|
+
*/
|
|
23
|
+
maxChunks?: number;
|
|
24
|
+
/**
|
|
25
|
+
* Optional. Filter to apply to the document metadata.
|
|
26
|
+
*/
|
|
27
|
+
filter?: MetadataFilter;
|
|
28
|
+
/**
|
|
29
|
+
* Optional. Turn on bm25 keyword search to perform hybrid search - semantic + keyword
|
|
30
|
+
*/
|
|
31
|
+
isBm25?: boolean;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Configuration settings for a local document index.
|
|
35
|
+
*/
|
|
36
|
+
export interface LocalDocumentIndexConfig {
|
|
37
|
+
/**
|
|
38
|
+
* Folder path where the index is stored.
|
|
39
|
+
*/
|
|
40
|
+
folderPath: string;
|
|
41
|
+
/**
|
|
42
|
+
* Optional. Name of the index file. Defaults to 'index.json'.
|
|
43
|
+
*/
|
|
44
|
+
indexName?: string;
|
|
45
|
+
/**
|
|
46
|
+
* Optional. Embeddings model to use for generating document embeddings.
|
|
47
|
+
*/
|
|
48
|
+
embeddings?: EmbeddingsModel;
|
|
49
|
+
/**
|
|
50
|
+
* Optional. Tokenizer to use for splitting text into tokens.
|
|
51
|
+
*/
|
|
52
|
+
tokenizer?: Tokenizer;
|
|
53
|
+
/**
|
|
54
|
+
* Optional. Configuration settings for splitting text into chunks.
|
|
55
|
+
*/
|
|
56
|
+
chunkingConfig?: Partial<TextSplitterConfig>;
|
|
57
|
+
/**
|
|
58
|
+
* Optional. File storage plugin to use for storing index files.
|
|
59
|
+
* @remarks
|
|
60
|
+
* If not specified, the LocalFileStorageClass will be used.
|
|
61
|
+
*/
|
|
62
|
+
storage?: FileStorage;
|
|
63
|
+
/**
|
|
64
|
+
* Optional. Codec for serialization format.
|
|
65
|
+
* @remarks
|
|
66
|
+
* If not specified, the JsonCodec will be used (backward-compatible).
|
|
67
|
+
*/
|
|
68
|
+
codec?: IndexCodec;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Represents a local index of documents stored on disk.
|
|
72
|
+
*/
|
|
73
|
+
export declare class LocalDocumentIndex extends LocalIndex<DocumentChunkMetadata> {
|
|
74
|
+
private readonly _embeddings?;
|
|
75
|
+
private readonly _tokenizer;
|
|
76
|
+
private readonly _chunkingConfig?;
|
|
77
|
+
private _catalog?;
|
|
78
|
+
private _newCatalog?;
|
|
79
|
+
/**
|
|
80
|
+
* Creates a new `LocalDocumentIndex` instance.
|
|
81
|
+
* @param config Configuration settings for the document index.
|
|
82
|
+
*/
|
|
83
|
+
constructor(config: LocalDocumentIndexConfig);
|
|
84
|
+
/**
|
|
85
|
+
* Returns the embeddings model used by the index (if configured.)
|
|
86
|
+
*/
|
|
87
|
+
get embeddings(): EmbeddingsModel | undefined;
|
|
88
|
+
/**
|
|
89
|
+
* Returns the tokenizer used by the index.
|
|
90
|
+
*/
|
|
91
|
+
get tokenizer(): Tokenizer;
|
|
92
|
+
/**
|
|
93
|
+
* Returns true if the document catalog exists.
|
|
94
|
+
*/
|
|
95
|
+
/** Name of the catalog file (derived from codec extension). */
|
|
96
|
+
private get _catalogName();
|
|
97
|
+
isCatalogCreated(): Promise<boolean>;
|
|
98
|
+
/**
|
|
99
|
+
* Returns the document ID for the given URI.
|
|
100
|
+
* @param uri URI of the document to lookup.
|
|
101
|
+
* @returns Document ID or undefined if not found.
|
|
102
|
+
*/
|
|
103
|
+
getDocumentId(uri: string): Promise<string | undefined>;
|
|
104
|
+
/**
|
|
105
|
+
* Returns the document URI for the given ID.
|
|
106
|
+
* @param documentId ID of the document to lookup.
|
|
107
|
+
* @returns Document URI or undefined if not found.
|
|
108
|
+
*/
|
|
109
|
+
getDocumentUri(documentId: string): Promise<string | undefined>;
|
|
110
|
+
/**
|
|
111
|
+
* Loads the document catalog from disk and returns its stats.
|
|
112
|
+
* @returns Catalog stats.
|
|
113
|
+
*/
|
|
114
|
+
getCatalogStats(): Promise<DocumentCatalogStats>;
|
|
115
|
+
/**
|
|
116
|
+
* Deletes a document from the index.
|
|
117
|
+
* @param uri URI of the document to delete.
|
|
118
|
+
*/
|
|
119
|
+
deleteDocument(uri: string): Promise<void>;
|
|
120
|
+
/**
|
|
121
|
+
* Adds a document to the catalog.
|
|
122
|
+
* @remarks
|
|
123
|
+
* A new update is started if one is not already in progress. If an document with the same uri
|
|
124
|
+
* already exists, it will be replaced.
|
|
125
|
+
* @param uri - Document URI
|
|
126
|
+
* @param text - Document text
|
|
127
|
+
* @param docType - Optional. Document type
|
|
128
|
+
* @param metadata - Optional. Document metadata to index
|
|
129
|
+
* @returns Inserted document
|
|
130
|
+
*/
|
|
131
|
+
upsertDocument(uri: string, text: string, docType?: string, metadata?: Record<string, MetadataTypes>): Promise<LocalDocument>;
|
|
132
|
+
/**
|
|
133
|
+
* Returns all documents in the index.
|
|
134
|
+
* @remarks
|
|
135
|
+
* Each document will contain all of the documents indexed chunks.
|
|
136
|
+
* @returns Array of documents.
|
|
137
|
+
*/
|
|
138
|
+
listDocuments(): Promise<LocalDocumentResult[]>;
|
|
139
|
+
/**
|
|
140
|
+
* Queries the index for documents similar to the given query.
|
|
141
|
+
* @param query Text to query for.
|
|
142
|
+
* @param options Optional. Query options.
|
|
143
|
+
* @returns Array of document results.
|
|
144
|
+
*/
|
|
145
|
+
queryDocuments(query: string, options?: DocumentQueryOptions): Promise<LocalDocumentResult[]>;
|
|
146
|
+
beginUpdate(): Promise<void>;
|
|
147
|
+
cancelUpdate(): void;
|
|
148
|
+
createIndex(config?: CreateIndexConfig): Promise<void>;
|
|
149
|
+
endUpdate(): Promise<void>;
|
|
150
|
+
protected loadIndexData(): Promise<void>;
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=LocalDocumentIndex.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LocalDocumentIndex.d.ts","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"LocalDocumentIndex.d.ts","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAAmC,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAClK,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAmB,UAAU,EAAa,MAAM,UAAU,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACrC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC7C;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IAEtB;;;;OAIG;IACH,KAAK,CAAC,EAAE,UAAU,CAAC;CACtB;AAED;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,UAAU,CAAC,qBAAqB,CAAC;IACrE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAkB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,CAAkB;IACnC,OAAO,CAAC,WAAW,CAAC,CAAkB;IAEtC;;;OAGG;gBACgB,MAAM,EAAE,wBAAwB;IAYnD;;OAEG;IACH,IAAW,UAAU,IAAI,eAAe,GAAG,SAAS,CAEnD;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,SAAS,CAEhC;IAED;;OAEG;IACH,+DAA+D;IAC/D,OAAO,KAAK,YAAY,GAEvB;IAEY,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjD;;;;OAIG;IACU,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAKpE;;;;OAIG;IACU,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAK5E;;;OAGG;IACU,eAAe,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAU7D;;;OAGG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2CvD;;;;;;;;;;OAUG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAiH1I;;;;;OAKG;IACU,aAAa,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAuB5D;;;;;OAKG;IACU,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAoD7F,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAKlC,YAAY,IAAI,IAAI;IAKd,WAAW,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtD,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;cAYvB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;CA0BjD"}
|