react-native-executorch 0.7.0-nightly-b4770df-20260205 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/common/rnexecutorch/ErrorCodes.h +15 -30
- package/common/rnexecutorch/host_objects/ModelHostObject.h +5 -0
- package/common/rnexecutorch/models/llm/LLM.cpp +7 -0
- package/common/rnexecutorch/models/llm/LLM.h +1 -0
- package/lib/module/constants/llmDefaults.js +31 -0
- package/lib/module/constants/llmDefaults.js.map +1 -1
- package/lib/module/constants/modelUrls.js +220 -0
- package/lib/module/constants/modelUrls.js.map +1 -1
- package/lib/module/constants/ocr/models.js +255 -73
- package/lib/module/constants/ocr/models.js.map +1 -1
- package/lib/module/constants/ocr/symbols.js +4 -0
- package/lib/module/constants/ocr/symbols.js.map +1 -1
- package/lib/module/constants/tts/models.js +6 -2
- package/lib/module/constants/tts/models.js.map +1 -1
- package/lib/module/constants/tts/voices.js +27 -3
- package/lib/module/constants/tts/voices.js.map +1 -1
- package/lib/module/constants/versions.js +2 -2
- package/lib/module/controllers/LLMController.js +9 -0
- package/lib/module/controllers/LLMController.js.map +1 -1
- package/lib/module/errors/ErrorCodes.js +54 -0
- package/lib/module/errors/ErrorCodes.js.map +1 -1
- package/lib/module/errors/errorUtils.js +30 -6
- package/lib/module/errors/errorUtils.js.map +1 -1
- package/lib/module/hooks/computer_vision/useClassification.js +7 -0
- package/lib/module/hooks/computer_vision/useClassification.js.map +1 -1
- package/lib/module/hooks/computer_vision/useImageEmbeddings.js +8 -0
- package/lib/module/hooks/computer_vision/useImageEmbeddings.js.map +1 -1
- package/lib/module/hooks/computer_vision/useImageSegmentation.js +7 -0
- package/lib/module/hooks/computer_vision/useImageSegmentation.js.map +1 -1
- package/lib/module/hooks/computer_vision/useOCR.js +7 -0
- package/lib/module/hooks/computer_vision/useOCR.js.map +1 -1
- package/lib/module/hooks/computer_vision/useObjectDetection.js +7 -0
- package/lib/module/hooks/computer_vision/useObjectDetection.js.map +1 -1
- package/lib/module/hooks/computer_vision/useStyleTransfer.js +7 -0
- package/lib/module/hooks/computer_vision/useStyleTransfer.js.map +1 -1
- package/lib/module/hooks/computer_vision/useTextToImage.js +7 -0
- package/lib/module/hooks/computer_vision/useTextToImage.js.map +1 -1
- package/lib/module/hooks/computer_vision/useVerticalOCR.js +7 -0
- package/lib/module/hooks/computer_vision/useVerticalOCR.js.map +1 -1
- package/lib/module/hooks/general/useExecutorchModule.js +8 -0
- package/lib/module/hooks/general/useExecutorchModule.js.map +1 -1
- package/lib/module/hooks/natural_language_processing/useLLM.js +11 -3
- package/lib/module/hooks/natural_language_processing/useLLM.js.map +1 -1
- package/lib/module/hooks/natural_language_processing/useSpeechToText.js +8 -0
- package/lib/module/hooks/natural_language_processing/useSpeechToText.js.map +1 -1
- package/lib/module/hooks/natural_language_processing/useTextEmbeddings.js +7 -0
- package/lib/module/hooks/natural_language_processing/useTextEmbeddings.js.map +1 -1
- package/lib/module/hooks/natural_language_processing/useTextToSpeech.js +8 -0
- package/lib/module/hooks/natural_language_processing/useTextToSpeech.js.map +1 -1
- package/lib/module/hooks/natural_language_processing/useTokenizer.js +7 -0
- package/lib/module/hooks/natural_language_processing/useTokenizer.js.map +1 -1
- package/lib/module/hooks/natural_language_processing/useVAD.js +7 -0
- package/lib/module/hooks/natural_language_processing/useVAD.js.map +1 -1
- package/lib/module/hooks/useModule.js +12 -0
- package/lib/module/hooks/useModule.js.map +1 -1
- package/lib/module/index.js +10 -1
- package/lib/module/index.js.map +1 -1
- package/lib/module/modules/BaseModule.js +22 -0
- package/lib/module/modules/BaseModule.js.map +1 -1
- package/lib/module/modules/computer_vision/ClassificationModule.js +20 -0
- package/lib/module/modules/computer_vision/ClassificationModule.js.map +1 -1
- package/lib/module/modules/computer_vision/ImageEmbeddingsModule.js +19 -0
- package/lib/module/modules/computer_vision/ImageEmbeddingsModule.js.map +1 -1
- package/lib/module/modules/computer_vision/ImageSegmentationModule.js +22 -0
- package/lib/module/modules/computer_vision/ImageSegmentationModule.js.map +1 -1
- package/lib/module/modules/computer_vision/OCRModule.js +26 -0
- package/lib/module/modules/computer_vision/OCRModule.js.map +1 -1
- package/lib/module/modules/computer_vision/ObjectDetectionModule.js +22 -0
- package/lib/module/modules/computer_vision/ObjectDetectionModule.js.map +1 -1
- package/lib/module/modules/computer_vision/StyleTransferModule.js +20 -0
- package/lib/module/modules/computer_vision/StyleTransferModule.js.map +1 -1
- package/lib/module/modules/computer_vision/TextToImageModule.js +33 -0
- package/lib/module/modules/computer_vision/TextToImageModule.js.map +1 -1
- package/lib/module/modules/computer_vision/VerticalOCRModule.js +27 -0
- package/lib/module/modules/computer_vision/VerticalOCRModule.js.map +1 -1
- package/lib/module/modules/general/ExecutorchModule.js +21 -0
- package/lib/module/modules/general/ExecutorchModule.js.map +1 -1
- package/lib/module/modules/natural_language_processing/LLMModule.js +104 -0
- package/lib/module/modules/natural_language_processing/LLMModule.js.map +1 -1
- package/lib/module/modules/natural_language_processing/SpeechToTextModule.js +67 -21
- package/lib/module/modules/natural_language_processing/SpeechToTextModule.js.map +1 -1
- package/lib/module/modules/natural_language_processing/TextEmbeddingsModule.js +21 -0
- package/lib/module/modules/natural_language_processing/TextEmbeddingsModule.js.map +1 -1
- package/lib/module/modules/natural_language_processing/TextToSpeechModule.js +40 -0
- package/lib/module/modules/natural_language_processing/TextToSpeechModule.js.map +1 -1
- package/lib/module/modules/natural_language_processing/TokenizerModule.js +54 -2
- package/lib/module/modules/natural_language_processing/TokenizerModule.js.map +1 -1
- package/lib/module/modules/natural_language_processing/VADModule.js +20 -0
- package/lib/module/modules/natural_language_processing/VADModule.js.map +1 -1
- package/lib/module/types/classification.js +4 -0
- package/lib/module/types/classification.js.map +1 -0
- package/lib/module/types/common.js +96 -0
- package/lib/module/types/common.js.map +1 -1
- package/lib/module/types/executorchModule.js +4 -0
- package/lib/module/types/executorchModule.js.map +1 -0
- package/lib/module/types/imageEmbeddings.js +4 -0
- package/lib/module/types/imageEmbeddings.js.map +1 -0
- package/lib/module/types/imageSegmentation.js +22 -1
- package/lib/module/types/imageSegmentation.js.map +1 -1
- package/lib/module/types/llm.js +80 -2
- package/lib/module/types/llm.js.map +1 -1
- package/lib/module/types/objectDetection.js +42 -2
- package/lib/module/types/objectDetection.js.map +1 -1
- package/lib/module/types/styleTransfer.js +4 -0
- package/lib/module/types/styleTransfer.js.map +1 -0
- package/lib/module/types/textEmbeddings.js +4 -0
- package/lib/module/types/textEmbeddings.js.map +1 -0
- package/lib/module/types/tokenizer.js +4 -0
- package/lib/module/types/tokenizer.js.map +1 -0
- package/lib/module/types/tti.js +4 -0
- package/lib/module/types/tti.js.map +1 -0
- package/lib/module/types/vad.js +2 -0
- package/lib/module/utils/ResourceFetcher.js +63 -0
- package/lib/module/utils/ResourceFetcher.js.map +1 -1
- package/lib/module/utils/llm.js +24 -1
- package/lib/module/utils/llm.js.map +1 -1
- package/lib/typescript/constants/llmDefaults.d.ts +27 -0
- package/lib/typescript/constants/llmDefaults.d.ts.map +1 -1
- package/lib/typescript/constants/modelUrls.d.ts +165 -0
- package/lib/typescript/constants/modelUrls.d.ts.map +1 -1
- package/lib/typescript/constants/ocr/models.d.ts +189 -315
- package/lib/typescript/constants/ocr/models.d.ts.map +1 -1
- package/lib/typescript/constants/ocr/symbols.d.ts +3 -0
- package/lib/typescript/constants/ocr/symbols.d.ts.map +1 -1
- package/lib/typescript/constants/tts/models.d.ts +4 -0
- package/lib/typescript/constants/tts/models.d.ts.map +1 -1
- package/lib/typescript/constants/tts/voices.d.ts +24 -0
- package/lib/typescript/constants/tts/voices.d.ts.map +1 -1
- package/lib/typescript/constants/versions.d.ts +2 -2
- package/lib/typescript/controllers/LLMController.d.ts +2 -0
- package/lib/typescript/controllers/LLMController.d.ts.map +1 -1
- package/lib/typescript/errors/ErrorCodes.d.ts +54 -0
- package/lib/typescript/errors/ErrorCodes.d.ts.map +1 -1
- package/lib/typescript/errors/errorUtils.d.ts +9 -0
- package/lib/typescript/errors/errorUtils.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useClassification.d.ts +9 -15
- package/lib/typescript/hooks/computer_vision/useClassification.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useImageEmbeddings.d.ts +9 -15
- package/lib/typescript/hooks/computer_vision/useImageEmbeddings.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useImageSegmentation.d.ts +9 -38
- package/lib/typescript/hooks/computer_vision/useImageSegmentation.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useOCR.d.ts +9 -19
- package/lib/typescript/hooks/computer_vision/useOCR.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useObjectDetection.d.ts +9 -15
- package/lib/typescript/hooks/computer_vision/useObjectDetection.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useStyleTransfer.d.ts +9 -15
- package/lib/typescript/hooks/computer_vision/useStyleTransfer.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useTextToImage.d.ts +9 -22
- package/lib/typescript/hooks/computer_vision/useTextToImage.d.ts.map +1 -1
- package/lib/typescript/hooks/computer_vision/useVerticalOCR.d.ts +9 -20
- package/lib/typescript/hooks/computer_vision/useVerticalOCR.d.ts.map +1 -1
- package/lib/typescript/hooks/general/useExecutorchModule.d.ts +9 -13
- package/lib/typescript/hooks/general/useExecutorchModule.d.ts.map +1 -1
- package/lib/typescript/hooks/natural_language_processing/useLLM.d.ts +9 -10
- package/lib/typescript/hooks/natural_language_processing/useLLM.d.ts.map +1 -1
- package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts +9 -19
- package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts.map +1 -1
- package/lib/typescript/hooks/natural_language_processing/useTextEmbeddings.d.ts +9 -16
- package/lib/typescript/hooks/natural_language_processing/useTextEmbeddings.d.ts.map +1 -1
- package/lib/typescript/hooks/natural_language_processing/useTextToSpeech.d.ts +9 -15
- package/lib/typescript/hooks/natural_language_processing/useTextToSpeech.d.ts.map +1 -1
- package/lib/typescript/hooks/natural_language_processing/useTokenizer.d.ts +9 -18
- package/lib/typescript/hooks/natural_language_processing/useTokenizer.d.ts.map +1 -1
- package/lib/typescript/hooks/natural_language_processing/useVAD.d.ts +9 -15
- package/lib/typescript/hooks/natural_language_processing/useVAD.d.ts.map +1 -1
- package/lib/typescript/hooks/useModule.d.ts +12 -0
- package/lib/typescript/hooks/useModule.d.ts.map +1 -1
- package/lib/typescript/index.d.ts +10 -1
- package/lib/typescript/index.d.ts.map +1 -1
- package/lib/typescript/modules/BaseModule.d.ts +20 -0
- package/lib/typescript/modules/BaseModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/ClassificationModule.d.ts +21 -1
- package/lib/typescript/modules/computer_vision/ClassificationModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/ImageEmbeddingsModule.d.ts +17 -0
- package/lib/typescript/modules/computer_vision/ImageEmbeddingsModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/ImageSegmentationModule.d.ts +21 -3
- package/lib/typescript/modules/computer_vision/ImageSegmentationModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/OCRModule.d.ts +25 -2
- package/lib/typescript/modules/computer_vision/OCRModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/ObjectDetectionModule.d.ts +20 -0
- package/lib/typescript/modules/computer_vision/ObjectDetectionModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/StyleTransferModule.d.ts +18 -0
- package/lib/typescript/modules/computer_vision/StyleTransferModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/TextToImageModule.d.ts +29 -0
- package/lib/typescript/modules/computer_vision/TextToImageModule.d.ts.map +1 -1
- package/lib/typescript/modules/computer_vision/VerticalOCRModule.d.ts +26 -2
- package/lib/typescript/modules/computer_vision/VerticalOCRModule.d.ts.map +1 -1
- package/lib/typescript/modules/general/ExecutorchModule.d.ts +19 -0
- package/lib/typescript/modules/general/ExecutorchModule.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts +97 -7
- package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts +62 -4
- package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/TextEmbeddingsModule.d.ts +19 -0
- package/lib/typescript/modules/natural_language_processing/TextEmbeddingsModule.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/TextToSpeechModule.d.ts +37 -2
- package/lib/typescript/modules/natural_language_processing/TextToSpeechModule.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/TokenizerModule.d.ts +47 -2
- package/lib/typescript/modules/natural_language_processing/TokenizerModule.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/VADModule.d.ts +18 -0
- package/lib/typescript/modules/natural_language_processing/VADModule.d.ts.map +1 -1
- package/lib/typescript/types/classification.d.ts +50 -0
- package/lib/typescript/types/classification.d.ts.map +1 -0
- package/lib/typescript/types/common.d.ts +92 -0
- package/lib/typescript/types/common.d.ts.map +1 -1
- package/lib/typescript/types/executorchModule.d.ts +45 -0
- package/lib/typescript/types/executorchModule.d.ts.map +1 -0
- package/lib/typescript/types/imageEmbeddings.d.ts +48 -0
- package/lib/typescript/types/imageEmbeddings.d.ts.map +1 -0
- package/lib/typescript/types/imageSegmentation.d.ts +55 -0
- package/lib/typescript/types/imageSegmentation.d.ts.map +1 -1
- package/lib/typescript/types/llm.d.ts +193 -5
- package/lib/typescript/types/llm.d.ts.map +1 -1
- package/lib/typescript/types/objectDetection.d.ts +71 -2
- package/lib/typescript/types/objectDetection.d.ts.map +1 -1
- package/lib/typescript/types/ocr.d.ts +97 -2
- package/lib/typescript/types/ocr.d.ts.map +1 -1
- package/lib/typescript/types/stt.d.ts +114 -0
- package/lib/typescript/types/stt.d.ts.map +1 -1
- package/lib/typescript/types/styleTransfer.d.ts +48 -0
- package/lib/typescript/types/styleTransfer.d.ts.map +1 -0
- package/lib/typescript/types/textEmbeddings.d.ts +53 -0
- package/lib/typescript/types/textEmbeddings.d.ts.map +1 -0
- package/lib/typescript/types/tokenizer.d.ts +75 -0
- package/lib/typescript/types/tokenizer.d.ts.map +1 -0
- package/lib/typescript/types/tti.d.ts +74 -0
- package/lib/typescript/types/tti.d.ts.map +1 -0
- package/lib/typescript/types/tts.d.ts +77 -0
- package/lib/typescript/types/tts.d.ts.map +1 -1
- package/lib/typescript/types/vad.d.ts +53 -0
- package/lib/typescript/types/vad.d.ts.map +1 -1
- package/lib/typescript/utils/ResourceFetcher.d.ts +55 -0
- package/lib/typescript/utils/ResourceFetcher.d.ts.map +1 -1
- package/lib/typescript/utils/llm.d.ts +22 -0
- package/lib/typescript/utils/llm.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/constants/llmDefaults.ts +27 -0
- package/src/constants/modelUrls.ts +166 -0
- package/src/constants/ocr/models.ts +192 -266
- package/src/constants/ocr/symbols.ts +3 -0
- package/src/constants/tts/models.ts +6 -2
- package/src/constants/tts/voices.ts +27 -3
- package/src/constants/versions.ts +2 -2
- package/src/controllers/LLMController.ts +15 -0
- package/src/errors/ErrorCodes.ts +54 -0
- package/src/errors/errorUtils.ts +37 -17
- package/src/hooks/computer_vision/useClassification.ts +15 -7
- package/src/hooks/computer_vision/useImageEmbeddings.ts +15 -7
- package/src/hooks/computer_vision/useImageSegmentation.ts +15 -7
- package/src/hooks/computer_vision/useOCR.ts +9 -21
- package/src/hooks/computer_vision/useObjectDetection.ts +15 -7
- package/src/hooks/computer_vision/useStyleTransfer.ts +15 -7
- package/src/hooks/computer_vision/useTextToImage.ts +9 -26
- package/src/hooks/computer_vision/useVerticalOCR.ts +9 -19
- package/src/hooks/general/useExecutorchModule.ts +12 -7
- package/src/hooks/natural_language_processing/useLLM.ts +24 -28
- package/src/hooks/natural_language_processing/useSpeechToText.ts +13 -5
- package/src/hooks/natural_language_processing/useTextEmbeddings.ts +15 -10
- package/src/hooks/natural_language_processing/useTextToSpeech.ts +10 -6
- package/src/hooks/natural_language_processing/useTokenizer.ts +10 -6
- package/src/hooks/natural_language_processing/useVAD.ts +9 -7
- package/src/hooks/useModule.ts +15 -0
- package/src/index.ts +10 -5
- package/src/modules/BaseModule.ts +20 -0
- package/src/modules/computer_vision/ClassificationModule.ts +19 -1
- package/src/modules/computer_vision/ImageEmbeddingsModule.ts +17 -0
- package/src/modules/computer_vision/ImageSegmentationModule.ts +21 -1
- package/src/modules/computer_vision/OCRModule.ts +25 -2
- package/src/modules/computer_vision/ObjectDetectionModule.ts +20 -0
- package/src/modules/computer_vision/StyleTransferModule.ts +18 -0
- package/src/modules/computer_vision/TextToImageModule.ts +29 -0
- package/src/modules/computer_vision/VerticalOCRModule.ts +26 -2
- package/src/modules/general/ExecutorchModule.ts +19 -0
- package/src/modules/natural_language_processing/LLMModule.ts +104 -18
- package/src/modules/natural_language_processing/SpeechToTextModule.ts +63 -39
- package/src/modules/natural_language_processing/TextEmbeddingsModule.ts +19 -0
- package/src/modules/natural_language_processing/TextToSpeechModule.ts +43 -2
- package/src/modules/natural_language_processing/TokenizerModule.ts +51 -3
- package/src/modules/natural_language_processing/VADModule.ts +18 -0
- package/src/types/classification.ts +51 -0
- package/src/types/common.ts +93 -0
- package/src/types/executorchModule.ts +50 -0
- package/src/types/imageEmbeddings.ts +51 -0
- package/src/types/imageSegmentation.ts +64 -1
- package/src/types/llm.ts +208 -11
- package/src/types/objectDetection.ts +79 -1
- package/src/types/ocr.ts +107 -2
- package/src/types/stt.ts +136 -1
- package/src/types/styleTransfer.ts +51 -0
- package/src/types/textEmbeddings.ts +58 -0
- package/src/types/tokenizer.ts +86 -0
- package/src/types/tti.ts +87 -0
- package/src/types/tts.ts +85 -2
- package/src/types/vad.ts +58 -0
- package/src/utils/ResourceFetcher.ts +55 -0
- package/src/utils/llm.ts +22 -1
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
import { LLMController } from '../../controllers/LLMController';
|
|
4
|
+
/**
|
|
5
|
+
* Module for managing a Large Language Model (LLM) instance.
|
|
6
|
+
*
|
|
7
|
+
* @category Typescript API
|
|
8
|
+
*/
|
|
4
9
|
export class LLMModule {
|
|
10
|
+
/**
|
|
11
|
+
* Creates a new instance of `LLMModule` with optional callbacks.
|
|
12
|
+
* @param optionalCallbacks - Object containing optional callbacks.
|
|
13
|
+
*
|
|
14
|
+
* @returns A new LLMModule instance.
|
|
15
|
+
*/
|
|
5
16
|
constructor({
|
|
6
17
|
tokenCallback,
|
|
7
18
|
messageHistoryCallback
|
|
@@ -11,17 +22,40 @@ export class LLMModule {
|
|
|
11
22
|
messageHistoryCallback
|
|
12
23
|
});
|
|
13
24
|
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Loads the LLM model and tokenizer.
|
|
28
|
+
*
|
|
29
|
+
* @param model - Object containing model, tokenizer, and tokenizer config sources.
|
|
30
|
+
* @param model.modelSource - `ResourceSource` that specifies the location of the model binary.
|
|
31
|
+
* @param model.tokenizerSource - `ResourceSource` pointing to the JSON file which contains the tokenizer.
|
|
32
|
+
* @param model.tokenizerConfigSource - `ResourceSource` pointing to the JSON file which contains the tokenizer config.
|
|
33
|
+
* @param onDownloadProgressCallback - Optional callback to track download progress (value between 0 and 1).
|
|
34
|
+
*/
|
|
14
35
|
async load(model, onDownloadProgressCallback = () => {}) {
|
|
15
36
|
await this.controller.load({
|
|
16
37
|
...model,
|
|
17
38
|
onDownloadProgressCallback
|
|
18
39
|
});
|
|
19
40
|
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Sets new token callback invoked on every token batch.
|
|
44
|
+
*
|
|
45
|
+
* @param tokenCallback - Callback function to handle new tokens.
|
|
46
|
+
*/
|
|
20
47
|
setTokenCallback({
|
|
21
48
|
tokenCallback
|
|
22
49
|
}) {
|
|
23
50
|
this.controller.setTokenCallback(tokenCallback);
|
|
24
51
|
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Configures chat and tool calling and generation settings.
|
|
55
|
+
* See [Configuring the model](https://docs.swmansion.com/react-native-executorch/docs/hooks/natural-language-processing/useLLM#configuring-the-model) for details.
|
|
56
|
+
*
|
|
57
|
+
* @param configuration - Configuration object containing `chatConfig`, `toolsConfig`, and `generationConfig`.
|
|
58
|
+
*/
|
|
25
59
|
configure({
|
|
26
60
|
chatConfig,
|
|
27
61
|
toolsConfig,
|
|
@@ -33,26 +67,96 @@ export class LLMModule {
|
|
|
33
67
|
generationConfig
|
|
34
68
|
});
|
|
35
69
|
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Runs model inference with raw input string.
|
|
73
|
+
* You need to provide entire conversation and prompt (in correct format and with special tokens!) in input string to this method.
|
|
74
|
+
* It doesn't manage conversation context. It is intended for users that need access to the model itself without any wrapper.
|
|
75
|
+
* If you want a simple chat with model the consider using `sendMessage`
|
|
76
|
+
*
|
|
77
|
+
* @param input - Raw input string containing the prompt and conversation history.
|
|
78
|
+
* @returns The generated response as a string.
|
|
79
|
+
*/
|
|
36
80
|
async forward(input) {
|
|
37
81
|
return await this.controller.forward(input);
|
|
38
82
|
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Runs model to complete chat passed in `messages` argument. It doesn't manage conversation context.
|
|
86
|
+
*
|
|
87
|
+
* @param messages - Array of messages representing the chat history.
|
|
88
|
+
* @param tools - Optional array of tools that can be used during generation.
|
|
89
|
+
* @returns The generated response as a string.
|
|
90
|
+
*/
|
|
39
91
|
async generate(messages, tools) {
|
|
40
92
|
return await this.controller.generate(messages, tools);
|
|
41
93
|
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Method to add user message to conversation.
|
|
97
|
+
* After model responds it will call `messageHistoryCallback()` containing both user message and model response.
|
|
98
|
+
* It also returns them.
|
|
99
|
+
*
|
|
100
|
+
* @param message - The message string to send.
|
|
101
|
+
* @returns - Updated message history including the new user message and model response.
|
|
102
|
+
*/
|
|
42
103
|
async sendMessage(message) {
|
|
43
104
|
await this.controller.sendMessage(message);
|
|
44
105
|
return this.controller.messageHistory;
|
|
45
106
|
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Deletes all messages starting with message on `index` position.
|
|
110
|
+
* After deletion it will call `messageHistoryCallback()` containing new history.
|
|
111
|
+
* It also returns it.
|
|
112
|
+
*
|
|
113
|
+
* @param index - The index of the message to delete from history.
|
|
114
|
+
* @returns - Updated message history after deletion.
|
|
115
|
+
*/
|
|
46
116
|
deleteMessage(index) {
|
|
47
117
|
this.controller.deleteMessage(index);
|
|
48
118
|
return this.controller.messageHistory;
|
|
49
119
|
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Interrupts model generation. It may return one more token after interrupt.
|
|
123
|
+
*/
|
|
50
124
|
interrupt() {
|
|
51
125
|
this.controller.interrupt();
|
|
52
126
|
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Returns the number of tokens generated in the last response.
|
|
130
|
+
*
|
|
131
|
+
* @returns The count of generated tokens.
|
|
132
|
+
*/
|
|
53
133
|
getGeneratedTokenCount() {
|
|
54
134
|
return this.controller.getGeneratedTokenCount();
|
|
55
135
|
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Returns the number of prompt tokens in the last message.
|
|
139
|
+
*
|
|
140
|
+
* @returns The count of prompt token.
|
|
141
|
+
*/
|
|
142
|
+
getPromptTokensCount() {
|
|
143
|
+
return this.controller.getPromptTokenCount();
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Returns the number of total tokens from the previous generation. This is a sum of prompt tokens and generated tokens.
|
|
148
|
+
*
|
|
149
|
+
* @returns The count of prompt and generated tokens.
|
|
150
|
+
*/
|
|
151
|
+
getTotalTokensCount() {
|
|
152
|
+
return this.controller.getTotalTokenCount();
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Method to delete the model from memory.
|
|
157
|
+
* Note you cannot delete model while it's generating.
|
|
158
|
+
* You need to interrupt it first and make sure model stopped generation.
|
|
159
|
+
*/
|
|
56
160
|
delete() {
|
|
57
161
|
this.controller.delete();
|
|
58
162
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["LLMController","LLMModule","constructor","tokenCallback","messageHistoryCallback","controller","load","model","onDownloadProgressCallback","setTokenCallback","configure","chatConfig","toolsConfig","generationConfig","forward","input","generate","messages","tools","sendMessage","message","messageHistory","deleteMessage","index","interrupt","getGeneratedTokenCount","delete"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/LLMModule.ts"],"mappings":";;AAAA,SAASA,aAAa,QAAQ,iCAAiC;
|
|
1
|
+
{"version":3,"names":["LLMController","LLMModule","constructor","tokenCallback","messageHistoryCallback","controller","load","model","onDownloadProgressCallback","setTokenCallback","configure","chatConfig","toolsConfig","generationConfig","forward","input","generate","messages","tools","sendMessage","message","messageHistory","deleteMessage","index","interrupt","getGeneratedTokenCount","getPromptTokensCount","getPromptTokenCount","getTotalTokensCount","getTotalTokenCount","delete"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/LLMModule.ts"],"mappings":";;AAAA,SAASA,aAAa,QAAQ,iCAAiC;AAI/D;AACA;AACA;AACA;AACA;AACA,OAAO,MAAMC,SAAS,CAAC;EAGrB;AACF;AACA;AACA;AACA;AACA;EACEC,WAAWA,CAAC;IACVC,aAAa;IACbC;EAWF,CAAC,GAAG,CAAC,CAAC,EAAE;IACN,IAAI,CAACC,UAAU,GAAG,IAAIL,aAAa,CAAC;MAClCG,aAAa;MACbC;IACF,CAAC,CAAC;EACJ;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACE,MAAME,IAAIA,CACRC,KAIC,EACDC,0BAAsD,GAAGA,CAAA,KAAM,CAAC,CAAC,EACjE;IACA,MAAM,IAAI,CAACH,UAAU,CAACC,IAAI,CAAC;MACzB,GAAGC,KAAK;MACRC;IACF,CAAC,CAAC;EACJ;;EAEA;AACF;AACA;AACA;AACA;EACEC,gBAAgBA,CAAC;IACfN;EAGF,CAAC,EAAE;IACD,IAAI,CAACE,UAAU,CAACI,gBAAgB,CAACN,aAAa,CAAC;EACjD;;EAEA;AACF;AACA;AACA;AACA;AACA;EACEO,SAASA,CAAC;IAAEC,UAAU;IAAEC,WAAW;IAAEC;EAA4B,CAAC,EAAE;IAClE,IAAI,CAACR,UAAU,CAACK,SAAS,CAAC;MAAEC,UAAU;MAAEC,WAAW;MAAEC;IAAiB,CAAC,CAAC;EAC1E;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACE,MAAMC,OAAOA,CAACC,KAAa,EAAmB;IAC5C,OAAO,MAAM,IAAI,CAACV,UAAU,CAACS,OAAO,CAACC,KAAK,CAAC;EAC7C;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;EACE,MAAMC,QAAQA,CAACC,QAAmB,EAAEC,KAAiB,EAAmB;IACtE,OAAO,MAAM,IAAI,CAACb,UAAU,CAACW,QAAQ,CAACC,QAAQ,EAAEC,KAAK,CAAC;EACxD;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;EACE,MAAMC,WAAWA,CAACC,OAAe,EAAsB;IACrD,MAAM,IAAI,CAACf,UAAU,CAACc,WAAW,CAACC,OAAO,CAAC;IAC1C,OAAO,IAAI,CAACf,UAAU,CAACgB,cAAc;EACvC;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;EACEC,aAAaA,CAACC,KAAa,EAAa;IACtC,IAAI,CAAClB,UAAU,CAACiB,aAAa,CAACC,KAAK,CAAC;IACpC,OAAO,IAAI,CAAClB,UAAU,CAACgB,cAAc;EACvC;;EAEA;AACF;AACA;EACEG,SAASA,CAAA,EAAG;IACV,IAAI,CAACnB,UAAU,CAACmB,SAAS,CAAC,CAAC;EAC7B;;EAEA;AACF;AACA;AACA;AACA;EACEC,sBAAsBA,CAAA,EAAW;IAC/B,OAAO,IAAI,CAACpB,UAAU,CAACoB,sBAAsB,CAAC,CAAC;EACjD;;EAEA;AACF;AACA;AACA;AACA;EACEC,oBAAoBA,CAAA,EAAG;IACrB,OAAO,IAAI,CAACrB,UAAU,CAACsB,mBAAmB,CAAC,CAAC;EAC9C;;EAEA;AACF;AACA;AACA;AACA;EACEC,mBAAmBA,CAAA,EAAG;IACpB,OAAO,IAAI,CAACvB,UAAU,CAACwB,kBAAkB,CAAC,CAAC;EAC7C;;EAEA;AACF;AACA;AACA;AACA;EACEC,MAAMA,CAAA,EAAG;IACP,IAAI,CAACzB,UAAU,CAACyB,MAAM,CAAC,CAAC;EAC1B;AACF","ignoreList":[]}
|
|
@@ -1,14 +1,27 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
-
import { Logger } from '../../common/Logger';
|
|
4
3
|
import { ResourceFetcher } from '../../utils/ResourceFetcher';
|
|
5
4
|
import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
|
|
6
5
|
import { RnExecutorchError, parseUnknownError } from '../../errors/errorUtils';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Module for Speech to Text (STT) functionalities.
|
|
9
|
+
*
|
|
10
|
+
* @category Typescript API
|
|
11
|
+
*/
|
|
7
12
|
export class SpeechToTextModule {
|
|
8
13
|
textDecoder = new TextDecoder('utf-8', {
|
|
9
14
|
fatal: false,
|
|
10
15
|
ignoreBOM: true
|
|
11
16
|
});
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Loads the model specified by the config object.
|
|
20
|
+
* `onDownloadProgressCallback` allows you to monitor the current progress of the model download.
|
|
21
|
+
*
|
|
22
|
+
* @param model - Configuration object containing model sources.
|
|
23
|
+
* @param onDownloadProgressCallback - Optional callback to monitor download progress.
|
|
24
|
+
*/
|
|
12
25
|
async load(model, onDownloadProgressCallback = () => {}) {
|
|
13
26
|
this.modelConfig = model;
|
|
14
27
|
const tokenizerLoadPromise = ResourceFetcher.fetch(undefined, model.tokenizerSource);
|
|
@@ -21,36 +34,63 @@ export class SpeechToTextModule {
|
|
|
21
34
|
}
|
|
22
35
|
this.nativeModule = await global.loadSpeechToText(encoderSource, decoderSource, tokenizerSources[0]);
|
|
23
36
|
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Unloads the model from memory.
|
|
40
|
+
*/
|
|
24
41
|
delete() {
|
|
25
42
|
this.nativeModule.unload();
|
|
26
43
|
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Runs the encoding part of the model on the provided waveform.
|
|
47
|
+
* Returns the encoded waveform as a Float32Array.
|
|
48
|
+
*
|
|
49
|
+
* @param waveform - The input audio waveform.
|
|
50
|
+
* @returns The encoded output.
|
|
51
|
+
*/
|
|
27
52
|
async encode(waveform) {
|
|
28
|
-
if (Array.isArray(waveform)) {
|
|
29
|
-
Logger.info('Passing waveform as number[] is deprecated, use Float32Array instead');
|
|
30
|
-
waveform = new Float32Array(waveform);
|
|
31
|
-
}
|
|
32
53
|
return new Float32Array(await this.nativeModule.encode(waveform));
|
|
33
54
|
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Runs the decoder of the model.
|
|
58
|
+
*
|
|
59
|
+
* @param tokens - The input tokens.
|
|
60
|
+
* @param encoderOutput - The encoder output.
|
|
61
|
+
* @returns Decoded output.
|
|
62
|
+
*/
|
|
34
63
|
async decode(tokens, encoderOutput) {
|
|
35
|
-
if (Array.isArray(tokens)) {
|
|
36
|
-
Logger.info('Passing tokens as number[] is deprecated, use Int32Array instead');
|
|
37
|
-
tokens = new Int32Array(tokens);
|
|
38
|
-
}
|
|
39
|
-
if (Array.isArray(encoderOutput)) {
|
|
40
|
-
Logger.info('Passing encoderOutput as number[] is deprecated, use Float32Array instead');
|
|
41
|
-
encoderOutput = new Float32Array(encoderOutput);
|
|
42
|
-
}
|
|
43
64
|
return new Float32Array(await this.nativeModule.decode(tokens, encoderOutput));
|
|
44
65
|
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Starts a transcription process for a given input array (16kHz waveform).
|
|
69
|
+
* For multilingual models, specify the language in `options`.
|
|
70
|
+
* Returns the transcription as a string. Passing `number[]` is deprecated.
|
|
71
|
+
*
|
|
72
|
+
* @param waveform - The Float32Array audio data.
|
|
73
|
+
* @param options - Decoding options including language.
|
|
74
|
+
* @returns The transcription string.
|
|
75
|
+
*/
|
|
45
76
|
async transcribe(waveform, options = {}) {
|
|
46
77
|
this.validateOptions(options);
|
|
47
|
-
if (Array.isArray(waveform)) {
|
|
48
|
-
Logger.info('Passing waveform as number[] is deprecated, use Float32Array instead');
|
|
49
|
-
waveform = new Float32Array(waveform);
|
|
50
|
-
}
|
|
51
78
|
const transcriptionBytes = await this.nativeModule.transcribe(waveform, options.language || '');
|
|
52
79
|
return this.textDecoder.decode(new Uint8Array(transcriptionBytes));
|
|
53
80
|
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Starts a streaming transcription session.
|
|
84
|
+
* Yields objects with `committed` and `nonCommitted` transcriptions.
|
|
85
|
+
* Committed transcription contains the part of the transcription that is finalized and will not change.
|
|
86
|
+
* Useful for displaying stable results during streaming.
|
|
87
|
+
* Non-committed transcription contains the part of the transcription that is still being processed and may change.
|
|
88
|
+
* Useful for displaying live, partial results during streaming.
|
|
89
|
+
* Use with `streamInsert` and `streamStop` to control the stream.
|
|
90
|
+
*
|
|
91
|
+
* @param options - Decoding options including language.
|
|
92
|
+
* @returns An async generator yielding transcription updates.
|
|
93
|
+
*/
|
|
54
94
|
async *stream(options = {}) {
|
|
55
95
|
this.validateOptions(options);
|
|
56
96
|
const queue = [];
|
|
@@ -94,13 +134,19 @@ export class SpeechToTextModule {
|
|
|
94
134
|
await new Promise(r => waiter = r);
|
|
95
135
|
}
|
|
96
136
|
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Inserts a new audio chunk into the streaming transcription session.
|
|
140
|
+
*
|
|
141
|
+
* @param waveform - The audio chunk to insert.
|
|
142
|
+
*/
|
|
97
143
|
streamInsert(waveform) {
|
|
98
|
-
if (Array.isArray(waveform)) {
|
|
99
|
-
Logger.info('Passing waveform as number[] is deprecated, use Float32Array instead');
|
|
100
|
-
waveform = new Float32Array(waveform);
|
|
101
|
-
}
|
|
102
144
|
this.nativeModule.streamInsert(waveform);
|
|
103
145
|
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Stops the current streaming transcription session.
|
|
149
|
+
*/
|
|
104
150
|
streamStop() {
|
|
105
151
|
this.nativeModule.streamStop();
|
|
106
152
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["
|
|
1
|
+
{"version":3,"names":["ResourceFetcher","RnExecutorchErrorCode","RnExecutorchError","parseUnknownError","SpeechToTextModule","textDecoder","TextDecoder","fatal","ignoreBOM","load","model","onDownloadProgressCallback","modelConfig","tokenizerLoadPromise","fetch","undefined","tokenizerSource","encoderDecoderPromise","encoderSource","decoderSource","tokenizerSources","encoderDecoderResults","Promise","all","DownloadInterrupted","nativeModule","global","loadSpeechToText","delete","unload","encode","waveform","Float32Array","decode","tokens","encoderOutput","transcribe","options","validateOptions","transcriptionBytes","language","Uint8Array","stream","queue","waiter","finished","error","wake","committed","nonCommitted","isDone","push","e","length","shift","r","streamInsert","streamStop","isMultilingual","InvalidConfig"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/SpeechToTextModule.ts"],"mappings":";;AACA,SAASA,eAAe,QAAQ,6BAA6B;AAC7D,SAASC,qBAAqB,QAAQ,yBAAyB;AAC/D,SAASC,iBAAiB,EAAEC,iBAAiB,QAAQ,yBAAyB;;AAE9E;AACA;AACA;AACA;AACA;AACA,OAAO,MAAMC,kBAAkB,CAAC;EAKtBC,WAAW,GAAG,IAAIC,WAAW,CAAC,OAAO,EAAE;IAC7CC,KAAK,EAAE,KAAK;IACZC,SAAS,EAAE;EACb,CAAC,CAAC;;EAEF;AACF;AACA;AACA;AACA;AACA;AACA;EACE,MAAaC,IAAIA,CACfC,KAA8B,EAC9BC,0BAAsD,GAAGA,CAAA,KAAM,CAAC,CAAC,EACjE;IACA,IAAI,CAACC,WAAW,GAAGF,KAAK;IAExB,MAAMG,oBAAoB,GAAGb,eAAe,CAACc,KAAK,CAChDC,SAAS,EACTL,KAAK,CAACM,eACR,CAAC;IACD,MAAMC,qBAAqB,GAAGjB,eAAe,CAACc,KAAK,CACjDH,0BAA0B,EAC1BD,KAAK,CAACQ,aAAa,EACnBR,KAAK,CAACS,aACR,CAAC;IACD,MAAM,CAACC,gBAAgB,EAAEC,qBAAqB,CAAC,GAAG,MAAMC,OAAO,CAACC,GAAG,CAAC,CAClEV,oBAAoB,EACpBI,qBAAqB,CACtB,CAAC;IACF,MAAMC,aAAa,GAAGG,qBAAqB,GAAG,CAAC,CAAC;IAChD,MAAMF,aAAa,GAAGE,qBAAqB,GAAG,CAAC,CAAC;IAChD,IAAI,CAACH,aAAa,IAAI,CAACC,aAAa,IAAI,CAACC,gBAAgB,EAAE;MACzD,MAAM,IAAIlB,iBAAiB,CACzBD,qBAAqB,CAACuB,mBAAmB,EACzC,2GACF,CAAC;IACH;IACA,IAAI,CAACC,YAAY,GAAG,MAAMC,MAAM,CAACC,gBAAgB,CAC/CT,aAAa,EACbC,aAAa,EACbC,gBAAgB,CAAC,CAAC,CACpB,CAAC;EACH;;EAEA;AACF;AACA;EACSQ,MAAMA,CAAA,EAAS;IACpB,IAAI,CAACH,YAAY,CAACI,MAAM,CAAC,CAAC;EAC5B;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;EACE,MAAaC,MAAMA,CAACC,QAAsB,EAAyB;IACjE,OAAO,IAAIC,YAAY,CAAC,MAAM,IAAI,CAACP,YAAY,CAACK,MAAM,CAACC,QAAQ,CAAC,CAAC;EACnE;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;EACE,MAAaE,MAAMA,CACjBC,MAAkB,EAClBC,aAA2B,EACJ;IACvB,OAAO,IAAIH,YAAY,CACrB,MAAM,IAAI,CAACP,YAAY,CAACQ,MAAM,CAACC,MAAM,EAAEC,aAAa,CACtD,CAAC;EACH;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACE,MAAaC,UAAUA,CACrBL,QAAsB,EACtBM,OAAwB,GAAG,CAAC,CAAC,EACZ;IACjB,IAAI,CAACC,eAAe,CAACD,OAAO,CAAC;IAC7B,MAAME,kBAAkB,GAAG,MAAM,IAAI,CAACd,YAAY,CAACW,UAAU,CAC3DL,QAAQ,EACRM,OAAO,CAACG,QAAQ,IAAI,EACtB,CAAC;IACD,OAAO,IAAI,CAACnC,WAAW,CAAC4B,MAAM,CAAC,IAAIQ,UAAU,CAACF,kBAAkB,CAAC,CAAC;EACpE;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACE,OAAcG,MAAMA,CAClBL,OAAwB,GAAG,CAAC,CAAC,EACgC;IAC7D,IAAI,CAACC,eAAe,CAACD,OAAO,CAAC;IAE7B,MAAMM,KAAoD,GAAG,EAAE;IAC/D,IAAIC,MAA2B,GAAG,IAAI;IACtC,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,KAAc;IAElB,MAAMC,IAAI,GAAGA,CAAA,KAAM;MACjBH,MAAM,GAAG,CAAC;MACVA,MAAM,GAAG,IAAI;IACf,CAAC;IAED,CAAC,YAAY;MACX,IAAI;QACF,MAAM,IAAI,CAACnB,YAAY,CAACiB,MAAM,CAC5B,CAACM,SAAmB,EAAEC,YAAsB,EAAEC,MAAe,KAAK;UAChEP,KAAK,CAACQ,IAAI,CAAC;YACTH,SAAS,EAAE,IAAI,CAAC3C,WAAW,CAAC4B,MAAM,CAAC,IAAIQ,UAAU,CAACO,SAAS,CAAC,CAAC;YAC7DC,YAAY,EAAE,IAAI,CAAC5C,WAAW,CAAC4B,MAAM,CACnC,IAAIQ,UAAU,CAACQ,YAAY,CAC7B;UACF,CAAC,CAAC;UACF,IAAIC,MAAM,EAAE;YACVL,QAAQ,GAAG,IAAI;UACjB;UACAE,IAAI,CAAC,CAAC;QACR,CAAC,EACDV,OAAO,CAACG,QAAQ,IAAI,EACtB,CAAC;QACDK,QAAQ,GAAG,IAAI;QACfE,IAAI,CAAC,CAAC;MACR,CAAC,CAAC,OAAOK,CAAC,EAAE;QACVN,KAAK,GAAGM,CAAC;QACTP,QAAQ,GAAG,IAAI;QACfE,IAAI,CAAC,CAAC;MACR;IACF,CAAC,EAAE,CAAC;IAEJ,OAAO,IAAI,EAAE;MACX,IAAIJ,KAAK,CAACU,MAAM,GAAG,CAAC,EAAE;QACpB,MAAMV,KAAK,CAACW,KAAK,CAAC,CAAE;QACpB,IAAIT,QAAQ,IAAIF,KAAK,CAACU,MAAM,KAAK,CAAC,EAAE;UAClC;QACF;QACA;MACF;MACA,IAAIP,KAAK,EAAE,MAAM3C,iBAAiB,CAAC2C,KAAK,CAAC;MACzC,IAAID,QAAQ,EAAE;MACd,MAAM,IAAIvB,OAAO,CAAQiC,CAAC,IAAMX,MAAM,GAAGW,CAAE,CAAC;IAC9C;EACF;;EAEA;AACF;AACA;AACA;AACA;EACSC,YAAYA,CAACzB,QAAsB,EAAQ;IAChD,IAAI,CAACN,YAAY,CAAC+B,YAAY,CAACzB,QAAQ,CAAC;EAC1C;;EAEA;AACF;AACA;EACS0B,UAAUA,CAAA,EAAS;IACxB,IAAI,CAAChC,YAAY,CAACgC,UAAU,CAAC,CAAC;EAChC;EAEQnB,eAAeA,CAACD,OAAwB,EAAE;IAChD,IAAI,CAAC,IAAI,CAACzB,WAAW,CAAC8C,cAAc,IAAIrB,OAAO,CAACG,QAAQ,EAAE;MACxD,MAAM,IAAItC,iBAAiB,CACzBD,qBAAqB,CAAC0D,aAAa,EACnC,gDACF,CAAC;IACH;IACA,IAAI,IAAI,CAAC/C,WAAW,CAAC8C,cAAc,IAAI,CAACrB,OAAO,CAACG,QAAQ,EAAE;MACxD,MAAM,IAAItC,iBAAiB,CACzBD,qBAAqB,CAAC0D,aAAa,EACnC,2CACF,CAAC;IACH;EACF;AACF","ignoreList":[]}
|
|
@@ -4,7 +4,21 @@ import { ResourceFetcher } from '../../utils/ResourceFetcher';
|
|
|
4
4
|
import { BaseModule } from '../BaseModule';
|
|
5
5
|
import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
|
|
6
6
|
import { RnExecutorchError } from '../../errors/errorUtils';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Module for generating text embeddings from input text.
|
|
10
|
+
*
|
|
11
|
+
* @category Typescript API
|
|
12
|
+
*/
|
|
7
13
|
export class TextEmbeddingsModule extends BaseModule {
|
|
14
|
+
/**
|
|
15
|
+
* Loads the model and tokenizer specified by the config object.
|
|
16
|
+
*
|
|
17
|
+
* @param model - Object containing model and tokenizer sources.
|
|
18
|
+
* @param model.modelSource - `ResourceSource` that specifies the location of the text embeddings model binary.
|
|
19
|
+
* @param model.tokenizerSource - `ResourceSource` that specifies the location of the tokenizer JSON file.
|
|
20
|
+
* @param onDownloadProgressCallback - Optional callback to track download progress (value between 0 and 1).
|
|
21
|
+
*/
|
|
8
22
|
async load(model, onDownloadProgressCallback = () => {}) {
|
|
9
23
|
const modelPromise = ResourceFetcher.fetch(onDownloadProgressCallback, model.modelSource);
|
|
10
24
|
const tokenizerPromise = ResourceFetcher.fetch(undefined, model.tokenizerSource);
|
|
@@ -16,6 +30,13 @@ export class TextEmbeddingsModule extends BaseModule {
|
|
|
16
30
|
}
|
|
17
31
|
this.nativeModule = global.loadTextEmbeddings(modelPath, tokenizerPath);
|
|
18
32
|
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Executes the model's forward pass, where `input` is a text that will be embedded.
|
|
36
|
+
*
|
|
37
|
+
* @param input - The text string to embed.
|
|
38
|
+
* @returns A Float32Array containing the vector embeddings.
|
|
39
|
+
*/
|
|
19
40
|
async forward(input) {
|
|
20
41
|
return new Float32Array(await this.nativeModule.generate(input));
|
|
21
42
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["ResourceFetcher","BaseModule","RnExecutorchErrorCode","RnExecutorchError","TextEmbeddingsModule","load","model","onDownloadProgressCallback","modelPromise","fetch","modelSource","tokenizerPromise","undefined","tokenizerSource","modelResult","tokenizerResult","Promise","all","modelPath","tokenizerPath","DownloadInterrupted","nativeModule","global","loadTextEmbeddings","forward","input","Float32Array","generate"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/TextEmbeddingsModule.ts"],"mappings":";;AACA,SAASA,eAAe,QAAQ,6BAA6B;AAC7D,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,qBAAqB,QAAQ,yBAAyB;AAC/D,SAASC,iBAAiB,QAAQ,yBAAyB;
|
|
1
|
+
{"version":3,"names":["ResourceFetcher","BaseModule","RnExecutorchErrorCode","RnExecutorchError","TextEmbeddingsModule","load","model","onDownloadProgressCallback","modelPromise","fetch","modelSource","tokenizerPromise","undefined","tokenizerSource","modelResult","tokenizerResult","Promise","all","modelPath","tokenizerPath","DownloadInterrupted","nativeModule","global","loadTextEmbeddings","forward","input","Float32Array","generate"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/TextEmbeddingsModule.ts"],"mappings":";;AACA,SAASA,eAAe,QAAQ,6BAA6B;AAC7D,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,qBAAqB,QAAQ,yBAAyB;AAC/D,SAASC,iBAAiB,QAAQ,yBAAyB;;AAE3D;AACA;AACA;AACA;AACA;AACA,OAAO,MAAMC,oBAAoB,SAASH,UAAU,CAAC;EACnD;AACF;AACA;AACA;AACA;AACA;AACA;AACA;EACE,MAAMI,IAAIA,CACRC,KAAuE,EACvEC,0BAAsD,GAAGA,CAAA,KAAM,CAAC,CAAC,EAClD;IACf,MAAMC,YAAY,GAAGR,eAAe,CAACS,KAAK,CACxCF,0BAA0B,EAC1BD,KAAK,CAACI,WACR,CAAC;IACD,MAAMC,gBAAgB,GAAGX,eAAe,CAACS,KAAK,CAC5CG,SAAS,EACTN,KAAK,CAACO,eACR,CAAC;IACD,MAAM,CAACC,WAAW,EAAEC,eAAe,CAAC,GAAG,MAAMC,OAAO,CAACC,GAAG,CAAC,CACvDT,YAAY,EACZG,gBAAgB,CACjB,CAAC;IACF,MAAMO,SAAS,GAAGJ,WAAW,GAAG,CAAC,CAAC;IAClC,MAAMK,aAAa,GAAGJ,eAAe,GAAG,CAAC,CAAC;IAC1C,IAAI,CAACG,SAAS,IAAI,CAACC,aAAa,EAAE;MAChC,MAAM,IAAIhB,iBAAiB,CACzBD,qBAAqB,CAACkB,mBAAmB,EACzC,2GACF,CAAC;IACH;IACA,IAAI,CAACC,YAAY,GAAGC,MAAM,CAACC,kBAAkB,CAACL,SAAS,EAAEC,aAAa,CAAC;EACzE;;EAEA;AACF;AACA;AACA;AACA;AACA;EACE,MAAMK,OAAOA,CAACC,KAAa,EAAyB;IAClD,OAAO,IAAIC,YAAY,CAAC,MAAM,IAAI,CAACL,YAAY,CAACM,QAAQ,CAACF,KAAK,CAAC,CAAC;EAClE;AACF","ignoreList":[]}
|
|
@@ -3,8 +3,24 @@
|
|
|
3
3
|
import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
|
|
4
4
|
import { RnExecutorchError } from '../../errors/errorUtils';
|
|
5
5
|
import { ResourceFetcher } from '../../utils/ResourceFetcher';
|
|
6
|
+
/**
|
|
7
|
+
* Module for Text to Speech (TTS) functionalities.
|
|
8
|
+
*
|
|
9
|
+
* @category Typescript API
|
|
10
|
+
*/
|
|
6
11
|
export class TextToSpeechModule {
|
|
12
|
+
/**
|
|
13
|
+
* Native module instance
|
|
14
|
+
*/
|
|
7
15
|
nativeModule = null;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Loads the model and voice assets specified by the config object.
|
|
19
|
+
* `onDownloadProgressCallback` allows you to monitor the current progress.
|
|
20
|
+
*
|
|
21
|
+
* @param config - Configuration object containing `model` source and `voice`.
|
|
22
|
+
* @param onDownloadProgressCallback - Optional callback to monitor download progress.
|
|
23
|
+
*/
|
|
8
24
|
async load(config, onDownloadProgressCallback = () => {}) {
|
|
9
25
|
// Select the text to speech model based on it's fixed identifier
|
|
10
26
|
if (config.model.type === 'kokoro') {
|
|
@@ -27,10 +43,26 @@ export class TextToSpeechModule {
|
|
|
27
43
|
const phonemizerPaths = paths.slice(3, 5);
|
|
28
44
|
this.nativeModule = global.loadTextToSpeechKokoro(voice.lang, phonemizerPaths[0], phonemizerPaths[1], modelPaths[0], modelPaths[1], voiceDataPath);
|
|
29
45
|
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Synthesizes the provided text into speech.
|
|
49
|
+
* Returns a promise that resolves to the full audio waveform as a `Float32Array`.
|
|
50
|
+
*
|
|
51
|
+
* @param text The input text to be synthesized.
|
|
52
|
+
* @param speed Optional speed multiplier for the speech synthesis (default is 1.0).
|
|
53
|
+
* @returns A promise resolving to the synthesized audio waveform.
|
|
54
|
+
*/
|
|
30
55
|
async forward(text, speed = 1.0) {
|
|
31
56
|
if (this.nativeModule == null) throw new RnExecutorchError(RnExecutorchErrorCode.ModuleNotLoaded, 'The model is currently not loaded. Please load the model before calling forward().');
|
|
32
57
|
return await this.nativeModule.generate(text, speed);
|
|
33
58
|
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Starts a streaming synthesis session. Yields audio chunks as they are generated.
|
|
62
|
+
*
|
|
63
|
+
* @param input - Input object containing text and optional speed.
|
|
64
|
+
* @returns An async generator yielding Float32Array audio chunks.
|
|
65
|
+
*/
|
|
34
66
|
async *stream({
|
|
35
67
|
text,
|
|
36
68
|
speed
|
|
@@ -71,9 +103,17 @@ export class TextToSpeechModule {
|
|
|
71
103
|
await new Promise(r => waiter = r);
|
|
72
104
|
}
|
|
73
105
|
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Stops the streaming process if there is any ongoing.
|
|
109
|
+
*/
|
|
74
110
|
streamStop() {
|
|
75
111
|
this.nativeModule.streamStop();
|
|
76
112
|
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Unloads the model from memory.
|
|
116
|
+
*/
|
|
77
117
|
delete() {
|
|
78
118
|
if (this.nativeModule !== null) {
|
|
79
119
|
this.nativeModule.unload();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["RnExecutorchErrorCode","RnExecutorchError","ResourceFetcher","TextToSpeechModule","nativeModule","load","config","onDownloadProgressCallback","model","type","loadKokoro","voice","extra","taggerSource","lexiconSource","InvalidConfig","paths","fetch","durationPredictorSource","synthesizerSource","voiceSource","length","some","p","DownloadInterrupted","modelPaths","slice","voiceDataPath","phonemizerPaths","global","loadTextToSpeechKokoro","lang","forward","text","speed","ModuleNotLoaded","generate","stream","queue","waiter","finished","error","wake","audio","push","Float32Array","e","shift","Promise","r","streamStop","delete","unload"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/TextToSpeechModule.ts"],"mappings":";;AAAA,SAASA,qBAAqB,QAAQ,yBAAyB;AAC/D,SAASC,iBAAiB,QAAQ,yBAAyB;AAC3D,SAASC,eAAe,QAAQ,6BAA6B;AAQ7D,OAAO,MAAMC,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"names":["RnExecutorchErrorCode","RnExecutorchError","ResourceFetcher","TextToSpeechModule","nativeModule","load","config","onDownloadProgressCallback","model","type","loadKokoro","voice","extra","taggerSource","lexiconSource","InvalidConfig","paths","fetch","durationPredictorSource","synthesizerSource","voiceSource","length","some","p","DownloadInterrupted","modelPaths","slice","voiceDataPath","phonemizerPaths","global","loadTextToSpeechKokoro","lang","forward","text","speed","ModuleNotLoaded","generate","stream","queue","waiter","finished","error","wake","audio","push","Float32Array","e","shift","Promise","r","streamStop","delete","unload"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/TextToSpeechModule.ts"],"mappings":";;AAAA,SAASA,qBAAqB,QAAQ,yBAAyB;AAC/D,SAASC,iBAAiB,QAAQ,yBAAyB;AAC3D,SAASC,eAAe,QAAQ,6BAA6B;AAQ7D;AACA;AACA;AACA;AACA;AACA,OAAO,MAAMC,kBAAkB,CAAC;EAC9B;AACF;AACA;EACEC,YAAY,GAAQ,IAAI;;EAExB;AACF;AACA;AACA;AACA;AACA;AACA;EACE,MAAaC,IAAIA,CACfC,MAA0B,EAC1BC,0BAAsD,GAAGA,CAAA,KAAM,CAAC,CAAC,EAClD;IACf;IACA,IAAID,MAAM,CAACE,KAAK,CAACC,IAAI,KAAK,QAAQ,EAAE;MAClC,MAAM,IAAI,CAACC,UAAU,CACnBJ,MAAM,CAACE,KAAK,EACZF,MAAM,CAACK,KAAK,EACZJ,0BACF,CAAC;IACH;IACA;EACF;;EAEA;EACA,MAAcG,UAAUA,CACtBF,KAAmB,EACnBG,KAAkB,EAClBJ,0BAAsD,EACvC;IACf,IACE,CAACI,KAAK,CAACC,KAAK,IACZ,CAACD,KAAK,CAACC,KAAK,CAACC,YAAY,IACzB,CAACF,KAAK,CAACC,KAAK,CAACE,aAAa,EAC1B;MACA,MAAM,IAAIb,iBAAiB,CACzBD,qBAAqB,CAACe,aAAa,EACnC,2FACF,CAAC;IACH;IAEA,MAAMC,KAAK,GAAG,MAAMd,eAAe,CAACe,KAAK,CACvCV,0BAA0B,EAC1BC,KAAK,CAACU,uBAAuB,EAC7BV,KAAK,CAACW,iBAAiB,EACvBR,KAAK,CAACS,WAAW,EACjBT,KAAK,CAACC,KAAK,CAACC,YAAY,EACxBF,KAAK,CAACC,KAAK,CAACE,aACd,CAAC;IAED,IAAIE,KAAK,KAAK,IAAI,IAAIA,KAAK,CAACK,MAAM,KAAK,CAAC,IAAIL,KAAK,CAACM,IAAI,CAAEC,CAAC,IAAKA,CAAC,IAAI,IAAI,CAAC,EAAE;MACxE,MAAM,IAAItB,iBAAiB,CACzBD,qBAAqB,CAACwB,mBAAmB,EACzC,2CACF,CAAC;IACH;IAEA,MAAMC,UAAU,GAAGT,KAAK,CAACU,KAAK,CAAC,CAAC,EAAE,CAAC,CAAqC;IACxE,MAAMC,aAAa,GAAGX,KAAK,CAAC,CAAC,CAAW;IACxC,MAAMY,eAAe,GAAGZ,KAAK,CAACU,KAAK,CAAC,CAAC,EAAE,CAAC,CAAqB;IAE7D,IAAI,CAACtB,YAAY,GAAGyB,MAAM,CAACC,sBAAsB,CAC/CnB,KAAK,CAACoB,IAAI,EACVH,eAAe,CAAC,CAAC,CAAC,EAClBA,eAAe,CAAC,CAAC,CAAC,EAClBH,UAAU,CAAC,CAAC,CAAC,EACbA,UAAU,CAAC,CAAC,CAAC,EACbE,aACF,CAAC;EACH;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;EACE,MAAaK,OAAOA,CAClBC,IAAY,EACZC,KAAa,GAAG,GAAG,EACI;IACvB,IAAI,IAAI,CAAC9B,YAAY,IAAI,IAAI,EAC3B,MAAM,IAAIH,iBAAiB,CACzBD,qBAAqB,CAACmC,eAAe,EACrC,oFACF,CAAC;IACH,OAAO,MAAM,IAAI,CAAC/B,YAAY,CAACgC,QAAQ,CAACH,IAAI,EAAEC,KAAK,CAAC;EACtD;;EAEA;AACF;AACA;AACA;AACA;AACA;EACE,OAAcG,MAAMA,CAAC;IACnBJ,IAAI;IACJC;EAC0B,CAAC,EAAgC;IAC3D;IACA,MAAMI,KAAqB,GAAG,EAAE;IAEhC,IAAIC,MAA2B,GAAG,IAAI;IACtC,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,KAAc;IAElB,MAAMC,IAAI,GAAGA,CAAA,KAAM;MACjBH,MAAM,GAAG,CAAC;MACVA,MAAM,GAAG,IAAI;IACf,CAAC;IAED,CAAC,YAAY;MACX,IAAI;QACF,MAAM,IAAI,CAACnC,YAAY,CAACiC,MAAM,CAACJ,IAAI,EAAEC,KAAK,EAAGS,KAAe,IAAK;UAC/DL,KAAK,CAACM,IAAI,CAAC,IAAIC,YAAY,CAACF,KAAK,CAAC,CAAC;UACnCD,IAAI,CAAC,CAAC;QACR,CAAC,CAAC;QACFF,QAAQ,GAAG,IAAI;QACfE,IAAI,CAAC,CAAC;MACR,CAAC,CAAC,OAAOI,CAAC,EAAE;QACVL,KAAK,GAAGK,CAAC;QACTN,QAAQ,GAAG,IAAI;QACfE,IAAI,CAAC,CAAC;MACR;IACF,CAAC,EAAE,CAAC;IAEJ,OAAO,IAAI,EAAE;MACX,IAAIJ,KAAK,CAACjB,MAAM,GAAG,CAAC,EAAE;QACpB,MAAMiB,KAAK,CAACS,KAAK,CAAC,CAAE;QACpB,IAAIP,QAAQ,IAAIF,KAAK,CAACjB,MAAM,KAAK,CAAC,EAAE;UAClC;QACF;QACA;MACF;MACA,IAAIoB,KAAK,EAAE,MAAMA,KAAK;MACtB,IAAID,QAAQ,EAAE;MACd,MAAM,IAAIQ,OAAO,CAAQC,CAAC,IAAMV,MAAM,GAAGU,CAAE,CAAC;IAC9C;EACF;;EAEA;AACF;AACA;EACSC,UAAUA,CAAA,EAAS;IACxB,IAAI,CAAC9C,YAAY,CAAC8C,UAAU,CAAC,CAAC;EAChC;;EAEA;AACF;AACA;EACEC,MAAMA,CAAA,EAAG;IACP,IAAI,IAAI,CAAC/C,YAAY,KAAK,IAAI,EAAE;MAC9B,IAAI,CAACA,YAAY,CAACgD,MAAM,CAAC,CAAC;IAC5B;EACF;AACF","ignoreList":[]}
|
|
@@ -3,7 +3,24 @@
|
|
|
3
3
|
import { ResourceFetcher } from '../../utils/ResourceFetcher';
|
|
4
4
|
import { RnExecutorchError } from '../../errors/errorUtils';
|
|
5
5
|
import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Module for Tokenizer functionalities.
|
|
9
|
+
*
|
|
10
|
+
* @category Typescript API
|
|
11
|
+
*/
|
|
6
12
|
export class TokenizerModule {
|
|
13
|
+
/**
|
|
14
|
+
* Native module instance
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Loads the tokenizer from the specified source.
|
|
19
|
+
* `tokenizerSource` is a string that points to the location of the tokenizer JSON file.
|
|
20
|
+
*
|
|
21
|
+
* @param tokenizer - Object containing `tokenizerSource`.
|
|
22
|
+
* @param onDownloadProgressCallback - Optional callback to monitor download progress.
|
|
23
|
+
*/
|
|
7
24
|
async load(tokenizer, onDownloadProgressCallback = () => {}) {
|
|
8
25
|
const paths = await ResourceFetcher.fetch(onDownloadProgressCallback, tokenizer.tokenizerSource);
|
|
9
26
|
const path = paths?.[0];
|
|
@@ -12,21 +29,56 @@ export class TokenizerModule {
|
|
|
12
29
|
}
|
|
13
30
|
this.nativeModule = global.loadTokenizerModule(path);
|
|
14
31
|
}
|
|
15
|
-
|
|
16
|
-
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Converts a string into an array of token IDs.
|
|
35
|
+
*
|
|
36
|
+
* @param input - The input string to be tokenized.
|
|
37
|
+
* @returns An array of token IDs.
|
|
38
|
+
*/
|
|
39
|
+
async encode(input) {
|
|
40
|
+
return await this.nativeModule.encode(input);
|
|
17
41
|
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Converts an array of token IDs into a string.
|
|
45
|
+
*
|
|
46
|
+
* @param tokens - Array of token IDs to be decoded.
|
|
47
|
+
* @param skipSpecialTokens - Whether to skip special tokens during decoding (default: true).
|
|
48
|
+
* @returns The decoded string.
|
|
49
|
+
*/
|
|
18
50
|
async decode(tokens, skipSpecialTokens = true) {
|
|
19
51
|
if (tokens.length === 0) {
|
|
20
52
|
return '';
|
|
21
53
|
}
|
|
22
54
|
return await this.nativeModule.decode(tokens, skipSpecialTokens);
|
|
23
55
|
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Returns the size of the tokenizer's vocabulary.
|
|
59
|
+
*
|
|
60
|
+
* @returns The vocabulary size.
|
|
61
|
+
*/
|
|
24
62
|
async getVocabSize() {
|
|
25
63
|
return await this.nativeModule.getVocabSize();
|
|
26
64
|
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Returns the token associated to the ID.
|
|
68
|
+
*
|
|
69
|
+
* @param tokenId - ID of the token.
|
|
70
|
+
* @returns The token string associated to ID.
|
|
71
|
+
*/
|
|
27
72
|
async idToToken(tokenId) {
|
|
28
73
|
return this.nativeModule.idToToken(tokenId);
|
|
29
74
|
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Returns the ID associated to the token.
|
|
78
|
+
*
|
|
79
|
+
* @param token - The token string.
|
|
80
|
+
* @returns The ID associated to the token.
|
|
81
|
+
*/
|
|
30
82
|
async tokenToId(token) {
|
|
31
83
|
return await this.nativeModule.tokenToId(token);
|
|
32
84
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["ResourceFetcher","RnExecutorchError","RnExecutorchErrorCode","TokenizerModule","load","tokenizer","onDownloadProgressCallback","paths","fetch","tokenizerSource","path","DownloadInterrupted","nativeModule","global","loadTokenizerModule","encode","
|
|
1
|
+
{"version":3,"names":["ResourceFetcher","RnExecutorchError","RnExecutorchErrorCode","TokenizerModule","load","tokenizer","onDownloadProgressCallback","paths","fetch","tokenizerSource","path","DownloadInterrupted","nativeModule","global","loadTokenizerModule","encode","input","decode","tokens","skipSpecialTokens","length","getVocabSize","idToToken","tokenId","tokenToId","token"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/TokenizerModule.ts"],"mappings":";;AACA,SAASA,eAAe,QAAQ,6BAA6B;AAC7D,SAASC,iBAAiB,QAAQ,yBAAyB;AAC3D,SAASC,qBAAqB,QAAQ,yBAAyB;;AAE/D;AACA;AACA;AACA;AACA;AACA,OAAO,MAAMC,eAAe,CAAC;EAC3B;AACF;AACA;;EAGE;AACF;AACA;AACA;AACA;AACA;AACA;EACE,MAAMC,IAAIA,CACRC,SAA8C,EAC9CC,0BAAsD,GAAGA,CAAA,KAAM,CAAC,CAAC,EAClD;IACf,MAAMC,KAAK,GAAG,MAAMP,eAAe,CAACQ,KAAK,CACvCF,0BAA0B,EAC1BD,SAAS,CAACI,eACZ,CAAC;IACD,MAAMC,IAAI,GAAGH,KAAK,GAAG,CAAC,CAAC;IACvB,IAAI,CAACG,IAAI,EAAE;MACT,MAAM,IAAIT,iBAAiB,CACzBC,qBAAqB,CAACS,mBAAmB,EACzC,2GACF,CAAC;IACH;IACA,IAAI,CAACC,YAAY,GAAGC,MAAM,CAACC,mBAAmB,CAACJ,IAAI,CAAC;EACtD;;EAEA;AACF;AACA;AACA;AACA;AACA;EACE,MAAMK,MAAMA,CAACC,KAAa,EAAqB;IAC7C,OAAO,MAAM,IAAI,CAACJ,YAAY,CAACG,MAAM,CAACC,KAAK,CAAC;EAC9C;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;EACE,MAAMC,MAAMA,CACVC,MAAgB,EAChBC,iBAA0B,GAAG,IAAI,EAChB;IACjB,IAAID,MAAM,CAACE,MAAM,KAAK,CAAC,EAAE;MACvB,OAAO,EAAE;IACX;IACA,OAAO,MAAM,IAAI,CAACR,YAAY,CAACK,MAAM,CAACC,MAAM,EAAEC,iBAAiB,CAAC;EAClE;;EAEA;AACF;AACA;AACA;AACA;EACE,MAAME,YAAYA,CAAA,EAAoB;IACpC,OAAO,MAAM,IAAI,CAACT,YAAY,CAACS,YAAY,CAAC,CAAC;EAC/C;;EAEA;AACF;AACA;AACA;AACA;AACA;EACE,MAAMC,SAASA,CAACC,OAAe,EAAmB;IAChD,OAAO,IAAI,CAACX,YAAY,CAACU,SAAS,CAACC,OAAO,CAAC;EAC7C;;EAEA;AACF;AACA;AACA;AACA;AACA;EACE,MAAMC,SAASA,CAACC,KAAa,EAAmB;IAC9C,OAAO,MAAM,IAAI,CAACb,YAAY,CAACY,SAAS,CAACC,KAAK,CAAC;EACjD;AACF","ignoreList":[]}
|
|
@@ -4,7 +4,20 @@ import { ResourceFetcher } from '../../utils/ResourceFetcher';
|
|
|
4
4
|
import { BaseModule } from '../BaseModule';
|
|
5
5
|
import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
|
|
6
6
|
import { RnExecutorchError } from '../../errors/errorUtils';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Module for Voice Activity Detection (VAD) functionalities.
|
|
10
|
+
*
|
|
11
|
+
* @category Typescript API
|
|
12
|
+
*/
|
|
7
13
|
export class VADModule extends BaseModule {
|
|
14
|
+
/**
|
|
15
|
+
* Loads the model, where `modelSource` is a string that specifies the location of the model binary.
|
|
16
|
+
* To track the download progress, supply a callback function `onDownloadProgressCallback`.
|
|
17
|
+
*
|
|
18
|
+
* @param model - Object containing `modelSource`.
|
|
19
|
+
* @param onDownloadProgressCallback - Optional callback to monitor download progress.
|
|
20
|
+
*/
|
|
8
21
|
async load(model, onDownloadProgressCallback = () => {}) {
|
|
9
22
|
const paths = await ResourceFetcher.fetch(onDownloadProgressCallback, model.modelSource);
|
|
10
23
|
if (paths === null || paths.length < 1) {
|
|
@@ -12,6 +25,13 @@ export class VADModule extends BaseModule {
|
|
|
12
25
|
}
|
|
13
26
|
this.nativeModule = global.loadVAD(paths[0] || '');
|
|
14
27
|
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Executes the model's forward pass, where `waveform` is a Float32Array representing the audio signal (16kHz).
|
|
31
|
+
*
|
|
32
|
+
* @param waveform - The input audio waveform as a Float32Array. It must represent a mono audio signal sampled at 16kHz.
|
|
33
|
+
* @returns A promise resolving to an array of detected speech segments.
|
|
34
|
+
*/
|
|
15
35
|
async forward(waveform) {
|
|
16
36
|
if (this.nativeModule == null) throw new RnExecutorchError(RnExecutorchErrorCode.ModuleNotLoaded, 'The model is currently not loaded. Please load the model before calling forward().');
|
|
17
37
|
return await this.nativeModule.generate(waveform);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["ResourceFetcher","BaseModule","RnExecutorchErrorCode","RnExecutorchError","VADModule","load","model","onDownloadProgressCallback","paths","fetch","modelSource","length","DownloadInterrupted","nativeModule","global","loadVAD","forward","waveform","ModuleNotLoaded","generate"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/VADModule.ts"],"mappings":";;AAAA,SAASA,eAAe,QAAQ,6BAA6B;AAG7D,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,qBAAqB,QAAQ,yBAAyB;AAC/D,SAASC,iBAAiB,QAAQ,yBAAyB;
|
|
1
|
+
{"version":3,"names":["ResourceFetcher","BaseModule","RnExecutorchErrorCode","RnExecutorchError","VADModule","load","model","onDownloadProgressCallback","paths","fetch","modelSource","length","DownloadInterrupted","nativeModule","global","loadVAD","forward","waveform","ModuleNotLoaded","generate"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/VADModule.ts"],"mappings":";;AAAA,SAASA,eAAe,QAAQ,6BAA6B;AAG7D,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,qBAAqB,QAAQ,yBAAyB;AAC/D,SAASC,iBAAiB,QAAQ,yBAAyB;;AAE3D;AACA;AACA;AACA;AACA;AACA,OAAO,MAAMC,SAAS,SAASH,UAAU,CAAC;EACxC;AACF;AACA;AACA;AACA;AACA;AACA;EACE,MAAMI,IAAIA,CACRC,KAAsC,EACtCC,0BAAsD,GAAGA,CAAA,KAAM,CAAC,CAAC,EAClD;IACf,MAAMC,KAAK,GAAG,MAAMR,eAAe,CAACS,KAAK,CACvCF,0BAA0B,EAC1BD,KAAK,CAACI,WACR,CAAC;IACD,IAAIF,KAAK,KAAK,IAAI,IAAIA,KAAK,CAACG,MAAM,GAAG,CAAC,EAAE;MACtC,MAAM,IAAIR,iBAAiB,CACzBD,qBAAqB,CAACU,mBAAmB,EACzC,2GACF,CAAC;IACH;IACA,IAAI,CAACC,YAAY,GAAGC,MAAM,CAACC,OAAO,CAACP,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;EACpD;;EAEA;AACF;AACA;AACA;AACA;AACA;EACE,MAAMQ,OAAOA,CAACC,QAAsB,EAAsB;IACxD,IAAI,IAAI,CAACJ,YAAY,IAAI,IAAI,EAC3B,MAAM,IAAIV,iBAAiB,CACzBD,qBAAqB,CAACgB,eAAe,EACrC,oFACF,CAAC;IACH,OAAO,MAAM,IAAI,CAACL,YAAY,CAACM,QAAQ,CAACF,QAAQ,CAAC;EACnD;AACF","ignoreList":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":[],"sourceRoot":"../../../src","sources":["types/classification.ts"],"mappings":"","ignoreList":[]}
|