react-native-executorch 0.5.1-rc.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -0
- package/common/rnexecutorch/models/speech_to_text/SpeechToText.cpp +4 -10
- package/common/rnexecutorch/models/speech_to_text/SpeechToText.h +1 -1
- package/common/rnexecutorch/models/speech_to_text/SpeechToTextStrategy.h +3 -2
- package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.cpp +16 -4
- package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.h +2 -2
- package/lib/Error.d.ts +30 -0
- package/lib/Error.js +50 -0
- package/lib/constants/directories.d.ts +1 -0
- package/lib/constants/directories.js +2 -0
- package/lib/constants/llmDefaults.d.ts +6 -0
- package/lib/constants/llmDefaults.js +16 -0
- package/lib/constants/modelUrls.d.ts +217 -83
- package/lib/constants/modelUrls.js +304 -98
- package/lib/constants/ocr/models.d.ts +882 -0
- package/lib/constants/ocr/models.js +182 -0
- package/lib/constants/ocr/symbols.d.ts +75 -0
- package/lib/constants/ocr/symbols.js +139 -0
- package/lib/{typescript/constants → constants}/sttDefaults.d.ts +0 -1
- package/lib/constants/sttDefaults.js +12 -10
- package/lib/controllers/LLMController.d.ts +47 -0
- package/lib/controllers/LLMController.js +14 -11
- package/lib/controllers/OCRController.d.ts +23 -0
- package/lib/controllers/OCRController.js +12 -5
- package/lib/controllers/SpeechToTextController.d.ts +8 -4
- package/lib/controllers/SpeechToTextController.js +15 -9
- package/lib/controllers/VerticalOCRController.d.ts +25 -0
- package/lib/controllers/VerticalOCRController.js +75 -0
- package/lib/hooks/computer_vision/useClassification.d.ts +15 -0
- package/lib/hooks/computer_vision/useClassification.js +7 -0
- package/lib/hooks/computer_vision/useImageEmbeddings.d.ts +15 -0
- package/lib/hooks/computer_vision/useImageEmbeddings.js +7 -0
- package/lib/hooks/computer_vision/useImageSegmentation.d.ts +38 -0
- package/lib/hooks/computer_vision/useImageSegmentation.js +7 -0
- package/lib/hooks/computer_vision/useOCR.d.ts +20 -0
- package/lib/hooks/computer_vision/useOCR.js +42 -0
- package/lib/hooks/computer_vision/useObjectDetection.d.ts +15 -0
- package/lib/hooks/computer_vision/useObjectDetection.js +7 -0
- package/lib/hooks/computer_vision/useStyleTransfer.d.ts +15 -0
- package/lib/hooks/computer_vision/useStyleTransfer.js +7 -0
- package/lib/hooks/computer_vision/useVerticalOCR.d.ts +21 -0
- package/lib/hooks/computer_vision/useVerticalOCR.js +45 -0
- package/lib/hooks/general/useExecutorchModule.d.ts +13 -0
- package/lib/hooks/general/useExecutorchModule.js +7 -0
- package/lib/hooks/natural_language_processing/useLLM.d.ts +10 -0
- package/lib/hooks/natural_language_processing/useLLM.js +78 -0
- package/lib/hooks/natural_language_processing/useSpeechToText.d.ts +27 -0
- package/lib/hooks/natural_language_processing/useSpeechToText.js +19 -14
- package/lib/hooks/natural_language_processing/useTextEmbeddings.d.ts +16 -0
- package/lib/hooks/natural_language_processing/useTextEmbeddings.js +7 -0
- package/lib/hooks/natural_language_processing/useTokenizer.d.ts +17 -0
- package/lib/hooks/natural_language_processing/useTokenizer.js +52 -0
- package/lib/hooks/useModule.d.ts +17 -0
- package/lib/hooks/useModule.js +45 -0
- package/lib/hooks/useNonStaticModule.d.ts +20 -0
- package/lib/hooks/useNonStaticModule.js +49 -0
- package/lib/index.d.ts +1 -1
- package/lib/index.js +3 -2
- package/lib/module/constants/modelUrls.js +61 -36
- package/lib/module/constants/modelUrls.js.map +1 -1
- package/lib/module/constants/ocr/models.js +1 -1
- package/lib/module/hooks/natural_language_processing/useSpeechToText.js +71 -34
- package/lib/module/hooks/natural_language_processing/useSpeechToText.js.map +1 -1
- package/lib/module/index.js +2 -3
- package/lib/module/index.js.map +1 -1
- package/lib/module/modules/natural_language_processing/SpeechToTextModule.js +72 -31
- package/lib/module/modules/natural_language_processing/SpeechToTextModule.js.map +1 -1
- package/lib/module/types/stt.js +1 -85
- package/lib/module/types/stt.js.map +1 -1
- package/lib/module/utils/ResourceFetcher.js +6 -8
- package/lib/module/utils/ResourceFetcher.js.map +1 -1
- package/lib/module/utils/ResourceFetcherUtils.js +20 -20
- package/lib/module/utils/ResourceFetcherUtils.js.map +1 -1
- package/lib/module/utils/SpeechToTextModule/ASR.js +191 -0
- package/lib/module/utils/SpeechToTextModule/ASR.js.map +1 -0
- package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js +73 -0
- package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js.map +1 -0
- package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js +56 -0
- package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js.map +1 -0
- package/lib/modules/BaseModule.d.ts +8 -0
- package/lib/modules/BaseModule.js +25 -0
- package/lib/modules/BaseNonStaticModule.d.ts +9 -0
- package/lib/modules/BaseNonStaticModule.js +14 -0
- package/lib/modules/computer_vision/ClassificationModule.d.ts +8 -0
- package/lib/modules/computer_vision/ClassificationModule.js +17 -0
- package/lib/modules/computer_vision/ImageEmbeddingsModule.d.ts +8 -0
- package/lib/modules/computer_vision/ImageEmbeddingsModule.js +17 -0
- package/lib/modules/computer_vision/ImageSegmentationModule.d.ts +11 -0
- package/lib/modules/computer_vision/ImageSegmentationModule.js +27 -0
- package/lib/modules/computer_vision/OCRModule.d.ts +15 -0
- package/lib/modules/computer_vision/OCRModule.js +20 -0
- package/lib/modules/computer_vision/ObjectDetectionModule.d.ts +9 -0
- package/lib/modules/computer_vision/ObjectDetectionModule.js +17 -0
- package/lib/modules/computer_vision/StyleTransferModule.d.ts +8 -0
- package/lib/modules/computer_vision/StyleTransferModule.js +17 -0
- package/lib/modules/computer_vision/VerticalOCRModule.d.ts +15 -0
- package/lib/modules/computer_vision/VerticalOCRModule.js +22 -0
- package/lib/modules/general/ExecutorchModule.d.ts +7 -0
- package/lib/modules/general/ExecutorchModule.js +14 -0
- package/lib/modules/natural_language_processing/LLMModule.d.ts +28 -0
- package/lib/modules/natural_language_processing/LLMModule.js +45 -0
- package/lib/modules/natural_language_processing/SpeechToTextModule.d.ts +18 -8
- package/lib/modules/natural_language_processing/SpeechToTextModule.js +21 -15
- package/lib/modules/natural_language_processing/TextEmbeddingsModule.d.ts +9 -0
- package/lib/modules/natural_language_processing/TextEmbeddingsModule.js +21 -0
- package/lib/modules/natural_language_processing/TokenizerModule.d.ts +12 -0
- package/lib/modules/natural_language_processing/TokenizerModule.js +5 -4
- package/lib/native/NativeETInstaller.d.ts +6 -0
- package/lib/native/NativeETInstaller.js +2 -0
- package/lib/native/NativeOCR.d.ts +8 -0
- package/lib/native/NativeOCR.js +2 -0
- package/lib/native/NativeVerticalOCR.d.ts +8 -0
- package/lib/native/NativeVerticalOCR.js +2 -0
- package/lib/types/common.d.ts +31 -0
- package/lib/types/common.js +25 -0
- package/lib/types/imageSegmentation.d.ts +24 -0
- package/lib/types/imageSegmentation.js +26 -0
- package/lib/types/llm.d.ts +46 -0
- package/lib/types/llm.js +9 -0
- package/lib/types/objectDetection.d.ts +104 -0
- package/lib/types/objectDetection.js +94 -0
- package/lib/types/ocr.d.ts +11 -0
- package/lib/types/ocr.js +1 -0
- package/lib/types/stt.d.ts +94 -0
- package/lib/types/stt.js +85 -0
- package/lib/typescript/constants/modelUrls.d.ts +24 -7
- package/lib/typescript/constants/modelUrls.d.ts.map +1 -1
- package/lib/typescript/constants/ocr/models.d.ts +126 -126
- package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts +15 -24
- package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts.map +1 -1
- package/lib/typescript/index.d.ts +2 -3
- package/lib/typescript/index.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts +19 -22
- package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts.map +1 -1
- package/lib/typescript/types/stt.d.ts +17 -91
- package/lib/typescript/types/stt.d.ts.map +1 -1
- package/lib/typescript/utils/ResourceFetcher.d.ts.map +1 -1
- package/lib/typescript/utils/ResourceFetcherUtils.d.ts.map +1 -1
- package/lib/typescript/utils/SpeechToTextModule/ASR.d.ts +27 -0
- package/lib/typescript/utils/SpeechToTextModule/ASR.d.ts.map +1 -0
- package/lib/typescript/utils/SpeechToTextModule/OnlineProcessor.d.ts +23 -0
- package/lib/typescript/utils/SpeechToTextModule/OnlineProcessor.d.ts.map +1 -0
- package/lib/typescript/utils/SpeechToTextModule/hypothesisBuffer.d.ts +13 -0
- package/lib/typescript/utils/SpeechToTextModule/hypothesisBuffer.d.ts.map +1 -0
- package/lib/utils/ResourceFetcher.d.ts +24 -0
- package/lib/utils/ResourceFetcher.js +305 -0
- package/lib/utils/ResourceFetcherUtils.d.ts +54 -0
- package/lib/utils/ResourceFetcherUtils.js +9 -0
- package/lib/utils/llm.d.ts +6 -0
- package/lib/utils/llm.js +1 -0
- package/lib/utils/stt.d.ts +1 -0
- package/lib/utils/stt.js +21 -0
- package/package.json +5 -3
- package/src/constants/modelUrls.ts +70 -37
- package/src/constants/ocr/models.ts +1 -1
- package/src/hooks/natural_language_processing/useSpeechToText.ts +87 -92
- package/src/index.ts +6 -8
- package/src/modules/natural_language_processing/SpeechToTextModule.ts +81 -69
- package/src/types/stt.ts +97 -92
- package/src/utils/ResourceFetcher.ts +9 -7
- package/src/utils/ResourceFetcherUtils.ts +15 -17
- package/src/utils/SpeechToTextModule/ASR.ts +303 -0
- package/src/utils/SpeechToTextModule/OnlineProcessor.ts +87 -0
- package/src/utils/SpeechToTextModule/hypothesisBuffer.ts +79 -0
- package/common/rnexecutorch/models/speech_to_text/MoonshineStrategy.cpp +0 -31
- package/common/rnexecutorch/models/speech_to_text/MoonshineStrategy.h +0 -21
- package/lib/module/constants/sttDefaults.js +0 -74
- package/lib/module/constants/sttDefaults.js.map +0 -1
- package/lib/module/controllers/SpeechToTextController.js +0 -320
- package/lib/module/controllers/SpeechToTextController.js.map +0 -1
- package/lib/typescript/constants/sttDefaults.d.ts.map +0 -1
- package/lib/typescript/controllers/SpeechToTextController.d.ts +0 -57
- package/lib/typescript/controllers/SpeechToTextController.d.ts.map +0 -1
- package/src/constants/sttDefaults.ts +0 -82
- package/src/controllers/SpeechToTextController.ts +0 -471
- package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/contents.xcworkspacedata +0 -7
- package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/xcuserdata/norbertklockiewicz.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
- package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/xcuserdata/norbertklockiewicz.xcuserdatad/xcschemes/xcschememanagement.plist +0 -14
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { VerticalOCRController } from '../../controllers/VerticalOCRController';
|
|
2
|
+
export class VerticalOCRModule {
|
|
3
|
+
controller;
|
|
4
|
+
constructor() {
|
|
5
|
+
this.controller = new VerticalOCRController();
|
|
6
|
+
}
|
|
7
|
+
async load(model, independentCharacters, onDownloadProgressCallback = () => { }) {
|
|
8
|
+
await this.controller.load({
|
|
9
|
+
detectorLarge: model.detectorLarge,
|
|
10
|
+
detectorNarrow: model.detectorNarrow,
|
|
11
|
+
}, {
|
|
12
|
+
recognizerLarge: model.recognizerLarge,
|
|
13
|
+
recognizerSmall: model.recognizerSmall,
|
|
14
|
+
}, model.language, independentCharacters, onDownloadProgressCallback);
|
|
15
|
+
}
|
|
16
|
+
async forward(input) {
|
|
17
|
+
return await this.controller.forward(input);
|
|
18
|
+
}
|
|
19
|
+
delete() {
|
|
20
|
+
this.controller.delete();
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { TensorPtr } from '../../types/common';
|
|
2
|
+
import { BaseNonStaticModule } from '../BaseNonStaticModule';
|
|
3
|
+
import { ResourceSource } from '../../types/common';
|
|
4
|
+
export declare class ExecutorchModule extends BaseNonStaticModule {
|
|
5
|
+
load(modelSource: ResourceSource, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
|
|
6
|
+
forward(inputTensor: TensorPtr[]): Promise<TensorPtr[]>;
|
|
7
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { BaseNonStaticModule } from '../BaseNonStaticModule';
|
|
2
|
+
import { ResourceFetcher } from '../../utils/ResourceFetcher';
|
|
3
|
+
export class ExecutorchModule extends BaseNonStaticModule {
|
|
4
|
+
async load(modelSource, onDownloadProgressCallback = () => { }) {
|
|
5
|
+
const paths = await ResourceFetcher.fetch(onDownloadProgressCallback, modelSource);
|
|
6
|
+
if (paths === null || paths.length < 1) {
|
|
7
|
+
throw new Error('Download interrupted.');
|
|
8
|
+
}
|
|
9
|
+
this.nativeModule = global.loadExecutorchModule(paths[0] || '');
|
|
10
|
+
}
|
|
11
|
+
async forward(inputTensor) {
|
|
12
|
+
return await this.forwardET(inputTensor);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { ResourceSource } from '../../types/common';
|
|
2
|
+
import { ChatConfig, LLMTool, Message, ToolsConfig } from '../../types/llm';
|
|
3
|
+
export declare class LLMModule {
|
|
4
|
+
private controller;
|
|
5
|
+
constructor({ tokenCallback, responseCallback, messageHistoryCallback, }?: {
|
|
6
|
+
tokenCallback?: (token: string) => void;
|
|
7
|
+
responseCallback?: (response: string) => void;
|
|
8
|
+
messageHistoryCallback?: (messageHistory: Message[]) => void;
|
|
9
|
+
});
|
|
10
|
+
load(model: {
|
|
11
|
+
modelSource: ResourceSource;
|
|
12
|
+
tokenizerSource: ResourceSource;
|
|
13
|
+
tokenizerConfigSource: ResourceSource;
|
|
14
|
+
}, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
|
|
15
|
+
setTokenCallback({ tokenCallback, }: {
|
|
16
|
+
tokenCallback: (token: string) => void;
|
|
17
|
+
}): void;
|
|
18
|
+
configure({ chatConfig, toolsConfig, }: {
|
|
19
|
+
chatConfig?: Partial<ChatConfig>;
|
|
20
|
+
toolsConfig?: ToolsConfig;
|
|
21
|
+
}): void;
|
|
22
|
+
forward(input: string): Promise<string>;
|
|
23
|
+
generate(messages: Message[], tools?: LLMTool[]): Promise<string>;
|
|
24
|
+
sendMessage(message: string): Promise<Message[]>;
|
|
25
|
+
deleteMessage(index: number): Message[];
|
|
26
|
+
interrupt(): void;
|
|
27
|
+
delete(): void;
|
|
28
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { LLMController } from '../../controllers/LLMController';
|
|
2
|
+
export class LLMModule {
|
|
3
|
+
controller;
|
|
4
|
+
constructor({ tokenCallback, responseCallback, messageHistoryCallback, } = {}) {
|
|
5
|
+
this.controller = new LLMController({
|
|
6
|
+
tokenCallback,
|
|
7
|
+
responseCallback,
|
|
8
|
+
messageHistoryCallback,
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
async load(model, onDownloadProgressCallback = () => { }) {
|
|
12
|
+
await this.controller.load({
|
|
13
|
+
...model,
|
|
14
|
+
onDownloadProgressCallback,
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
setTokenCallback({ tokenCallback, }) {
|
|
18
|
+
this.controller.setTokenCallback(tokenCallback);
|
|
19
|
+
}
|
|
20
|
+
configure({ chatConfig, toolsConfig, }) {
|
|
21
|
+
this.controller.configure({ chatConfig, toolsConfig });
|
|
22
|
+
}
|
|
23
|
+
async forward(input) {
|
|
24
|
+
await this.controller.forward(input);
|
|
25
|
+
return this.controller.response;
|
|
26
|
+
}
|
|
27
|
+
async generate(messages, tools) {
|
|
28
|
+
await this.controller.generate(messages, tools);
|
|
29
|
+
return this.controller.response;
|
|
30
|
+
}
|
|
31
|
+
async sendMessage(message) {
|
|
32
|
+
await this.controller.sendMessage(message);
|
|
33
|
+
return this.controller.messageHistory;
|
|
34
|
+
}
|
|
35
|
+
deleteMessage(index) {
|
|
36
|
+
this.controller.deleteMessage(index);
|
|
37
|
+
return this.controller.messageHistory;
|
|
38
|
+
}
|
|
39
|
+
interrupt() {
|
|
40
|
+
this.controller.interrupt();
|
|
41
|
+
}
|
|
42
|
+
delete() {
|
|
43
|
+
this.controller.delete();
|
|
44
|
+
}
|
|
45
|
+
}
|
|
@@ -3,12 +3,22 @@ import { SpeechToTextController } from '../../controllers/SpeechToTextController
|
|
|
3
3
|
import { AvailableModels, SpeechToTextLanguage } from '../../types/stt';
|
|
4
4
|
import { STREAMING_ACTION } from '../../constants/sttDefaults';
|
|
5
5
|
export declare class SpeechToTextModule {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
6
|
+
private module;
|
|
7
|
+
constructor({ transcribeCallback, overlapSeconds, windowSize, streamingConfig, }?: {
|
|
8
|
+
transcribeCallback?: (sequence: string) => void;
|
|
9
|
+
overlapSeconds?: ConstructorParameters<typeof SpeechToTextController>['0']['overlapSeconds'];
|
|
10
|
+
windowSize?: ConstructorParameters<typeof SpeechToTextController>['0']['windowSize'];
|
|
11
|
+
streamingConfig?: ConstructorParameters<typeof SpeechToTextController>['0']['streamingConfig'];
|
|
12
|
+
});
|
|
13
|
+
load(model: {
|
|
14
|
+
modelName: AvailableModels;
|
|
15
|
+
encoderSource?: ResourceSource;
|
|
16
|
+
decoderSource?: ResourceSource;
|
|
17
|
+
tokenizerSource?: ResourceSource;
|
|
18
|
+
}, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
|
|
19
|
+
configureStreaming(overlapSeconds: Parameters<SpeechToTextController['configureStreaming']>[0], windowSize: Parameters<SpeechToTextController['configureStreaming']>[1], streamingConfig: Parameters<SpeechToTextController['configureStreaming']>[2]): void;
|
|
20
|
+
encode(waveform: Float32Array): Promise<null>;
|
|
21
|
+
decode(seq: number[]): Promise<number>;
|
|
22
|
+
transcribe(waveform: number[], audioLanguage?: SpeechToTextLanguage): ReturnType<SpeechToTextController['transcribe']>;
|
|
23
|
+
streamingTranscribe(streamAction: STREAMING_ACTION, waveform?: number[], audioLanguage?: SpeechToTextLanguage): ReturnType<SpeechToTextController['streamingTranscribe']>;
|
|
14
24
|
}
|
|
@@ -1,30 +1,36 @@
|
|
|
1
1
|
import { SpeechToTextController } from '../../controllers/SpeechToTextController';
|
|
2
2
|
export class SpeechToTextModule {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
static async load(modelName, transcribeCallback, modelDownloadProgressCallback, encoderSource, decoderSource, tokenizerSource, overlapSeconds, windowSize, streamingConfig) {
|
|
3
|
+
module;
|
|
4
|
+
constructor({ transcribeCallback, overlapSeconds, windowSize, streamingConfig, } = {}) {
|
|
6
5
|
this.module = new SpeechToTextController({
|
|
7
|
-
transcribeCallback: transcribeCallback,
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
streamingConfig: streamingConfig,
|
|
6
|
+
transcribeCallback: transcribeCallback || (() => { }),
|
|
7
|
+
overlapSeconds,
|
|
8
|
+
windowSize,
|
|
9
|
+
streamingConfig,
|
|
12
10
|
});
|
|
13
|
-
await this.module.loadModel((modelName = modelName), (encoderSource = encoderSource), (decoderSource = decoderSource), (tokenizerSource = tokenizerSource));
|
|
14
11
|
}
|
|
15
|
-
|
|
16
|
-
this.module
|
|
12
|
+
async load(model, onDownloadProgressCallback = () => { }) {
|
|
13
|
+
await this.module.load({
|
|
14
|
+
modelName: model.modelName,
|
|
15
|
+
encoderSource: model.encoderSource,
|
|
16
|
+
decoderSource: model.decoderSource,
|
|
17
|
+
tokenizerSource: model.tokenizerSource,
|
|
18
|
+
onDownloadProgressCallback,
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
configureStreaming(overlapSeconds, windowSize, streamingConfig) {
|
|
22
|
+
this.module.configureStreaming(overlapSeconds, windowSize, streamingConfig);
|
|
17
23
|
}
|
|
18
|
-
|
|
24
|
+
async encode(waveform) {
|
|
19
25
|
return await this.module.encode(waveform);
|
|
20
26
|
}
|
|
21
|
-
|
|
27
|
+
async decode(seq) {
|
|
22
28
|
return await this.module.decode(seq);
|
|
23
29
|
}
|
|
24
|
-
|
|
30
|
+
async transcribe(waveform, audioLanguage) {
|
|
25
31
|
return await this.module.transcribe(waveform, audioLanguage);
|
|
26
32
|
}
|
|
27
|
-
|
|
33
|
+
async streamingTranscribe(streamAction, waveform, audioLanguage) {
|
|
28
34
|
return await this.module.streamingTranscribe(streamAction, waveform, audioLanguage);
|
|
29
35
|
}
|
|
30
36
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { ResourceSource } from '../../types/common';
|
|
2
|
+
import { BaseNonStaticModule } from '../BaseNonStaticModule';
|
|
3
|
+
export declare class TextEmbeddingsModule extends BaseNonStaticModule {
|
|
4
|
+
load(model: {
|
|
5
|
+
modelSource: ResourceSource;
|
|
6
|
+
tokenizerSource: ResourceSource;
|
|
7
|
+
}, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
|
|
8
|
+
forward(input: string): Promise<Float32Array>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { ResourceFetcher } from '../../utils/ResourceFetcher';
|
|
2
|
+
import { BaseNonStaticModule } from '../BaseNonStaticModule';
|
|
3
|
+
export class TextEmbeddingsModule extends BaseNonStaticModule {
|
|
4
|
+
async load(model, onDownloadProgressCallback = () => { }) {
|
|
5
|
+
const modelPromise = ResourceFetcher.fetch(onDownloadProgressCallback, model.modelSource);
|
|
6
|
+
const tokenizerPromise = ResourceFetcher.fetch(undefined, model.tokenizerSource);
|
|
7
|
+
const [modelResult, tokenizerResult] = await Promise.all([
|
|
8
|
+
modelPromise,
|
|
9
|
+
tokenizerPromise,
|
|
10
|
+
]);
|
|
11
|
+
const modelPath = modelResult?.[0];
|
|
12
|
+
const tokenizerPath = tokenizerResult?.[0];
|
|
13
|
+
if (!modelPath || !tokenizerPath) {
|
|
14
|
+
throw new Error('Download interrupted.');
|
|
15
|
+
}
|
|
16
|
+
this.nativeModule = global.loadTextEmbeddings(modelPath, tokenizerPath);
|
|
17
|
+
}
|
|
18
|
+
async forward(input) {
|
|
19
|
+
return new Float32Array(await this.nativeModule.generate(input));
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { ResourceSource } from '../../types/common';
|
|
2
|
+
export declare class TokenizerModule {
|
|
3
|
+
nativeModule: any;
|
|
4
|
+
load(tokenizer: {
|
|
5
|
+
tokenizerSource: ResourceSource;
|
|
6
|
+
}, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
|
|
7
|
+
encode(s: string): Promise<any>;
|
|
8
|
+
decode(tokens: number[], skipSpecialTokens?: boolean): Promise<any>;
|
|
9
|
+
getVocabSize(): Promise<number>;
|
|
10
|
+
idToToken(tokenId: number): Promise<string>;
|
|
11
|
+
tokenToId(token: string): Promise<number>;
|
|
12
|
+
}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { ResourceFetcher } from '../../utils/ResourceFetcher';
|
|
2
2
|
export class TokenizerModule {
|
|
3
3
|
nativeModule;
|
|
4
|
-
async load(
|
|
5
|
-
const paths = await ResourceFetcher.fetch(onDownloadProgressCallback,
|
|
6
|
-
|
|
4
|
+
async load(tokenizer, onDownloadProgressCallback = () => { }) {
|
|
5
|
+
const paths = await ResourceFetcher.fetch(onDownloadProgressCallback, tokenizer.tokenizerSource);
|
|
6
|
+
const path = paths?.[0];
|
|
7
|
+
if (!path) {
|
|
7
8
|
throw new Error('Download interrupted.');
|
|
8
9
|
}
|
|
9
|
-
this.nativeModule = global.loadTokenizerModule(
|
|
10
|
+
this.nativeModule = global.loadTokenizerModule(path);
|
|
10
11
|
}
|
|
11
12
|
async encode(s) {
|
|
12
13
|
return await this.nativeModule.encode(s);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { TurboModule } from 'react-native';
|
|
2
|
+
import { OCRDetection } from '../types/ocr';
|
|
3
|
+
export interface Spec extends TurboModule {
|
|
4
|
+
loadModule(detectorSource: string, recognizerSourceLarge: string, recognizerSourceMedium: string, recognizerSourceSmall: string, symbols: string): Promise<number>;
|
|
5
|
+
forward(input: string): Promise<OCRDetection[]>;
|
|
6
|
+
}
|
|
7
|
+
declare const _default: Spec | null;
|
|
8
|
+
export default _default;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { TurboModule } from 'react-native';
|
|
2
|
+
import { OCRDetection } from '../types/ocr';
|
|
3
|
+
export interface Spec extends TurboModule {
|
|
4
|
+
loadModule(detectorLargeSource: string, detectorNarrowSource: string, recognizerSource: string, symbols: string, independentCharacters: boolean): Promise<number>;
|
|
5
|
+
forward(input: string): Promise<OCRDetection[]>;
|
|
6
|
+
}
|
|
7
|
+
declare const _default: Spec | null;
|
|
8
|
+
export default _default;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export type ResourceSource = string | number | object;
|
|
2
|
+
export declare enum ScalarType {
|
|
3
|
+
BYTE = 0,
|
|
4
|
+
CHAR = 1,
|
|
5
|
+
SHORT = 2,
|
|
6
|
+
INT = 3,
|
|
7
|
+
LONG = 4,
|
|
8
|
+
HALF = 5,
|
|
9
|
+
FLOAT = 6,
|
|
10
|
+
DOUBLE = 7,
|
|
11
|
+
BOOL = 11,
|
|
12
|
+
QINT8 = 12,
|
|
13
|
+
QUINT8 = 13,
|
|
14
|
+
QINT32 = 14,
|
|
15
|
+
QUINT4X2 = 16,
|
|
16
|
+
QUINT2X4 = 17,
|
|
17
|
+
BITS16 = 22,
|
|
18
|
+
FLOAT8E5M2 = 23,
|
|
19
|
+
FLOAT8E4M3FN = 24,
|
|
20
|
+
FLOAT8E5M2FNUZ = 25,
|
|
21
|
+
FLOAT8E4M3FNUZ = 26,
|
|
22
|
+
UINT16 = 27,
|
|
23
|
+
UINT32 = 28,
|
|
24
|
+
UINT64 = 29
|
|
25
|
+
}
|
|
26
|
+
export type TensorBuffer = ArrayBuffer | Float32Array | Float64Array | Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array | BigInt64Array | BigUint64Array;
|
|
27
|
+
export interface TensorPtr {
|
|
28
|
+
dataPtr: TensorBuffer;
|
|
29
|
+
sizes: number[];
|
|
30
|
+
scalarType: ScalarType;
|
|
31
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export var ScalarType;
|
|
2
|
+
(function (ScalarType) {
|
|
3
|
+
ScalarType[ScalarType["BYTE"] = 0] = "BYTE";
|
|
4
|
+
ScalarType[ScalarType["CHAR"] = 1] = "CHAR";
|
|
5
|
+
ScalarType[ScalarType["SHORT"] = 2] = "SHORT";
|
|
6
|
+
ScalarType[ScalarType["INT"] = 3] = "INT";
|
|
7
|
+
ScalarType[ScalarType["LONG"] = 4] = "LONG";
|
|
8
|
+
ScalarType[ScalarType["HALF"] = 5] = "HALF";
|
|
9
|
+
ScalarType[ScalarType["FLOAT"] = 6] = "FLOAT";
|
|
10
|
+
ScalarType[ScalarType["DOUBLE"] = 7] = "DOUBLE";
|
|
11
|
+
ScalarType[ScalarType["BOOL"] = 11] = "BOOL";
|
|
12
|
+
ScalarType[ScalarType["QINT8"] = 12] = "QINT8";
|
|
13
|
+
ScalarType[ScalarType["QUINT8"] = 13] = "QUINT8";
|
|
14
|
+
ScalarType[ScalarType["QINT32"] = 14] = "QINT32";
|
|
15
|
+
ScalarType[ScalarType["QUINT4X2"] = 16] = "QUINT4X2";
|
|
16
|
+
ScalarType[ScalarType["QUINT2X4"] = 17] = "QUINT2X4";
|
|
17
|
+
ScalarType[ScalarType["BITS16"] = 22] = "BITS16";
|
|
18
|
+
ScalarType[ScalarType["FLOAT8E5M2"] = 23] = "FLOAT8E5M2";
|
|
19
|
+
ScalarType[ScalarType["FLOAT8E4M3FN"] = 24] = "FLOAT8E4M3FN";
|
|
20
|
+
ScalarType[ScalarType["FLOAT8E5M2FNUZ"] = 25] = "FLOAT8E5M2FNUZ";
|
|
21
|
+
ScalarType[ScalarType["FLOAT8E4M3FNUZ"] = 26] = "FLOAT8E4M3FNUZ";
|
|
22
|
+
ScalarType[ScalarType["UINT16"] = 27] = "UINT16";
|
|
23
|
+
ScalarType[ScalarType["UINT32"] = 28] = "UINT32";
|
|
24
|
+
ScalarType[ScalarType["UINT64"] = 29] = "UINT64";
|
|
25
|
+
})(ScalarType || (ScalarType = {}));
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export declare enum DeeplabLabel {
|
|
2
|
+
BACKGROUND = 0,
|
|
3
|
+
AEROPLANE = 1,
|
|
4
|
+
BICYCLE = 2,
|
|
5
|
+
BIRD = 3,
|
|
6
|
+
BOAT = 4,
|
|
7
|
+
BOTTLE = 5,
|
|
8
|
+
BUS = 6,
|
|
9
|
+
CAR = 7,
|
|
10
|
+
CAT = 8,
|
|
11
|
+
CHAIR = 9,
|
|
12
|
+
COW = 10,
|
|
13
|
+
DININGTABLE = 11,
|
|
14
|
+
DOG = 12,
|
|
15
|
+
HORSE = 13,
|
|
16
|
+
MOTORBIKE = 14,
|
|
17
|
+
PERSON = 15,
|
|
18
|
+
POTTEDPLANT = 16,
|
|
19
|
+
SHEEP = 17,
|
|
20
|
+
SOFA = 18,
|
|
21
|
+
TRAIN = 19,
|
|
22
|
+
TVMONITOR = 20,
|
|
23
|
+
ARGMAX = 21
|
|
24
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/* eslint-disable @cspell/spellchecker */
|
|
2
|
+
export var DeeplabLabel;
|
|
3
|
+
(function (DeeplabLabel) {
|
|
4
|
+
DeeplabLabel[DeeplabLabel["BACKGROUND"] = 0] = "BACKGROUND";
|
|
5
|
+
DeeplabLabel[DeeplabLabel["AEROPLANE"] = 1] = "AEROPLANE";
|
|
6
|
+
DeeplabLabel[DeeplabLabel["BICYCLE"] = 2] = "BICYCLE";
|
|
7
|
+
DeeplabLabel[DeeplabLabel["BIRD"] = 3] = "BIRD";
|
|
8
|
+
DeeplabLabel[DeeplabLabel["BOAT"] = 4] = "BOAT";
|
|
9
|
+
DeeplabLabel[DeeplabLabel["BOTTLE"] = 5] = "BOTTLE";
|
|
10
|
+
DeeplabLabel[DeeplabLabel["BUS"] = 6] = "BUS";
|
|
11
|
+
DeeplabLabel[DeeplabLabel["CAR"] = 7] = "CAR";
|
|
12
|
+
DeeplabLabel[DeeplabLabel["CAT"] = 8] = "CAT";
|
|
13
|
+
DeeplabLabel[DeeplabLabel["CHAIR"] = 9] = "CHAIR";
|
|
14
|
+
DeeplabLabel[DeeplabLabel["COW"] = 10] = "COW";
|
|
15
|
+
DeeplabLabel[DeeplabLabel["DININGTABLE"] = 11] = "DININGTABLE";
|
|
16
|
+
DeeplabLabel[DeeplabLabel["DOG"] = 12] = "DOG";
|
|
17
|
+
DeeplabLabel[DeeplabLabel["HORSE"] = 13] = "HORSE";
|
|
18
|
+
DeeplabLabel[DeeplabLabel["MOTORBIKE"] = 14] = "MOTORBIKE";
|
|
19
|
+
DeeplabLabel[DeeplabLabel["PERSON"] = 15] = "PERSON";
|
|
20
|
+
DeeplabLabel[DeeplabLabel["POTTEDPLANT"] = 16] = "POTTEDPLANT";
|
|
21
|
+
DeeplabLabel[DeeplabLabel["SHEEP"] = 17] = "SHEEP";
|
|
22
|
+
DeeplabLabel[DeeplabLabel["SOFA"] = 18] = "SOFA";
|
|
23
|
+
DeeplabLabel[DeeplabLabel["TRAIN"] = 19] = "TRAIN";
|
|
24
|
+
DeeplabLabel[DeeplabLabel["TVMONITOR"] = 20] = "TVMONITOR";
|
|
25
|
+
DeeplabLabel[DeeplabLabel["ARGMAX"] = 21] = "ARGMAX";
|
|
26
|
+
})(DeeplabLabel || (DeeplabLabel = {}));
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export interface LLMType {
|
|
2
|
+
messageHistory: Message[];
|
|
3
|
+
response: string;
|
|
4
|
+
token: string;
|
|
5
|
+
isReady: boolean;
|
|
6
|
+
isGenerating: boolean;
|
|
7
|
+
downloadProgress: number;
|
|
8
|
+
error: string | null;
|
|
9
|
+
configure: ({ chatConfig, toolsConfig, }: {
|
|
10
|
+
chatConfig?: Partial<ChatConfig>;
|
|
11
|
+
toolsConfig?: ToolsConfig;
|
|
12
|
+
}) => void;
|
|
13
|
+
generate: (messages: Message[], tools?: LLMTool[]) => Promise<void>;
|
|
14
|
+
sendMessage: (message: string) => Promise<void>;
|
|
15
|
+
deleteMessage: (index: number) => void;
|
|
16
|
+
interrupt: () => void;
|
|
17
|
+
}
|
|
18
|
+
export type MessageRole = 'user' | 'assistant' | 'system';
|
|
19
|
+
export interface Message {
|
|
20
|
+
role: MessageRole;
|
|
21
|
+
content: string;
|
|
22
|
+
}
|
|
23
|
+
export interface ToolCall {
|
|
24
|
+
toolName: string;
|
|
25
|
+
arguments: Object;
|
|
26
|
+
}
|
|
27
|
+
export type LLMTool = Object;
|
|
28
|
+
export interface ChatConfig {
|
|
29
|
+
initialMessageHistory: Message[];
|
|
30
|
+
contextWindowLength: number;
|
|
31
|
+
systemPrompt: string;
|
|
32
|
+
}
|
|
33
|
+
export interface ToolsConfig {
|
|
34
|
+
tools: LLMTool[];
|
|
35
|
+
executeToolCallback: (call: ToolCall) => Promise<string | null>;
|
|
36
|
+
displayToolCalls?: boolean;
|
|
37
|
+
}
|
|
38
|
+
export declare const SPECIAL_TOKENS: {
|
|
39
|
+
BOS_TOKEN: string;
|
|
40
|
+
EOS_TOKEN: string;
|
|
41
|
+
UNK_TOKEN: string;
|
|
42
|
+
SEP_TOKEN: string;
|
|
43
|
+
PAD_TOKEN: string;
|
|
44
|
+
CLS_TOKEN: string;
|
|
45
|
+
MASK_TOKEN: string;
|
|
46
|
+
};
|
package/lib/types/llm.js
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
export interface Bbox {
|
|
2
|
+
x1: number;
|
|
3
|
+
x2: number;
|
|
4
|
+
y1: number;
|
|
5
|
+
y2: number;
|
|
6
|
+
}
|
|
7
|
+
export interface Detection {
|
|
8
|
+
bbox: Bbox;
|
|
9
|
+
label: keyof typeof CocoLabel;
|
|
10
|
+
score: number;
|
|
11
|
+
}
|
|
12
|
+
declare enum CocoLabel {
|
|
13
|
+
PERSON = 1,
|
|
14
|
+
BICYCLE = 2,
|
|
15
|
+
CAR = 3,
|
|
16
|
+
MOTORCYCLE = 4,
|
|
17
|
+
AIRPLANE = 5,
|
|
18
|
+
BUS = 6,
|
|
19
|
+
TRAIN = 7,
|
|
20
|
+
TRUCK = 8,
|
|
21
|
+
BOAT = 9,
|
|
22
|
+
TRAFFIC_LIGHT = 10,
|
|
23
|
+
FIRE_HYDRANT = 11,
|
|
24
|
+
STREET_SIGN = 12,
|
|
25
|
+
STOP_SIGN = 13,
|
|
26
|
+
PARKING = 14,
|
|
27
|
+
BENCH = 15,
|
|
28
|
+
BIRD = 16,
|
|
29
|
+
CAT = 17,
|
|
30
|
+
DOG = 18,
|
|
31
|
+
HORSE = 19,
|
|
32
|
+
SHEEP = 20,
|
|
33
|
+
COW = 21,
|
|
34
|
+
ELEPHANT = 22,
|
|
35
|
+
BEAR = 23,
|
|
36
|
+
ZEBRA = 24,
|
|
37
|
+
GIRAFFE = 25,
|
|
38
|
+
HAT = 26,
|
|
39
|
+
BACKPACK = 27,
|
|
40
|
+
UMBRELLA = 28,
|
|
41
|
+
SHOE = 29,
|
|
42
|
+
EYE = 30,
|
|
43
|
+
HANDBAG = 31,
|
|
44
|
+
TIE = 32,
|
|
45
|
+
SUITCASE = 33,
|
|
46
|
+
FRISBEE = 34,
|
|
47
|
+
SKIS = 35,
|
|
48
|
+
SNOWBOARD = 36,
|
|
49
|
+
SPORTS = 37,
|
|
50
|
+
KITE = 38,
|
|
51
|
+
BASEBALL = 39,
|
|
52
|
+
SKATEBOARD = 41,
|
|
53
|
+
SURFBOARD = 42,
|
|
54
|
+
TENNIS_RACKET = 43,
|
|
55
|
+
BOTTLE = 44,
|
|
56
|
+
PLATE = 45,
|
|
57
|
+
WINE_GLASS = 46,
|
|
58
|
+
CUP = 47,
|
|
59
|
+
FORK = 48,
|
|
60
|
+
KNIFE = 49,
|
|
61
|
+
SPOON = 50,
|
|
62
|
+
BOWL = 51,
|
|
63
|
+
BANANA = 52,
|
|
64
|
+
APPLE = 53,
|
|
65
|
+
SANDWICH = 54,
|
|
66
|
+
ORANGE = 55,
|
|
67
|
+
BROCCOLI = 56,
|
|
68
|
+
CARROT = 57,
|
|
69
|
+
HOT_DOG = 58,
|
|
70
|
+
PIZZA = 59,
|
|
71
|
+
DONUT = 60,
|
|
72
|
+
CAKE = 61,
|
|
73
|
+
CHAIR = 62,
|
|
74
|
+
COUCH = 63,
|
|
75
|
+
POTTED_PLANT = 64,
|
|
76
|
+
BED = 65,
|
|
77
|
+
MIRROR = 66,
|
|
78
|
+
DINING_TABLE = 67,
|
|
79
|
+
WINDOW = 68,
|
|
80
|
+
DESK = 69,
|
|
81
|
+
TOILET = 70,
|
|
82
|
+
DOOR = 71,
|
|
83
|
+
TV = 72,
|
|
84
|
+
LAPTOP = 73,
|
|
85
|
+
MOUSE = 74,
|
|
86
|
+
REMOTE = 75,
|
|
87
|
+
KEYBOARD = 76,
|
|
88
|
+
CELL_PHONE = 77,
|
|
89
|
+
MICROWAVE = 78,
|
|
90
|
+
OVEN = 79,
|
|
91
|
+
TOASTER = 80,
|
|
92
|
+
SINK = 81,
|
|
93
|
+
REFRIGERATOR = 82,
|
|
94
|
+
BLENDER = 83,
|
|
95
|
+
BOOK = 84,
|
|
96
|
+
CLOCK = 85,
|
|
97
|
+
VASE = 86,
|
|
98
|
+
SCISSORS = 87,
|
|
99
|
+
TEDDY_BEAR = 88,
|
|
100
|
+
HAIR_DRIER = 89,
|
|
101
|
+
TOOTHBRUSH = 90,
|
|
102
|
+
HAIR_BRUSH = 91
|
|
103
|
+
}
|
|
104
|
+
export {};
|