react-native-executorch 0.5.1 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/Error.d.ts +30 -0
- package/lib/Error.js +50 -0
- package/lib/common/Logger.d.ts +8 -0
- package/lib/common/Logger.js +19 -0
- package/lib/constants/directories.d.ts +1 -0
- package/lib/constants/directories.js +2 -0
- package/lib/constants/llmDefaults.d.ts +6 -0
- package/lib/constants/llmDefaults.js +16 -0
- package/lib/constants/modelUrls.d.ts +223 -0
- package/lib/constants/modelUrls.js +322 -0
- package/lib/constants/ocr/models.d.ts +882 -0
- package/lib/constants/ocr/models.js +182 -0
- package/lib/constants/ocr/symbols.d.ts +75 -0
- package/lib/constants/ocr/symbols.js +139 -0
- package/lib/constants/sttDefaults.d.ts +28 -0
- package/lib/constants/sttDefaults.js +68 -0
- package/lib/controllers/LLMController.d.ts +47 -0
- package/lib/controllers/LLMController.js +213 -0
- package/lib/controllers/OCRController.d.ts +23 -0
- package/lib/controllers/OCRController.js +72 -0
- package/lib/controllers/SpeechToTextController.d.ts +56 -0
- package/lib/controllers/SpeechToTextController.js +349 -0
- package/lib/controllers/VerticalOCRController.d.ts +25 -0
- package/lib/controllers/VerticalOCRController.js +75 -0
- package/lib/hooks/computer_vision/useClassification.d.ts +15 -0
- package/lib/hooks/computer_vision/useClassification.js +7 -0
- package/lib/hooks/computer_vision/useImageEmbeddings.d.ts +15 -0
- package/lib/hooks/computer_vision/useImageEmbeddings.js +7 -0
- package/lib/hooks/computer_vision/useImageSegmentation.d.ts +38 -0
- package/lib/hooks/computer_vision/useImageSegmentation.js +7 -0
- package/lib/hooks/computer_vision/useOCR.d.ts +20 -0
- package/lib/hooks/computer_vision/useOCR.js +42 -0
- package/lib/hooks/computer_vision/useObjectDetection.d.ts +15 -0
- package/lib/hooks/computer_vision/useObjectDetection.js +7 -0
- package/lib/hooks/computer_vision/useStyleTransfer.d.ts +15 -0
- package/lib/hooks/computer_vision/useStyleTransfer.js +7 -0
- package/lib/hooks/computer_vision/useVerticalOCR.d.ts +21 -0
- package/lib/hooks/computer_vision/useVerticalOCR.js +45 -0
- package/lib/hooks/general/useExecutorchModule.d.ts +13 -0
- package/lib/hooks/general/useExecutorchModule.js +7 -0
- package/lib/hooks/natural_language_processing/useLLM.d.ts +10 -0
- package/lib/hooks/natural_language_processing/useLLM.js +78 -0
- package/lib/hooks/natural_language_processing/useSpeechToText.d.ts +27 -0
- package/lib/hooks/natural_language_processing/useSpeechToText.js +49 -0
- package/lib/hooks/natural_language_processing/useTextEmbeddings.d.ts +16 -0
- package/lib/hooks/natural_language_processing/useTextEmbeddings.js +7 -0
- package/lib/hooks/natural_language_processing/useTokenizer.d.ts +17 -0
- package/lib/hooks/natural_language_processing/useTokenizer.js +52 -0
- package/lib/hooks/useModule.d.ts +17 -0
- package/lib/hooks/useModule.js +45 -0
- package/lib/hooks/useNonStaticModule.d.ts +20 -0
- package/lib/hooks/useNonStaticModule.js +49 -0
- package/lib/index.d.ts +50 -0
- package/lib/index.js +60 -0
- package/lib/module/utils/ResourceFetcher.js +6 -8
- package/lib/module/utils/ResourceFetcher.js.map +1 -1
- package/lib/module/utils/ResourceFetcherUtils.js +20 -20
- package/lib/module/utils/ResourceFetcherUtils.js.map +1 -1
- package/lib/modules/BaseModule.d.ts +8 -0
- package/lib/modules/BaseModule.js +25 -0
- package/lib/modules/BaseNonStaticModule.d.ts +9 -0
- package/lib/modules/BaseNonStaticModule.js +14 -0
- package/lib/modules/computer_vision/ClassificationModule.d.ts +8 -0
- package/lib/modules/computer_vision/ClassificationModule.js +17 -0
- package/lib/modules/computer_vision/ImageEmbeddingsModule.d.ts +8 -0
- package/lib/modules/computer_vision/ImageEmbeddingsModule.js +17 -0
- package/lib/modules/computer_vision/ImageSegmentationModule.d.ts +11 -0
- package/lib/modules/computer_vision/ImageSegmentationModule.js +27 -0
- package/lib/modules/computer_vision/OCRModule.d.ts +15 -0
- package/lib/modules/computer_vision/OCRModule.js +20 -0
- package/lib/modules/computer_vision/ObjectDetectionModule.d.ts +9 -0
- package/lib/modules/computer_vision/ObjectDetectionModule.js +17 -0
- package/lib/modules/computer_vision/StyleTransferModule.d.ts +8 -0
- package/lib/modules/computer_vision/StyleTransferModule.js +17 -0
- package/lib/modules/computer_vision/VerticalOCRModule.d.ts +15 -0
- package/lib/modules/computer_vision/VerticalOCRModule.js +22 -0
- package/lib/modules/general/ExecutorchModule.d.ts +7 -0
- package/lib/modules/general/ExecutorchModule.js +14 -0
- package/lib/modules/natural_language_processing/LLMModule.d.ts +28 -0
- package/lib/modules/natural_language_processing/LLMModule.js +45 -0
- package/lib/modules/natural_language_processing/SpeechToTextModule.d.ts +24 -0
- package/lib/modules/natural_language_processing/SpeechToTextModule.js +36 -0
- package/lib/modules/natural_language_processing/TextEmbeddingsModule.d.ts +9 -0
- package/lib/modules/natural_language_processing/TextEmbeddingsModule.js +21 -0
- package/lib/modules/natural_language_processing/TokenizerModule.d.ts +12 -0
- package/lib/modules/natural_language_processing/TokenizerModule.js +30 -0
- package/lib/native/NativeETInstaller.d.ts +6 -0
- package/lib/native/NativeETInstaller.js +2 -0
- package/lib/native/NativeOCR.d.ts +8 -0
- package/lib/native/NativeOCR.js +2 -0
- package/lib/native/NativeVerticalOCR.d.ts +8 -0
- package/lib/native/NativeVerticalOCR.js +2 -0
- package/lib/native/RnExecutorchModules.d.ts +3 -0
- package/lib/native/RnExecutorchModules.js +16 -0
- package/lib/types/common.d.ts +31 -0
- package/lib/types/common.js +25 -0
- package/lib/types/imageSegmentation.d.ts +24 -0
- package/lib/types/imageSegmentation.js +26 -0
- package/lib/types/llm.d.ts +46 -0
- package/lib/types/llm.js +9 -0
- package/lib/types/objectDetection.d.ts +104 -0
- package/lib/types/objectDetection.js +94 -0
- package/lib/types/ocr.d.ts +11 -0
- package/lib/types/ocr.js +1 -0
- package/lib/types/stt.d.ts +94 -0
- package/lib/types/stt.js +85 -0
- package/lib/typescript/utils/ResourceFetcher.d.ts.map +1 -1
- package/lib/typescript/utils/ResourceFetcherUtils.d.ts.map +1 -1
- package/lib/utils/ResourceFetcher.d.ts +24 -0
- package/lib/utils/ResourceFetcher.js +305 -0
- package/lib/utils/ResourceFetcherUtils.d.ts +54 -0
- package/lib/utils/ResourceFetcherUtils.js +128 -0
- package/lib/utils/llm.d.ts +6 -0
- package/lib/utils/llm.js +73 -0
- package/lib/utils/stt.d.ts +1 -0
- package/lib/utils/stt.js +21 -0
- package/package.json +1 -1
- package/src/utils/ResourceFetcher.ts +9 -7
- package/src/utils/ResourceFetcherUtils.ts +15 -17
- package/ios/RnExecutorch.xcodeproj/project.xcworkspace/contents.xcworkspacedata +0 -7
- package/ios/RnExecutorch.xcodeproj/project.xcworkspace/xcuserdata/jakubchmura.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
- package/ios/RnExecutorch.xcodeproj/xcuserdata/jakubchmura.xcuserdatad/xcschemes/xcschememanagement.plist +0 -14
- package/lib/tsconfig.tsbuildinfo +0 -1
- package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/contents.xcworkspacedata +0 -7
- package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/xcuserdata/jakubchmura.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
- package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/xcuserdata/jakubchmura.xcuserdatad/xcschemes/xcschememanagement.plist +0 -14
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { ResourceSource } from '../../types/common';
|
|
2
|
+
import { OCRDetection, OCRLanguage } from '../../types/ocr';
|
|
3
|
+
interface OCRModule {
|
|
4
|
+
error: string | null;
|
|
5
|
+
isReady: boolean;
|
|
6
|
+
isGenerating: boolean;
|
|
7
|
+
forward: (input: string) => Promise<OCRDetection[]>;
|
|
8
|
+
downloadProgress: number;
|
|
9
|
+
}
|
|
10
|
+
export declare const useVerticalOCR: ({ model, independentCharacters, preventLoad, }: {
|
|
11
|
+
model: {
|
|
12
|
+
detectorLarge: ResourceSource;
|
|
13
|
+
detectorNarrow: ResourceSource;
|
|
14
|
+
recognizerLarge: ResourceSource;
|
|
15
|
+
recognizerSmall: ResourceSource;
|
|
16
|
+
language: OCRLanguage;
|
|
17
|
+
};
|
|
18
|
+
independentCharacters?: boolean;
|
|
19
|
+
preventLoad?: boolean;
|
|
20
|
+
}) => OCRModule;
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { useEffect, useMemo, useState } from 'react';
|
|
2
|
+
import { VerticalOCRController } from '../../controllers/VerticalOCRController';
|
|
3
|
+
export const useVerticalOCR = ({ model, independentCharacters = false, preventLoad = false, }) => {
|
|
4
|
+
const [error, setError] = useState(null);
|
|
5
|
+
const [isReady, setIsReady] = useState(false);
|
|
6
|
+
const [isGenerating, setIsGenerating] = useState(false);
|
|
7
|
+
const [downloadProgress, setDownloadProgress] = useState(0);
|
|
8
|
+
const controllerInstance = useMemo(() => new VerticalOCRController({
|
|
9
|
+
isReadyCallback: setIsReady,
|
|
10
|
+
isGeneratingCallback: setIsGenerating,
|
|
11
|
+
errorCallback: setError,
|
|
12
|
+
}), []);
|
|
13
|
+
useEffect(() => {
|
|
14
|
+
if (preventLoad)
|
|
15
|
+
return;
|
|
16
|
+
(async () => {
|
|
17
|
+
await controllerInstance.load({
|
|
18
|
+
detectorLarge: model.detectorLarge,
|
|
19
|
+
detectorNarrow: model.detectorNarrow,
|
|
20
|
+
}, {
|
|
21
|
+
recognizerLarge: model.recognizerLarge,
|
|
22
|
+
recognizerSmall: model.recognizerSmall,
|
|
23
|
+
}, model.language, independentCharacters, setDownloadProgress);
|
|
24
|
+
})();
|
|
25
|
+
return () => {
|
|
26
|
+
controllerInstance.delete();
|
|
27
|
+
};
|
|
28
|
+
}, [
|
|
29
|
+
controllerInstance,
|
|
30
|
+
model.detectorLarge,
|
|
31
|
+
model.detectorNarrow,
|
|
32
|
+
model.recognizerLarge,
|
|
33
|
+
model.recognizerSmall,
|
|
34
|
+
model.language,
|
|
35
|
+
independentCharacters,
|
|
36
|
+
preventLoad,
|
|
37
|
+
]);
|
|
38
|
+
return {
|
|
39
|
+
error,
|
|
40
|
+
isReady,
|
|
41
|
+
isGenerating,
|
|
42
|
+
forward: controllerInstance.forward,
|
|
43
|
+
downloadProgress,
|
|
44
|
+
};
|
|
45
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { ResourceSource } from '../../types/common';
|
|
2
|
+
interface Props {
|
|
3
|
+
modelSource: ResourceSource;
|
|
4
|
+
preventLoad?: boolean;
|
|
5
|
+
}
|
|
6
|
+
export declare const useExecutorchModule: ({ modelSource, preventLoad, }: Props) => {
|
|
7
|
+
error: string | null;
|
|
8
|
+
isReady: boolean;
|
|
9
|
+
isGenerating: boolean;
|
|
10
|
+
downloadProgress: number;
|
|
11
|
+
forward: (inputTensor: import("../../types/common").TensorPtr[]) => Promise<import("../../types/common").TensorPtr[]>;
|
|
12
|
+
};
|
|
13
|
+
export {};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { ExecutorchModule } from '../../modules/general/ExecutorchModule';
|
|
2
|
+
import { useNonStaticModule } from '../useNonStaticModule';
|
|
3
|
+
export const useExecutorchModule = ({ modelSource, preventLoad = false, }) => useNonStaticModule({
|
|
4
|
+
module: ExecutorchModule,
|
|
5
|
+
model: { modelSource },
|
|
6
|
+
preventLoad,
|
|
7
|
+
});
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { ResourceSource } from '../../types/common';
|
|
2
|
+
import { LLMType } from '../../types/llm';
|
|
3
|
+
export declare const useLLM: ({ model, preventLoad, }: {
|
|
4
|
+
model: {
|
|
5
|
+
modelSource: ResourceSource;
|
|
6
|
+
tokenizerSource: ResourceSource;
|
|
7
|
+
tokenizerConfigSource: ResourceSource;
|
|
8
|
+
};
|
|
9
|
+
preventLoad?: boolean;
|
|
10
|
+
}) => LLMType;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { useCallback, useEffect, useMemo, useState } from 'react';
|
|
2
|
+
import { LLMController } from '../../controllers/LLMController';
|
|
3
|
+
/*
|
|
4
|
+
Hook version of LLMModule
|
|
5
|
+
*/
|
|
6
|
+
export const useLLM = ({ model, preventLoad = false, }) => {
|
|
7
|
+
const [token, setToken] = useState('');
|
|
8
|
+
const [response, setResponse] = useState('');
|
|
9
|
+
const [messageHistory, setMessageHistory] = useState([]);
|
|
10
|
+
const [isReady, setIsReady] = useState(false);
|
|
11
|
+
const [isGenerating, setIsGenerating] = useState(false);
|
|
12
|
+
const [downloadProgress, setDownloadProgress] = useState(0);
|
|
13
|
+
const [error, setError] = useState(null);
|
|
14
|
+
const tokenCallback = useCallback((newToken) => {
|
|
15
|
+
setToken(newToken);
|
|
16
|
+
setResponse((prevResponse) => prevResponse + newToken);
|
|
17
|
+
}, []);
|
|
18
|
+
const controllerInstance = useMemo(() => new LLMController({
|
|
19
|
+
tokenCallback: tokenCallback,
|
|
20
|
+
messageHistoryCallback: setMessageHistory,
|
|
21
|
+
isReadyCallback: setIsReady,
|
|
22
|
+
isGeneratingCallback: setIsGenerating,
|
|
23
|
+
}), [tokenCallback]);
|
|
24
|
+
useEffect(() => {
|
|
25
|
+
setDownloadProgress(0);
|
|
26
|
+
setError(null);
|
|
27
|
+
if (preventLoad)
|
|
28
|
+
return;
|
|
29
|
+
(async () => {
|
|
30
|
+
try {
|
|
31
|
+
await controllerInstance.load({
|
|
32
|
+
modelSource: model.modelSource,
|
|
33
|
+
tokenizerSource: model.tokenizerSource,
|
|
34
|
+
tokenizerConfigSource: model.tokenizerConfigSource,
|
|
35
|
+
onDownloadProgressCallback: setDownloadProgress,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
catch (e) {
|
|
39
|
+
setError(e);
|
|
40
|
+
}
|
|
41
|
+
})();
|
|
42
|
+
return () => {
|
|
43
|
+
controllerInstance.delete();
|
|
44
|
+
};
|
|
45
|
+
}, [
|
|
46
|
+
controllerInstance,
|
|
47
|
+
model.modelSource,
|
|
48
|
+
model.tokenizerSource,
|
|
49
|
+
model.tokenizerConfigSource,
|
|
50
|
+
preventLoad,
|
|
51
|
+
]);
|
|
52
|
+
// memoization of returned functions
|
|
53
|
+
const configure = useCallback(({ chatConfig, toolsConfig, }) => controllerInstance.configure({ chatConfig, toolsConfig }), [controllerInstance]);
|
|
54
|
+
const generate = useCallback((messages, tools) => {
|
|
55
|
+
setResponse('');
|
|
56
|
+
return controllerInstance.generate(messages, tools);
|
|
57
|
+
}, [controllerInstance]);
|
|
58
|
+
const sendMessage = useCallback((message) => {
|
|
59
|
+
setResponse('');
|
|
60
|
+
return controllerInstance.sendMessage(message);
|
|
61
|
+
}, [controllerInstance]);
|
|
62
|
+
const deleteMessage = useCallback((index) => controllerInstance.deleteMessage(index), [controllerInstance]);
|
|
63
|
+
const interrupt = useCallback(() => controllerInstance.interrupt(), [controllerInstance]);
|
|
64
|
+
return {
|
|
65
|
+
messageHistory,
|
|
66
|
+
response,
|
|
67
|
+
token,
|
|
68
|
+
isReady,
|
|
69
|
+
isGenerating,
|
|
70
|
+
downloadProgress,
|
|
71
|
+
error,
|
|
72
|
+
configure: configure,
|
|
73
|
+
generate: generate,
|
|
74
|
+
sendMessage: sendMessage,
|
|
75
|
+
deleteMessage: deleteMessage,
|
|
76
|
+
interrupt: interrupt,
|
|
77
|
+
};
|
|
78
|
+
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { SpeechToTextController } from '../../controllers/SpeechToTextController';
|
|
2
|
+
import { ResourceSource } from '../../types/common';
|
|
3
|
+
import { STREAMING_ACTION } from '../../constants/sttDefaults';
|
|
4
|
+
import { AvailableModels, SpeechToTextLanguage } from '../../types/stt';
|
|
5
|
+
interface SpeechToTextModule {
|
|
6
|
+
isReady: boolean;
|
|
7
|
+
isGenerating: boolean;
|
|
8
|
+
sequence: string;
|
|
9
|
+
downloadProgress: number;
|
|
10
|
+
configureStreaming: SpeechToTextController['configureStreaming'];
|
|
11
|
+
error: Error | undefined;
|
|
12
|
+
transcribe: (input: number[], audioLanguage?: SpeechToTextLanguage) => ReturnType<SpeechToTextController['transcribe']>;
|
|
13
|
+
streamingTranscribe: (streamAction: STREAMING_ACTION, input?: number[], audioLanguage?: SpeechToTextLanguage) => ReturnType<SpeechToTextController['streamingTranscribe']>;
|
|
14
|
+
}
|
|
15
|
+
export declare const useSpeechToText: ({ model, overlapSeconds, windowSize, streamingConfig, preventLoad, }: {
|
|
16
|
+
model: {
|
|
17
|
+
modelName: AvailableModels;
|
|
18
|
+
encoderSource: ResourceSource;
|
|
19
|
+
decoderSource: ResourceSource;
|
|
20
|
+
tokenizerSource: ResourceSource;
|
|
21
|
+
};
|
|
22
|
+
overlapSeconds?: ConstructorParameters<typeof SpeechToTextController>["0"]["overlapSeconds"];
|
|
23
|
+
windowSize?: ConstructorParameters<typeof SpeechToTextController>["0"]["windowSize"];
|
|
24
|
+
streamingConfig?: ConstructorParameters<typeof SpeechToTextController>["0"]["streamingConfig"];
|
|
25
|
+
preventLoad?: boolean;
|
|
26
|
+
}) => SpeechToTextModule;
|
|
27
|
+
export {};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { useEffect, useMemo, useState } from 'react';
|
|
2
|
+
import { SpeechToTextController } from '../../controllers/SpeechToTextController';
|
|
3
|
+
export const useSpeechToText = ({ model, overlapSeconds, windowSize, streamingConfig, preventLoad = false, }) => {
|
|
4
|
+
const [sequence, setSequence] = useState('');
|
|
5
|
+
const [isReady, setIsReady] = useState(false);
|
|
6
|
+
const [downloadProgress, setDownloadProgress] = useState(0);
|
|
7
|
+
const [isGenerating, setIsGenerating] = useState(false);
|
|
8
|
+
const [error, setError] = useState();
|
|
9
|
+
const controllerInstance = useMemo(() => new SpeechToTextController({
|
|
10
|
+
transcribeCallback: setSequence,
|
|
11
|
+
isReadyCallback: setIsReady,
|
|
12
|
+
isGeneratingCallback: setIsGenerating,
|
|
13
|
+
onErrorCallback: setError,
|
|
14
|
+
}), []);
|
|
15
|
+
useEffect(() => {
|
|
16
|
+
controllerInstance.configureStreaming(overlapSeconds, windowSize, streamingConfig);
|
|
17
|
+
}, [controllerInstance, overlapSeconds, windowSize, streamingConfig]);
|
|
18
|
+
useEffect(() => {
|
|
19
|
+
const loadModel = async () => {
|
|
20
|
+
await controllerInstance.load({
|
|
21
|
+
modelName: model.modelName,
|
|
22
|
+
encoderSource: model.encoderSource,
|
|
23
|
+
decoderSource: model.decoderSource,
|
|
24
|
+
tokenizerSource: model.tokenizerSource,
|
|
25
|
+
onDownloadProgressCallback: setDownloadProgress,
|
|
26
|
+
});
|
|
27
|
+
};
|
|
28
|
+
if (!preventLoad) {
|
|
29
|
+
loadModel();
|
|
30
|
+
}
|
|
31
|
+
}, [
|
|
32
|
+
controllerInstance,
|
|
33
|
+
model.modelName,
|
|
34
|
+
model.encoderSource,
|
|
35
|
+
model.decoderSource,
|
|
36
|
+
model.tokenizerSource,
|
|
37
|
+
preventLoad,
|
|
38
|
+
]);
|
|
39
|
+
return {
|
|
40
|
+
isReady,
|
|
41
|
+
isGenerating,
|
|
42
|
+
downloadProgress,
|
|
43
|
+
configureStreaming: controllerInstance.configureStreaming,
|
|
44
|
+
sequence,
|
|
45
|
+
error,
|
|
46
|
+
transcribe: (waveform, audioLanguage) => controllerInstance.transcribe(waveform, audioLanguage),
|
|
47
|
+
streamingTranscribe: (streamAction, waveform, audioLanguage) => controllerInstance.streamingTranscribe(streamAction, waveform, audioLanguage),
|
|
48
|
+
};
|
|
49
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { ResourceSource } from '../../types/common';
|
|
2
|
+
interface Props {
|
|
3
|
+
model: {
|
|
4
|
+
modelSource: ResourceSource;
|
|
5
|
+
tokenizerSource: ResourceSource;
|
|
6
|
+
};
|
|
7
|
+
preventLoad?: boolean;
|
|
8
|
+
}
|
|
9
|
+
export declare const useTextEmbeddings: ({ model, preventLoad }: Props) => {
|
|
10
|
+
error: string | null;
|
|
11
|
+
isReady: boolean;
|
|
12
|
+
isGenerating: boolean;
|
|
13
|
+
downloadProgress: number;
|
|
14
|
+
forward: (input: string) => Promise<Float32Array<ArrayBufferLike>>;
|
|
15
|
+
};
|
|
16
|
+
export {};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { TextEmbeddingsModule } from '../../modules/natural_language_processing/TextEmbeddingsModule';
|
|
2
|
+
import { useNonStaticModule } from '../useNonStaticModule';
|
|
3
|
+
export const useTextEmbeddings = ({ model, preventLoad = false }) => useNonStaticModule({
|
|
4
|
+
module: TextEmbeddingsModule,
|
|
5
|
+
model,
|
|
6
|
+
preventLoad,
|
|
7
|
+
});
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { ResourceSource } from '../../types/common';
|
|
2
|
+
export declare const useTokenizer: ({ tokenizer, preventLoad, }: {
|
|
3
|
+
tokenizer: {
|
|
4
|
+
tokenizerSource: ResourceSource;
|
|
5
|
+
};
|
|
6
|
+
preventLoad?: boolean;
|
|
7
|
+
}) => {
|
|
8
|
+
error: string | null;
|
|
9
|
+
isReady: boolean;
|
|
10
|
+
isGenerating: boolean;
|
|
11
|
+
downloadProgress: number;
|
|
12
|
+
decode: (tokens: number[], skipSpecialTokens?: boolean | undefined) => Promise<Promise<any>>;
|
|
13
|
+
encode: (s: string) => Promise<Promise<any>>;
|
|
14
|
+
getVocabSize: () => Promise<Promise<number>>;
|
|
15
|
+
idToToken: (tokenId: number) => Promise<Promise<string>>;
|
|
16
|
+
tokenToId: (token: string) => Promise<Promise<number>>;
|
|
17
|
+
};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { useEffect, useMemo, useState } from 'react';
|
|
2
|
+
import { TokenizerModule } from '../../modules/natural_language_processing/TokenizerModule';
|
|
3
|
+
import { ETError, getError } from '../../Error';
|
|
4
|
+
export const useTokenizer = ({ tokenizer, preventLoad = false, }) => {
|
|
5
|
+
const [error, setError] = useState(null);
|
|
6
|
+
const [isReady, setIsReady] = useState(false);
|
|
7
|
+
const [isGenerating, setIsGenerating] = useState(false);
|
|
8
|
+
const [downloadProgress, setDownloadProgress] = useState(0);
|
|
9
|
+
const _tokenizer = useMemo(() => new TokenizerModule(), []);
|
|
10
|
+
useEffect(() => {
|
|
11
|
+
if (preventLoad)
|
|
12
|
+
return;
|
|
13
|
+
(async () => {
|
|
14
|
+
setDownloadProgress(0);
|
|
15
|
+
setError(null);
|
|
16
|
+
try {
|
|
17
|
+
setIsReady(false);
|
|
18
|
+
await _tokenizer.load({ tokenizerSource: tokenizer.tokenizerSource }, setDownloadProgress);
|
|
19
|
+
setIsReady(true);
|
|
20
|
+
}
|
|
21
|
+
catch (err) {
|
|
22
|
+
setError(err.message);
|
|
23
|
+
}
|
|
24
|
+
})();
|
|
25
|
+
}, [_tokenizer, tokenizer.tokenizerSource, preventLoad]);
|
|
26
|
+
const stateWrapper = (fn) => {
|
|
27
|
+
return (...args) => {
|
|
28
|
+
if (!isReady)
|
|
29
|
+
throw new Error(getError(ETError.ModuleNotLoaded));
|
|
30
|
+
if (isGenerating)
|
|
31
|
+
throw new Error(getError(ETError.ModelGenerating));
|
|
32
|
+
try {
|
|
33
|
+
setIsGenerating(true);
|
|
34
|
+
return fn.apply(_tokenizer, args);
|
|
35
|
+
}
|
|
36
|
+
finally {
|
|
37
|
+
setIsGenerating(false);
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
return {
|
|
42
|
+
error,
|
|
43
|
+
isReady,
|
|
44
|
+
isGenerating,
|
|
45
|
+
downloadProgress,
|
|
46
|
+
decode: stateWrapper(TokenizerModule.prototype.decode),
|
|
47
|
+
encode: stateWrapper(TokenizerModule.prototype.encode),
|
|
48
|
+
getVocabSize: stateWrapper(TokenizerModule.prototype.getVocabSize),
|
|
49
|
+
idToToken: stateWrapper(TokenizerModule.prototype.idToToken),
|
|
50
|
+
tokenToId: stateWrapper(TokenizerModule.prototype.tokenToId),
|
|
51
|
+
};
|
|
52
|
+
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
interface Module {
|
|
2
|
+
load: (...args: any[]) => Promise<void>;
|
|
3
|
+
forward: (...input: any[]) => Promise<any>;
|
|
4
|
+
onDownloadProgress: (cb: (progress: number) => void) => void;
|
|
5
|
+
}
|
|
6
|
+
export declare const useModule: <M extends Module, LoadArgs extends Parameters<M["load"]>, ForwardArgs extends Parameters<M["forward"]>, ForwardReturn extends Awaited<ReturnType<M["forward"]>>>({ module, loadArgs, preventLoad, }: {
|
|
7
|
+
module: M;
|
|
8
|
+
loadArgs: LoadArgs;
|
|
9
|
+
preventLoad?: boolean;
|
|
10
|
+
}) => {
|
|
11
|
+
error: string | null;
|
|
12
|
+
isReady: boolean;
|
|
13
|
+
isGenerating: boolean;
|
|
14
|
+
downloadProgress: number;
|
|
15
|
+
forward: (...input: ForwardArgs) => Promise<ForwardReturn>;
|
|
16
|
+
};
|
|
17
|
+
export {};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { useEffect, useState } from 'react';
|
|
2
|
+
import { ETError, getError } from '../Error';
|
|
3
|
+
export const useModule = ({ module, loadArgs, preventLoad = false, }) => {
|
|
4
|
+
const [error, setError] = useState(null);
|
|
5
|
+
const [isReady, setIsReady] = useState(false);
|
|
6
|
+
const [isGenerating, setIsGenerating] = useState(false);
|
|
7
|
+
const [downloadProgress, setDownloadProgress] = useState(0);
|
|
8
|
+
useEffect(() => {
|
|
9
|
+
const loadModule = async () => {
|
|
10
|
+
try {
|
|
11
|
+
setIsReady(false);
|
|
12
|
+
module.onDownloadProgress(setDownloadProgress);
|
|
13
|
+
await module.load(...loadArgs);
|
|
14
|
+
setIsReady(true);
|
|
15
|
+
}
|
|
16
|
+
catch (err) {
|
|
17
|
+
setError(err.message);
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
if (!preventLoad) {
|
|
21
|
+
loadModule();
|
|
22
|
+
}
|
|
23
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
24
|
+
}, [...loadArgs, preventLoad]);
|
|
25
|
+
const forward = async (...input) => {
|
|
26
|
+
if (!isReady)
|
|
27
|
+
throw new Error(getError(ETError.ModuleNotLoaded));
|
|
28
|
+
if (isGenerating)
|
|
29
|
+
throw new Error(getError(ETError.ModelGenerating));
|
|
30
|
+
try {
|
|
31
|
+
setIsGenerating(true);
|
|
32
|
+
return await module.forward(...input);
|
|
33
|
+
}
|
|
34
|
+
finally {
|
|
35
|
+
setIsGenerating(false);
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
return {
|
|
39
|
+
error,
|
|
40
|
+
isReady,
|
|
41
|
+
isGenerating,
|
|
42
|
+
downloadProgress,
|
|
43
|
+
forward,
|
|
44
|
+
};
|
|
45
|
+
};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
interface Module {
|
|
2
|
+
load: (...args: any[]) => Promise<void>;
|
|
3
|
+
forward: (...args: any[]) => Promise<any>;
|
|
4
|
+
delete: () => void;
|
|
5
|
+
}
|
|
6
|
+
interface ModuleConstructor<M extends Module> {
|
|
7
|
+
new (): M;
|
|
8
|
+
}
|
|
9
|
+
export declare const useNonStaticModule: <M extends Module, LoadArgs extends Parameters<M["load"]>, ForwardArgs extends Parameters<M["forward"]>, ForwardReturn extends Awaited<ReturnType<M["forward"]>>>({ module, model, preventLoad, }: {
|
|
10
|
+
module: ModuleConstructor<M>;
|
|
11
|
+
model: LoadArgs[0];
|
|
12
|
+
preventLoad?: boolean;
|
|
13
|
+
}) => {
|
|
14
|
+
error: string | null;
|
|
15
|
+
isReady: boolean;
|
|
16
|
+
isGenerating: boolean;
|
|
17
|
+
downloadProgress: number;
|
|
18
|
+
forward: (...input: ForwardArgs) => Promise<ForwardReturn>;
|
|
19
|
+
};
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { useEffect, useState, useMemo } from 'react';
|
|
2
|
+
import { ETError, getError } from '../Error';
|
|
3
|
+
export const useNonStaticModule = ({ module, model, preventLoad = false, }) => {
|
|
4
|
+
const [error, setError] = useState(null);
|
|
5
|
+
const [isReady, setIsReady] = useState(false);
|
|
6
|
+
const [isGenerating, setIsGenerating] = useState(false);
|
|
7
|
+
const [downloadProgress, setDownloadProgress] = useState(0);
|
|
8
|
+
const moduleInstance = useMemo(() => new module(), [module]);
|
|
9
|
+
useEffect(() => {
|
|
10
|
+
if (preventLoad)
|
|
11
|
+
return;
|
|
12
|
+
(async () => {
|
|
13
|
+
setDownloadProgress(0);
|
|
14
|
+
setError(null);
|
|
15
|
+
try {
|
|
16
|
+
setIsReady(false);
|
|
17
|
+
await moduleInstance.load(model, setDownloadProgress);
|
|
18
|
+
setIsReady(true);
|
|
19
|
+
}
|
|
20
|
+
catch (err) {
|
|
21
|
+
setError(err.message);
|
|
22
|
+
}
|
|
23
|
+
})();
|
|
24
|
+
return () => {
|
|
25
|
+
moduleInstance.delete();
|
|
26
|
+
};
|
|
27
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
28
|
+
}, [moduleInstance, ...Object.values(model), preventLoad]);
|
|
29
|
+
const forward = async (...input) => {
|
|
30
|
+
if (!isReady)
|
|
31
|
+
throw new Error(getError(ETError.ModuleNotLoaded));
|
|
32
|
+
if (isGenerating)
|
|
33
|
+
throw new Error(getError(ETError.ModelGenerating));
|
|
34
|
+
try {
|
|
35
|
+
setIsGenerating(true);
|
|
36
|
+
return await moduleInstance.forward(...input);
|
|
37
|
+
}
|
|
38
|
+
finally {
|
|
39
|
+
setIsGenerating(false);
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
return {
|
|
43
|
+
error,
|
|
44
|
+
isReady,
|
|
45
|
+
isGenerating,
|
|
46
|
+
downloadProgress,
|
|
47
|
+
forward,
|
|
48
|
+
};
|
|
49
|
+
};
|
package/lib/index.d.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { SpeechToTextLanguage } from './types/stt';
|
|
2
|
+
declare global {
|
|
3
|
+
var loadStyleTransfer: (source: string) => any;
|
|
4
|
+
var loadImageSegmentation: (source: string) => any;
|
|
5
|
+
var loadClassification: (source: string) => any;
|
|
6
|
+
var loadObjectDetection: (source: string) => any;
|
|
7
|
+
var loadExecutorchModule: (source: string) => any;
|
|
8
|
+
var loadTokenizerModule: (source: string) => any;
|
|
9
|
+
var loadImageEmbeddings: (source: string) => any;
|
|
10
|
+
var loadTextEmbeddings: (modelSource: string, tokenizerSource: string) => any;
|
|
11
|
+
var loadLLM: (modelSource: string, tokenizerSource: string) => any;
|
|
12
|
+
var loadSpeechToText: (encoderSource: string, decoderSource: string, modelName: string) => any;
|
|
13
|
+
var loadOCR: (detectorSource: string, recognizerLarge: string, recognizerMedium: string, recognizerSmall: string, symbols: string) => any;
|
|
14
|
+
var loadVerticalOCR: (detectorLarge: string, detectorNarrow: string, recognizer: string, symbols: string, independentCharacters?: boolean) => any;
|
|
15
|
+
}
|
|
16
|
+
export * from './hooks/computer_vision/useClassification';
|
|
17
|
+
export * from './hooks/computer_vision/useObjectDetection';
|
|
18
|
+
export * from './hooks/computer_vision/useStyleTransfer';
|
|
19
|
+
export * from './hooks/computer_vision/useImageSegmentation';
|
|
20
|
+
export * from './hooks/computer_vision/useOCR';
|
|
21
|
+
export * from './hooks/computer_vision/useVerticalOCR';
|
|
22
|
+
export * from './hooks/computer_vision/useImageEmbeddings';
|
|
23
|
+
export * from './hooks/natural_language_processing/useLLM';
|
|
24
|
+
export * from './hooks/natural_language_processing/useSpeechToText';
|
|
25
|
+
export * from './hooks/natural_language_processing/useTextEmbeddings';
|
|
26
|
+
export * from './hooks/natural_language_processing/useTokenizer';
|
|
27
|
+
export * from './hooks/general/useExecutorchModule';
|
|
28
|
+
export * from './modules/computer_vision/ClassificationModule';
|
|
29
|
+
export * from './modules/computer_vision/ObjectDetectionModule';
|
|
30
|
+
export * from './modules/computer_vision/StyleTransferModule';
|
|
31
|
+
export * from './modules/computer_vision/ImageSegmentationModule';
|
|
32
|
+
export * from './modules/computer_vision/OCRModule';
|
|
33
|
+
export * from './modules/computer_vision/VerticalOCRModule';
|
|
34
|
+
export * from './modules/general/ExecutorchModule';
|
|
35
|
+
export * from './modules/computer_vision/ImageEmbeddingsModule';
|
|
36
|
+
export * from './modules/natural_language_processing/LLMModule';
|
|
37
|
+
export * from './modules/natural_language_processing/SpeechToTextModule';
|
|
38
|
+
export * from './modules/natural_language_processing/TextEmbeddingsModule';
|
|
39
|
+
export * from './modules/natural_language_processing/TokenizerModule';
|
|
40
|
+
export * from './utils/ResourceFetcher';
|
|
41
|
+
export * from './utils/llm';
|
|
42
|
+
export * from './types/objectDetection';
|
|
43
|
+
export * from './types/ocr';
|
|
44
|
+
export * from './types/imageSegmentation';
|
|
45
|
+
export * from './types/llm';
|
|
46
|
+
export { SpeechToTextLanguage };
|
|
47
|
+
export * from './constants/modelUrls';
|
|
48
|
+
export * from './constants/ocr/models';
|
|
49
|
+
export * from './constants/llmDefaults';
|
|
50
|
+
export { STREAMING_ACTION, MODES, AvailableModels, } from './constants/sttDefaults';
|
package/lib/index.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { SpeechToTextLanguage } from './types/stt';
|
|
2
|
+
import { ETInstallerNativeModule } from './native/RnExecutorchModules';
|
|
3
|
+
// eslint-disable no-var
|
|
4
|
+
if (global.loadStyleTransfer == null ||
|
|
5
|
+
global.loadImageSegmentation == null ||
|
|
6
|
+
global.loadExecutorchModule == null ||
|
|
7
|
+
global.loadClassification == null ||
|
|
8
|
+
global.loadObjectDetection == null ||
|
|
9
|
+
global.loadTokenizerModule == null ||
|
|
10
|
+
global.loadTextEmbeddings == null ||
|
|
11
|
+
global.loadImageEmbeddings == null ||
|
|
12
|
+
global.loadLLM == null ||
|
|
13
|
+
global.loadSpeechToText == null ||
|
|
14
|
+
global.loadOCR == null ||
|
|
15
|
+
global.loadVerticalOCR == null) {
|
|
16
|
+
if (!ETInstallerNativeModule) {
|
|
17
|
+
throw new Error(`Failed to install react-native-executorch: The native module could not be found.`);
|
|
18
|
+
}
|
|
19
|
+
ETInstallerNativeModule.install();
|
|
20
|
+
}
|
|
21
|
+
// hooks
|
|
22
|
+
export * from './hooks/computer_vision/useClassification';
|
|
23
|
+
export * from './hooks/computer_vision/useObjectDetection';
|
|
24
|
+
export * from './hooks/computer_vision/useStyleTransfer';
|
|
25
|
+
export * from './hooks/computer_vision/useImageSegmentation';
|
|
26
|
+
export * from './hooks/computer_vision/useOCR';
|
|
27
|
+
export * from './hooks/computer_vision/useVerticalOCR';
|
|
28
|
+
export * from './hooks/computer_vision/useImageEmbeddings';
|
|
29
|
+
export * from './hooks/natural_language_processing/useLLM';
|
|
30
|
+
export * from './hooks/natural_language_processing/useSpeechToText';
|
|
31
|
+
export * from './hooks/natural_language_processing/useTextEmbeddings';
|
|
32
|
+
export * from './hooks/natural_language_processing/useTokenizer';
|
|
33
|
+
export * from './hooks/general/useExecutorchModule';
|
|
34
|
+
// modules
|
|
35
|
+
export * from './modules/computer_vision/ClassificationModule';
|
|
36
|
+
export * from './modules/computer_vision/ObjectDetectionModule';
|
|
37
|
+
export * from './modules/computer_vision/StyleTransferModule';
|
|
38
|
+
export * from './modules/computer_vision/ImageSegmentationModule';
|
|
39
|
+
export * from './modules/computer_vision/OCRModule';
|
|
40
|
+
export * from './modules/computer_vision/VerticalOCRModule';
|
|
41
|
+
export * from './modules/general/ExecutorchModule';
|
|
42
|
+
export * from './modules/computer_vision/ImageEmbeddingsModule';
|
|
43
|
+
export * from './modules/natural_language_processing/LLMModule';
|
|
44
|
+
export * from './modules/natural_language_processing/SpeechToTextModule';
|
|
45
|
+
export * from './modules/natural_language_processing/TextEmbeddingsModule';
|
|
46
|
+
export * from './modules/natural_language_processing/TokenizerModule';
|
|
47
|
+
// utils
|
|
48
|
+
export * from './utils/ResourceFetcher';
|
|
49
|
+
export * from './utils/llm';
|
|
50
|
+
// types
|
|
51
|
+
export * from './types/objectDetection';
|
|
52
|
+
export * from './types/ocr';
|
|
53
|
+
export * from './types/imageSegmentation';
|
|
54
|
+
export * from './types/llm';
|
|
55
|
+
export { SpeechToTextLanguage };
|
|
56
|
+
// constants
|
|
57
|
+
export * from './constants/modelUrls';
|
|
58
|
+
export * from './constants/ocr/models';
|
|
59
|
+
export * from './constants/llmDefaults';
|
|
60
|
+
export { STREAMING_ACTION, MODES, AvailableModels, } from './constants/sttDefaults';
|
|
@@ -29,8 +29,9 @@
|
|
|
29
29
|
* - Implements linked list behavior via the `.next` attribute
|
|
30
30
|
* - Automatically processes subsequent downloads when `.next` contains a valid resource
|
|
31
31
|
*/
|
|
32
|
-
import { cacheDirectory, createDownloadResumable, moveAsync, FileSystemSessionType, writeAsStringAsync, EncodingType, deleteAsync, readDirectoryAsync } from 'expo-file-system';
|
|
32
|
+
import { cacheDirectory, copyAsync, createDownloadResumable, moveAsync, FileSystemSessionType, writeAsStringAsync, EncodingType, deleteAsync, readDirectoryAsync } from 'expo-file-system';
|
|
33
33
|
import { Asset } from 'expo-asset';
|
|
34
|
+
import { Platform } from 'react-native';
|
|
34
35
|
import { RNEDirectory } from '../constants/directories';
|
|
35
36
|
import { ResourceFetcherUtils, HTTP_CODE, DownloadStatus, SourceType } from './ResourceFetcherUtils';
|
|
36
37
|
export class ResourceFetcher {
|
|
@@ -226,17 +227,14 @@ export class ResourceFetcher {
|
|
|
226
227
|
const uri = asset.uri;
|
|
227
228
|
const filename = ResourceFetcherUtils.getFilenameFromUri(uri);
|
|
228
229
|
const fileUri = `${RNEDirectory}${filename}`;
|
|
229
|
-
|
|
230
|
+
// On Android, file uri does not contain file extension, so we add it manually
|
|
231
|
+
const fileUriWithType = Platform.OS === 'android' ? `${fileUri}.${asset.type}` : fileUri;
|
|
230
232
|
if (await ResourceFetcherUtils.checkFileExists(fileUri)) {
|
|
231
233
|
return ResourceFetcherUtils.removeFilePrefix(fileUri);
|
|
232
234
|
}
|
|
233
235
|
await ResourceFetcherUtils.createDirectoryIfNoExists();
|
|
234
|
-
await
|
|
235
|
-
|
|
236
|
-
throw new Error(`Asset local URI is not available for ${source}`);
|
|
237
|
-
}
|
|
238
|
-
await moveAsync({
|
|
239
|
-
from: asset.localUri,
|
|
236
|
+
await copyAsync({
|
|
237
|
+
from: asset.uri,
|
|
240
238
|
to: fileUriWithType
|
|
241
239
|
});
|
|
242
240
|
return ResourceFetcherUtils.removeFilePrefix(fileUriWithType);
|