react-native-executorch 0.5.1-rc.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/README.md +132 -0
  2. package/common/rnexecutorch/models/speech_to_text/SpeechToText.cpp +4 -10
  3. package/common/rnexecutorch/models/speech_to_text/SpeechToText.h +1 -1
  4. package/common/rnexecutorch/models/speech_to_text/SpeechToTextStrategy.h +3 -2
  5. package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.cpp +16 -4
  6. package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.h +2 -2
  7. package/lib/Error.d.ts +30 -0
  8. package/lib/Error.js +50 -0
  9. package/lib/constants/directories.d.ts +1 -0
  10. package/lib/constants/directories.js +2 -0
  11. package/lib/constants/llmDefaults.d.ts +6 -0
  12. package/lib/constants/llmDefaults.js +16 -0
  13. package/lib/constants/modelUrls.d.ts +217 -83
  14. package/lib/constants/modelUrls.js +304 -98
  15. package/lib/constants/ocr/models.d.ts +882 -0
  16. package/lib/constants/ocr/models.js +182 -0
  17. package/lib/constants/ocr/symbols.d.ts +75 -0
  18. package/lib/constants/ocr/symbols.js +139 -0
  19. package/lib/{typescript/constants → constants}/sttDefaults.d.ts +0 -1
  20. package/lib/constants/sttDefaults.js +12 -10
  21. package/lib/controllers/LLMController.d.ts +47 -0
  22. package/lib/controllers/LLMController.js +14 -11
  23. package/lib/controllers/OCRController.d.ts +23 -0
  24. package/lib/controllers/OCRController.js +12 -5
  25. package/lib/controllers/SpeechToTextController.d.ts +8 -4
  26. package/lib/controllers/SpeechToTextController.js +15 -9
  27. package/lib/controllers/VerticalOCRController.d.ts +25 -0
  28. package/lib/controllers/VerticalOCRController.js +75 -0
  29. package/lib/hooks/computer_vision/useClassification.d.ts +15 -0
  30. package/lib/hooks/computer_vision/useClassification.js +7 -0
  31. package/lib/hooks/computer_vision/useImageEmbeddings.d.ts +15 -0
  32. package/lib/hooks/computer_vision/useImageEmbeddings.js +7 -0
  33. package/lib/hooks/computer_vision/useImageSegmentation.d.ts +38 -0
  34. package/lib/hooks/computer_vision/useImageSegmentation.js +7 -0
  35. package/lib/hooks/computer_vision/useOCR.d.ts +20 -0
  36. package/lib/hooks/computer_vision/useOCR.js +42 -0
  37. package/lib/hooks/computer_vision/useObjectDetection.d.ts +15 -0
  38. package/lib/hooks/computer_vision/useObjectDetection.js +7 -0
  39. package/lib/hooks/computer_vision/useStyleTransfer.d.ts +15 -0
  40. package/lib/hooks/computer_vision/useStyleTransfer.js +7 -0
  41. package/lib/hooks/computer_vision/useVerticalOCR.d.ts +21 -0
  42. package/lib/hooks/computer_vision/useVerticalOCR.js +45 -0
  43. package/lib/hooks/general/useExecutorchModule.d.ts +13 -0
  44. package/lib/hooks/general/useExecutorchModule.js +7 -0
  45. package/lib/hooks/natural_language_processing/useLLM.d.ts +10 -0
  46. package/lib/hooks/natural_language_processing/useLLM.js +78 -0
  47. package/lib/hooks/natural_language_processing/useSpeechToText.d.ts +27 -0
  48. package/lib/hooks/natural_language_processing/useSpeechToText.js +19 -14
  49. package/lib/hooks/natural_language_processing/useTextEmbeddings.d.ts +16 -0
  50. package/lib/hooks/natural_language_processing/useTextEmbeddings.js +7 -0
  51. package/lib/hooks/natural_language_processing/useTokenizer.d.ts +17 -0
  52. package/lib/hooks/natural_language_processing/useTokenizer.js +52 -0
  53. package/lib/hooks/useModule.d.ts +17 -0
  54. package/lib/hooks/useModule.js +45 -0
  55. package/lib/hooks/useNonStaticModule.d.ts +20 -0
  56. package/lib/hooks/useNonStaticModule.js +49 -0
  57. package/lib/index.d.ts +1 -1
  58. package/lib/index.js +3 -2
  59. package/lib/module/constants/modelUrls.js +61 -36
  60. package/lib/module/constants/modelUrls.js.map +1 -1
  61. package/lib/module/constants/ocr/models.js +1 -1
  62. package/lib/module/hooks/natural_language_processing/useSpeechToText.js +71 -34
  63. package/lib/module/hooks/natural_language_processing/useSpeechToText.js.map +1 -1
  64. package/lib/module/index.js +2 -3
  65. package/lib/module/index.js.map +1 -1
  66. package/lib/module/modules/natural_language_processing/SpeechToTextModule.js +72 -31
  67. package/lib/module/modules/natural_language_processing/SpeechToTextModule.js.map +1 -1
  68. package/lib/module/types/stt.js +1 -85
  69. package/lib/module/types/stt.js.map +1 -1
  70. package/lib/module/utils/ResourceFetcher.js +6 -8
  71. package/lib/module/utils/ResourceFetcher.js.map +1 -1
  72. package/lib/module/utils/ResourceFetcherUtils.js +20 -20
  73. package/lib/module/utils/ResourceFetcherUtils.js.map +1 -1
  74. package/lib/module/utils/SpeechToTextModule/ASR.js +191 -0
  75. package/lib/module/utils/SpeechToTextModule/ASR.js.map +1 -0
  76. package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js +73 -0
  77. package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js.map +1 -0
  78. package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js +56 -0
  79. package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js.map +1 -0
  80. package/lib/modules/BaseModule.d.ts +8 -0
  81. package/lib/modules/BaseModule.js +25 -0
  82. package/lib/modules/BaseNonStaticModule.d.ts +9 -0
  83. package/lib/modules/BaseNonStaticModule.js +14 -0
  84. package/lib/modules/computer_vision/ClassificationModule.d.ts +8 -0
  85. package/lib/modules/computer_vision/ClassificationModule.js +17 -0
  86. package/lib/modules/computer_vision/ImageEmbeddingsModule.d.ts +8 -0
  87. package/lib/modules/computer_vision/ImageEmbeddingsModule.js +17 -0
  88. package/lib/modules/computer_vision/ImageSegmentationModule.d.ts +11 -0
  89. package/lib/modules/computer_vision/ImageSegmentationModule.js +27 -0
  90. package/lib/modules/computer_vision/OCRModule.d.ts +15 -0
  91. package/lib/modules/computer_vision/OCRModule.js +20 -0
  92. package/lib/modules/computer_vision/ObjectDetectionModule.d.ts +9 -0
  93. package/lib/modules/computer_vision/ObjectDetectionModule.js +17 -0
  94. package/lib/modules/computer_vision/StyleTransferModule.d.ts +8 -0
  95. package/lib/modules/computer_vision/StyleTransferModule.js +17 -0
  96. package/lib/modules/computer_vision/VerticalOCRModule.d.ts +15 -0
  97. package/lib/modules/computer_vision/VerticalOCRModule.js +22 -0
  98. package/lib/modules/general/ExecutorchModule.d.ts +7 -0
  99. package/lib/modules/general/ExecutorchModule.js +14 -0
  100. package/lib/modules/natural_language_processing/LLMModule.d.ts +28 -0
  101. package/lib/modules/natural_language_processing/LLMModule.js +45 -0
  102. package/lib/modules/natural_language_processing/SpeechToTextModule.d.ts +18 -8
  103. package/lib/modules/natural_language_processing/SpeechToTextModule.js +21 -15
  104. package/lib/modules/natural_language_processing/TextEmbeddingsModule.d.ts +9 -0
  105. package/lib/modules/natural_language_processing/TextEmbeddingsModule.js +21 -0
  106. package/lib/modules/natural_language_processing/TokenizerModule.d.ts +12 -0
  107. package/lib/modules/natural_language_processing/TokenizerModule.js +5 -4
  108. package/lib/native/NativeETInstaller.d.ts +6 -0
  109. package/lib/native/NativeETInstaller.js +2 -0
  110. package/lib/native/NativeOCR.d.ts +8 -0
  111. package/lib/native/NativeOCR.js +2 -0
  112. package/lib/native/NativeVerticalOCR.d.ts +8 -0
  113. package/lib/native/NativeVerticalOCR.js +2 -0
  114. package/lib/types/common.d.ts +31 -0
  115. package/lib/types/common.js +25 -0
  116. package/lib/types/imageSegmentation.d.ts +24 -0
  117. package/lib/types/imageSegmentation.js +26 -0
  118. package/lib/types/llm.d.ts +46 -0
  119. package/lib/types/llm.js +9 -0
  120. package/lib/types/objectDetection.d.ts +104 -0
  121. package/lib/types/objectDetection.js +94 -0
  122. package/lib/types/ocr.d.ts +11 -0
  123. package/lib/types/ocr.js +1 -0
  124. package/lib/types/stt.d.ts +94 -0
  125. package/lib/types/stt.js +85 -0
  126. package/lib/typescript/constants/modelUrls.d.ts +24 -7
  127. package/lib/typescript/constants/modelUrls.d.ts.map +1 -1
  128. package/lib/typescript/constants/ocr/models.d.ts +126 -126
  129. package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts +15 -24
  130. package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts.map +1 -1
  131. package/lib/typescript/index.d.ts +2 -3
  132. package/lib/typescript/index.d.ts.map +1 -1
  133. package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts +19 -22
  134. package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts.map +1 -1
  135. package/lib/typescript/types/stt.d.ts +17 -91
  136. package/lib/typescript/types/stt.d.ts.map +1 -1
  137. package/lib/typescript/utils/ResourceFetcher.d.ts.map +1 -1
  138. package/lib/typescript/utils/ResourceFetcherUtils.d.ts.map +1 -1
  139. package/lib/typescript/utils/SpeechToTextModule/ASR.d.ts +27 -0
  140. package/lib/typescript/utils/SpeechToTextModule/ASR.d.ts.map +1 -0
  141. package/lib/typescript/utils/SpeechToTextModule/OnlineProcessor.d.ts +23 -0
  142. package/lib/typescript/utils/SpeechToTextModule/OnlineProcessor.d.ts.map +1 -0
  143. package/lib/typescript/utils/SpeechToTextModule/hypothesisBuffer.d.ts +13 -0
  144. package/lib/typescript/utils/SpeechToTextModule/hypothesisBuffer.d.ts.map +1 -0
  145. package/lib/utils/ResourceFetcher.d.ts +24 -0
  146. package/lib/utils/ResourceFetcher.js +305 -0
  147. package/lib/utils/ResourceFetcherUtils.d.ts +54 -0
  148. package/lib/utils/ResourceFetcherUtils.js +9 -0
  149. package/lib/utils/llm.d.ts +6 -0
  150. package/lib/utils/llm.js +1 -0
  151. package/lib/utils/stt.d.ts +1 -0
  152. package/lib/utils/stt.js +21 -0
  153. package/package.json +5 -3
  154. package/src/constants/modelUrls.ts +70 -37
  155. package/src/constants/ocr/models.ts +1 -1
  156. package/src/hooks/natural_language_processing/useSpeechToText.ts +87 -92
  157. package/src/index.ts +6 -8
  158. package/src/modules/natural_language_processing/SpeechToTextModule.ts +81 -69
  159. package/src/types/stt.ts +97 -92
  160. package/src/utils/ResourceFetcher.ts +9 -7
  161. package/src/utils/ResourceFetcherUtils.ts +15 -17
  162. package/src/utils/SpeechToTextModule/ASR.ts +303 -0
  163. package/src/utils/SpeechToTextModule/OnlineProcessor.ts +87 -0
  164. package/src/utils/SpeechToTextModule/hypothesisBuffer.ts +79 -0
  165. package/common/rnexecutorch/models/speech_to_text/MoonshineStrategy.cpp +0 -31
  166. package/common/rnexecutorch/models/speech_to_text/MoonshineStrategy.h +0 -21
  167. package/lib/module/constants/sttDefaults.js +0 -74
  168. package/lib/module/constants/sttDefaults.js.map +0 -1
  169. package/lib/module/controllers/SpeechToTextController.js +0 -320
  170. package/lib/module/controllers/SpeechToTextController.js.map +0 -1
  171. package/lib/typescript/constants/sttDefaults.d.ts.map +0 -1
  172. package/lib/typescript/controllers/SpeechToTextController.d.ts +0 -57
  173. package/lib/typescript/controllers/SpeechToTextController.d.ts.map +0 -1
  174. package/src/constants/sttDefaults.ts +0 -82
  175. package/src/controllers/SpeechToTextController.ts +0 -471
  176. package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/contents.xcworkspacedata +0 -7
  177. package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/xcuserdata/norbertklockiewicz.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
  178. package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/xcuserdata/norbertklockiewicz.xcuserdatad/xcschemes/xcschememanagement.plist +0 -14
@@ -0,0 +1,22 @@
1
+ import { VerticalOCRController } from '../../controllers/VerticalOCRController';
2
+ export class VerticalOCRModule {
3
+ controller;
4
+ constructor() {
5
+ this.controller = new VerticalOCRController();
6
+ }
7
+ async load(model, independentCharacters, onDownloadProgressCallback = () => { }) {
8
+ await this.controller.load({
9
+ detectorLarge: model.detectorLarge,
10
+ detectorNarrow: model.detectorNarrow,
11
+ }, {
12
+ recognizerLarge: model.recognizerLarge,
13
+ recognizerSmall: model.recognizerSmall,
14
+ }, model.language, independentCharacters, onDownloadProgressCallback);
15
+ }
16
+ async forward(input) {
17
+ return await this.controller.forward(input);
18
+ }
19
+ delete() {
20
+ this.controller.delete();
21
+ }
22
+ }
@@ -0,0 +1,7 @@
1
+ import { TensorPtr } from '../../types/common';
2
+ import { BaseNonStaticModule } from '../BaseNonStaticModule';
3
+ import { ResourceSource } from '../../types/common';
4
+ export declare class ExecutorchModule extends BaseNonStaticModule {
5
+ load(modelSource: ResourceSource, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
6
+ forward(inputTensor: TensorPtr[]): Promise<TensorPtr[]>;
7
+ }
@@ -0,0 +1,14 @@
1
+ import { BaseNonStaticModule } from '../BaseNonStaticModule';
2
+ import { ResourceFetcher } from '../../utils/ResourceFetcher';
3
+ export class ExecutorchModule extends BaseNonStaticModule {
4
+ async load(modelSource, onDownloadProgressCallback = () => { }) {
5
+ const paths = await ResourceFetcher.fetch(onDownloadProgressCallback, modelSource);
6
+ if (paths === null || paths.length < 1) {
7
+ throw new Error('Download interrupted.');
8
+ }
9
+ this.nativeModule = global.loadExecutorchModule(paths[0] || '');
10
+ }
11
+ async forward(inputTensor) {
12
+ return await this.forwardET(inputTensor);
13
+ }
14
+ }
@@ -0,0 +1,28 @@
1
+ import { ResourceSource } from '../../types/common';
2
+ import { ChatConfig, LLMTool, Message, ToolsConfig } from '../../types/llm';
3
+ export declare class LLMModule {
4
+ private controller;
5
+ constructor({ tokenCallback, responseCallback, messageHistoryCallback, }?: {
6
+ tokenCallback?: (token: string) => void;
7
+ responseCallback?: (response: string) => void;
8
+ messageHistoryCallback?: (messageHistory: Message[]) => void;
9
+ });
10
+ load(model: {
11
+ modelSource: ResourceSource;
12
+ tokenizerSource: ResourceSource;
13
+ tokenizerConfigSource: ResourceSource;
14
+ }, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
15
+ setTokenCallback({ tokenCallback, }: {
16
+ tokenCallback: (token: string) => void;
17
+ }): void;
18
+ configure({ chatConfig, toolsConfig, }: {
19
+ chatConfig?: Partial<ChatConfig>;
20
+ toolsConfig?: ToolsConfig;
21
+ }): void;
22
+ forward(input: string): Promise<string>;
23
+ generate(messages: Message[], tools?: LLMTool[]): Promise<string>;
24
+ sendMessage(message: string): Promise<Message[]>;
25
+ deleteMessage(index: number): Message[];
26
+ interrupt(): void;
27
+ delete(): void;
28
+ }
@@ -0,0 +1,45 @@
1
+ import { LLMController } from '../../controllers/LLMController';
2
+ export class LLMModule {
3
+ controller;
4
+ constructor({ tokenCallback, responseCallback, messageHistoryCallback, } = {}) {
5
+ this.controller = new LLMController({
6
+ tokenCallback,
7
+ responseCallback,
8
+ messageHistoryCallback,
9
+ });
10
+ }
11
+ async load(model, onDownloadProgressCallback = () => { }) {
12
+ await this.controller.load({
13
+ ...model,
14
+ onDownloadProgressCallback,
15
+ });
16
+ }
17
+ setTokenCallback({ tokenCallback, }) {
18
+ this.controller.setTokenCallback(tokenCallback);
19
+ }
20
+ configure({ chatConfig, toolsConfig, }) {
21
+ this.controller.configure({ chatConfig, toolsConfig });
22
+ }
23
+ async forward(input) {
24
+ await this.controller.forward(input);
25
+ return this.controller.response;
26
+ }
27
+ async generate(messages, tools) {
28
+ await this.controller.generate(messages, tools);
29
+ return this.controller.response;
30
+ }
31
+ async sendMessage(message) {
32
+ await this.controller.sendMessage(message);
33
+ return this.controller.messageHistory;
34
+ }
35
+ deleteMessage(index) {
36
+ this.controller.deleteMessage(index);
37
+ return this.controller.messageHistory;
38
+ }
39
+ interrupt() {
40
+ this.controller.interrupt();
41
+ }
42
+ delete() {
43
+ this.controller.delete();
44
+ }
45
+ }
@@ -3,12 +3,22 @@ import { SpeechToTextController } from '../../controllers/SpeechToTextController
3
3
  import { AvailableModels, SpeechToTextLanguage } from '../../types/stt';
4
4
  import { STREAMING_ACTION } from '../../constants/sttDefaults';
5
5
  export declare class SpeechToTextModule {
6
- static module: SpeechToTextController;
7
- static onDownloadProgressCallback: (_downloadProgress: number) => void;
8
- static load(modelName: AvailableModels, transcribeCallback: (sequence: string) => void, modelDownloadProgressCallback?: (downloadProgress: number) => void, encoderSource?: ResourceSource, decoderSource?: ResourceSource, tokenizerSource?: ResourceSource, overlapSeconds?: ConstructorParameters<typeof SpeechToTextController>['0']['overlapSeconds'], windowSize?: ConstructorParameters<typeof SpeechToTextController>['0']['windowSize'], streamingConfig?: ConstructorParameters<typeof SpeechToTextController>['0']['streamingConfig']): Promise<void>;
9
- static configureStreaming(overlapSeconds: Parameters<SpeechToTextController['configureStreaming']>[0], windowSize: Parameters<SpeechToTextController['configureStreaming']>[1], streamingConfig: Parameters<SpeechToTextController['configureStreaming']>[2]): void;
10
- static encode(waveform: Float32Array): Promise<null>;
11
- static decode(seq: number[]): Promise<number>;
12
- static transcribe(waveform: number[], audioLanguage?: SpeechToTextLanguage): ReturnType<SpeechToTextController['transcribe']>;
13
- static streamingTranscribe(streamAction: STREAMING_ACTION, waveform?: number[], audioLanguage?: SpeechToTextLanguage): ReturnType<SpeechToTextController['streamingTranscribe']>;
6
+ private module;
7
+ constructor({ transcribeCallback, overlapSeconds, windowSize, streamingConfig, }?: {
8
+ transcribeCallback?: (sequence: string) => void;
9
+ overlapSeconds?: ConstructorParameters<typeof SpeechToTextController>['0']['overlapSeconds'];
10
+ windowSize?: ConstructorParameters<typeof SpeechToTextController>['0']['windowSize'];
11
+ streamingConfig?: ConstructorParameters<typeof SpeechToTextController>['0']['streamingConfig'];
12
+ });
13
+ load(model: {
14
+ modelName: AvailableModels;
15
+ encoderSource?: ResourceSource;
16
+ decoderSource?: ResourceSource;
17
+ tokenizerSource?: ResourceSource;
18
+ }, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
19
+ configureStreaming(overlapSeconds: Parameters<SpeechToTextController['configureStreaming']>[0], windowSize: Parameters<SpeechToTextController['configureStreaming']>[1], streamingConfig: Parameters<SpeechToTextController['configureStreaming']>[2]): void;
20
+ encode(waveform: Float32Array): Promise<null>;
21
+ decode(seq: number[]): Promise<number>;
22
+ transcribe(waveform: number[], audioLanguage?: SpeechToTextLanguage): ReturnType<SpeechToTextController['transcribe']>;
23
+ streamingTranscribe(streamAction: STREAMING_ACTION, waveform?: number[], audioLanguage?: SpeechToTextLanguage): ReturnType<SpeechToTextController['streamingTranscribe']>;
14
24
  }
@@ -1,30 +1,36 @@
1
1
  import { SpeechToTextController } from '../../controllers/SpeechToTextController';
2
2
  export class SpeechToTextModule {
3
- static module;
4
- static onDownloadProgressCallback = (_downloadProgress) => { };
5
- static async load(modelName, transcribeCallback, modelDownloadProgressCallback, encoderSource, decoderSource, tokenizerSource, overlapSeconds, windowSize, streamingConfig) {
3
+ module;
4
+ constructor({ transcribeCallback, overlapSeconds, windowSize, streamingConfig, } = {}) {
6
5
  this.module = new SpeechToTextController({
7
- transcribeCallback: transcribeCallback,
8
- modelDownloadProgressCallback: modelDownloadProgressCallback,
9
- overlapSeconds: overlapSeconds,
10
- windowSize: windowSize,
11
- streamingConfig: streamingConfig,
6
+ transcribeCallback: transcribeCallback || (() => { }),
7
+ overlapSeconds,
8
+ windowSize,
9
+ streamingConfig,
12
10
  });
13
- await this.module.loadModel((modelName = modelName), (encoderSource = encoderSource), (decoderSource = decoderSource), (tokenizerSource = tokenizerSource));
14
11
  }
15
- static configureStreaming(overlapSeconds, windowSize, streamingConfig) {
16
- this.module?.configureStreaming(overlapSeconds, windowSize, streamingConfig);
12
+ async load(model, onDownloadProgressCallback = () => { }) {
13
+ await this.module.load({
14
+ modelName: model.modelName,
15
+ encoderSource: model.encoderSource,
16
+ decoderSource: model.decoderSource,
17
+ tokenizerSource: model.tokenizerSource,
18
+ onDownloadProgressCallback,
19
+ });
20
+ }
21
+ configureStreaming(overlapSeconds, windowSize, streamingConfig) {
22
+ this.module.configureStreaming(overlapSeconds, windowSize, streamingConfig);
17
23
  }
18
- static async encode(waveform) {
24
+ async encode(waveform) {
19
25
  return await this.module.encode(waveform);
20
26
  }
21
- static async decode(seq) {
27
+ async decode(seq) {
22
28
  return await this.module.decode(seq);
23
29
  }
24
- static async transcribe(waveform, audioLanguage) {
30
+ async transcribe(waveform, audioLanguage) {
25
31
  return await this.module.transcribe(waveform, audioLanguage);
26
32
  }
27
- static async streamingTranscribe(streamAction, waveform, audioLanguage) {
33
+ async streamingTranscribe(streamAction, waveform, audioLanguage) {
28
34
  return await this.module.streamingTranscribe(streamAction, waveform, audioLanguage);
29
35
  }
30
36
  }
@@ -0,0 +1,9 @@
1
+ import { ResourceSource } from '../../types/common';
2
+ import { BaseNonStaticModule } from '../BaseNonStaticModule';
3
+ export declare class TextEmbeddingsModule extends BaseNonStaticModule {
4
+ load(model: {
5
+ modelSource: ResourceSource;
6
+ tokenizerSource: ResourceSource;
7
+ }, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
8
+ forward(input: string): Promise<Float32Array>;
9
+ }
@@ -0,0 +1,21 @@
1
+ import { ResourceFetcher } from '../../utils/ResourceFetcher';
2
+ import { BaseNonStaticModule } from '../BaseNonStaticModule';
3
+ export class TextEmbeddingsModule extends BaseNonStaticModule {
4
+ async load(model, onDownloadProgressCallback = () => { }) {
5
+ const modelPromise = ResourceFetcher.fetch(onDownloadProgressCallback, model.modelSource);
6
+ const tokenizerPromise = ResourceFetcher.fetch(undefined, model.tokenizerSource);
7
+ const [modelResult, tokenizerResult] = await Promise.all([
8
+ modelPromise,
9
+ tokenizerPromise,
10
+ ]);
11
+ const modelPath = modelResult?.[0];
12
+ const tokenizerPath = tokenizerResult?.[0];
13
+ if (!modelPath || !tokenizerPath) {
14
+ throw new Error('Download interrupted.');
15
+ }
16
+ this.nativeModule = global.loadTextEmbeddings(modelPath, tokenizerPath);
17
+ }
18
+ async forward(input) {
19
+ return new Float32Array(await this.nativeModule.generate(input));
20
+ }
21
+ }
@@ -0,0 +1,12 @@
1
+ import { ResourceSource } from '../../types/common';
2
+ export declare class TokenizerModule {
3
+ nativeModule: any;
4
+ load(tokenizer: {
5
+ tokenizerSource: ResourceSource;
6
+ }, onDownloadProgressCallback?: (progress: number) => void): Promise<void>;
7
+ encode(s: string): Promise<any>;
8
+ decode(tokens: number[], skipSpecialTokens?: boolean): Promise<any>;
9
+ getVocabSize(): Promise<number>;
10
+ idToToken(tokenId: number): Promise<string>;
11
+ tokenToId(token: string): Promise<number>;
12
+ }
@@ -1,12 +1,13 @@
1
1
  import { ResourceFetcher } from '../../utils/ResourceFetcher';
2
2
  export class TokenizerModule {
3
3
  nativeModule;
4
- async load(modelSource, onDownloadProgressCallback = () => { }) {
5
- const paths = await ResourceFetcher.fetch(onDownloadProgressCallback, modelSource);
6
- if (paths === null || paths.length < 1) {
4
+ async load(tokenizer, onDownloadProgressCallback = () => { }) {
5
+ const paths = await ResourceFetcher.fetch(onDownloadProgressCallback, tokenizer.tokenizerSource);
6
+ const path = paths?.[0];
7
+ if (!path) {
7
8
  throw new Error('Download interrupted.');
8
9
  }
9
- this.nativeModule = global.loadTokenizerModule(paths[0] || '');
10
+ this.nativeModule = global.loadTokenizerModule(path);
10
11
  }
11
12
  async encode(s) {
12
13
  return await this.nativeModule.encode(s);
@@ -0,0 +1,6 @@
1
+ import type { TurboModule } from 'react-native';
2
+ export interface Spec extends TurboModule {
3
+ install(): boolean;
4
+ }
5
+ declare const _default: Spec | null;
6
+ export default _default;
@@ -0,0 +1,2 @@
1
+ import { TurboModuleRegistry } from 'react-native';
2
+ export default TurboModuleRegistry.get('ETInstaller');
@@ -0,0 +1,8 @@
1
+ import type { TurboModule } from 'react-native';
2
+ import { OCRDetection } from '../types/ocr';
3
+ export interface Spec extends TurboModule {
4
+ loadModule(detectorSource: string, recognizerSourceLarge: string, recognizerSourceMedium: string, recognizerSourceSmall: string, symbols: string): Promise<number>;
5
+ forward(input: string): Promise<OCRDetection[]>;
6
+ }
7
+ declare const _default: Spec | null;
8
+ export default _default;
@@ -0,0 +1,2 @@
1
+ import { TurboModuleRegistry } from 'react-native';
2
+ export default TurboModuleRegistry.get('OCR');
@@ -0,0 +1,8 @@
1
+ import type { TurboModule } from 'react-native';
2
+ import { OCRDetection } from '../types/ocr';
3
+ export interface Spec extends TurboModule {
4
+ loadModule(detectorLargeSource: string, detectorNarrowSource: string, recognizerSource: string, symbols: string, independentCharacters: boolean): Promise<number>;
5
+ forward(input: string): Promise<OCRDetection[]>;
6
+ }
7
+ declare const _default: Spec | null;
8
+ export default _default;
@@ -0,0 +1,2 @@
1
+ import { TurboModuleRegistry } from 'react-native';
2
+ export default TurboModuleRegistry.get('VerticalOCR');
@@ -0,0 +1,31 @@
1
+ export type ResourceSource = string | number | object;
2
+ export declare enum ScalarType {
3
+ BYTE = 0,
4
+ CHAR = 1,
5
+ SHORT = 2,
6
+ INT = 3,
7
+ LONG = 4,
8
+ HALF = 5,
9
+ FLOAT = 6,
10
+ DOUBLE = 7,
11
+ BOOL = 11,
12
+ QINT8 = 12,
13
+ QUINT8 = 13,
14
+ QINT32 = 14,
15
+ QUINT4X2 = 16,
16
+ QUINT2X4 = 17,
17
+ BITS16 = 22,
18
+ FLOAT8E5M2 = 23,
19
+ FLOAT8E4M3FN = 24,
20
+ FLOAT8E5M2FNUZ = 25,
21
+ FLOAT8E4M3FNUZ = 26,
22
+ UINT16 = 27,
23
+ UINT32 = 28,
24
+ UINT64 = 29
25
+ }
26
+ export type TensorBuffer = ArrayBuffer | Float32Array | Float64Array | Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array | BigInt64Array | BigUint64Array;
27
+ export interface TensorPtr {
28
+ dataPtr: TensorBuffer;
29
+ sizes: number[];
30
+ scalarType: ScalarType;
31
+ }
@@ -0,0 +1,25 @@
1
+ export var ScalarType;
2
+ (function (ScalarType) {
3
+ ScalarType[ScalarType["BYTE"] = 0] = "BYTE";
4
+ ScalarType[ScalarType["CHAR"] = 1] = "CHAR";
5
+ ScalarType[ScalarType["SHORT"] = 2] = "SHORT";
6
+ ScalarType[ScalarType["INT"] = 3] = "INT";
7
+ ScalarType[ScalarType["LONG"] = 4] = "LONG";
8
+ ScalarType[ScalarType["HALF"] = 5] = "HALF";
9
+ ScalarType[ScalarType["FLOAT"] = 6] = "FLOAT";
10
+ ScalarType[ScalarType["DOUBLE"] = 7] = "DOUBLE";
11
+ ScalarType[ScalarType["BOOL"] = 11] = "BOOL";
12
+ ScalarType[ScalarType["QINT8"] = 12] = "QINT8";
13
+ ScalarType[ScalarType["QUINT8"] = 13] = "QUINT8";
14
+ ScalarType[ScalarType["QINT32"] = 14] = "QINT32";
15
+ ScalarType[ScalarType["QUINT4X2"] = 16] = "QUINT4X2";
16
+ ScalarType[ScalarType["QUINT2X4"] = 17] = "QUINT2X4";
17
+ ScalarType[ScalarType["BITS16"] = 22] = "BITS16";
18
+ ScalarType[ScalarType["FLOAT8E5M2"] = 23] = "FLOAT8E5M2";
19
+ ScalarType[ScalarType["FLOAT8E4M3FN"] = 24] = "FLOAT8E4M3FN";
20
+ ScalarType[ScalarType["FLOAT8E5M2FNUZ"] = 25] = "FLOAT8E5M2FNUZ";
21
+ ScalarType[ScalarType["FLOAT8E4M3FNUZ"] = 26] = "FLOAT8E4M3FNUZ";
22
+ ScalarType[ScalarType["UINT16"] = 27] = "UINT16";
23
+ ScalarType[ScalarType["UINT32"] = 28] = "UINT32";
24
+ ScalarType[ScalarType["UINT64"] = 29] = "UINT64";
25
+ })(ScalarType || (ScalarType = {}));
@@ -0,0 +1,24 @@
1
+ export declare enum DeeplabLabel {
2
+ BACKGROUND = 0,
3
+ AEROPLANE = 1,
4
+ BICYCLE = 2,
5
+ BIRD = 3,
6
+ BOAT = 4,
7
+ BOTTLE = 5,
8
+ BUS = 6,
9
+ CAR = 7,
10
+ CAT = 8,
11
+ CHAIR = 9,
12
+ COW = 10,
13
+ DININGTABLE = 11,
14
+ DOG = 12,
15
+ HORSE = 13,
16
+ MOTORBIKE = 14,
17
+ PERSON = 15,
18
+ POTTEDPLANT = 16,
19
+ SHEEP = 17,
20
+ SOFA = 18,
21
+ TRAIN = 19,
22
+ TVMONITOR = 20,
23
+ ARGMAX = 21
24
+ }
@@ -0,0 +1,26 @@
1
+ /* eslint-disable @cspell/spellchecker */
2
+ export var DeeplabLabel;
3
+ (function (DeeplabLabel) {
4
+ DeeplabLabel[DeeplabLabel["BACKGROUND"] = 0] = "BACKGROUND";
5
+ DeeplabLabel[DeeplabLabel["AEROPLANE"] = 1] = "AEROPLANE";
6
+ DeeplabLabel[DeeplabLabel["BICYCLE"] = 2] = "BICYCLE";
7
+ DeeplabLabel[DeeplabLabel["BIRD"] = 3] = "BIRD";
8
+ DeeplabLabel[DeeplabLabel["BOAT"] = 4] = "BOAT";
9
+ DeeplabLabel[DeeplabLabel["BOTTLE"] = 5] = "BOTTLE";
10
+ DeeplabLabel[DeeplabLabel["BUS"] = 6] = "BUS";
11
+ DeeplabLabel[DeeplabLabel["CAR"] = 7] = "CAR";
12
+ DeeplabLabel[DeeplabLabel["CAT"] = 8] = "CAT";
13
+ DeeplabLabel[DeeplabLabel["CHAIR"] = 9] = "CHAIR";
14
+ DeeplabLabel[DeeplabLabel["COW"] = 10] = "COW";
15
+ DeeplabLabel[DeeplabLabel["DININGTABLE"] = 11] = "DININGTABLE";
16
+ DeeplabLabel[DeeplabLabel["DOG"] = 12] = "DOG";
17
+ DeeplabLabel[DeeplabLabel["HORSE"] = 13] = "HORSE";
18
+ DeeplabLabel[DeeplabLabel["MOTORBIKE"] = 14] = "MOTORBIKE";
19
+ DeeplabLabel[DeeplabLabel["PERSON"] = 15] = "PERSON";
20
+ DeeplabLabel[DeeplabLabel["POTTEDPLANT"] = 16] = "POTTEDPLANT";
21
+ DeeplabLabel[DeeplabLabel["SHEEP"] = 17] = "SHEEP";
22
+ DeeplabLabel[DeeplabLabel["SOFA"] = 18] = "SOFA";
23
+ DeeplabLabel[DeeplabLabel["TRAIN"] = 19] = "TRAIN";
24
+ DeeplabLabel[DeeplabLabel["TVMONITOR"] = 20] = "TVMONITOR";
25
+ DeeplabLabel[DeeplabLabel["ARGMAX"] = 21] = "ARGMAX";
26
+ })(DeeplabLabel || (DeeplabLabel = {}));
@@ -0,0 +1,46 @@
1
+ export interface LLMType {
2
+ messageHistory: Message[];
3
+ response: string;
4
+ token: string;
5
+ isReady: boolean;
6
+ isGenerating: boolean;
7
+ downloadProgress: number;
8
+ error: string | null;
9
+ configure: ({ chatConfig, toolsConfig, }: {
10
+ chatConfig?: Partial<ChatConfig>;
11
+ toolsConfig?: ToolsConfig;
12
+ }) => void;
13
+ generate: (messages: Message[], tools?: LLMTool[]) => Promise<void>;
14
+ sendMessage: (message: string) => Promise<void>;
15
+ deleteMessage: (index: number) => void;
16
+ interrupt: () => void;
17
+ }
18
+ export type MessageRole = 'user' | 'assistant' | 'system';
19
+ export interface Message {
20
+ role: MessageRole;
21
+ content: string;
22
+ }
23
+ export interface ToolCall {
24
+ toolName: string;
25
+ arguments: Object;
26
+ }
27
+ export type LLMTool = Object;
28
+ export interface ChatConfig {
29
+ initialMessageHistory: Message[];
30
+ contextWindowLength: number;
31
+ systemPrompt: string;
32
+ }
33
+ export interface ToolsConfig {
34
+ tools: LLMTool[];
35
+ executeToolCallback: (call: ToolCall) => Promise<string | null>;
36
+ displayToolCalls?: boolean;
37
+ }
38
+ export declare const SPECIAL_TOKENS: {
39
+ BOS_TOKEN: string;
40
+ EOS_TOKEN: string;
41
+ UNK_TOKEN: string;
42
+ SEP_TOKEN: string;
43
+ PAD_TOKEN: string;
44
+ CLS_TOKEN: string;
45
+ MASK_TOKEN: string;
46
+ };
@@ -0,0 +1,9 @@
1
+ export const SPECIAL_TOKENS = {
2
+ BOS_TOKEN: 'bos_token',
3
+ EOS_TOKEN: 'eos_token',
4
+ UNK_TOKEN: 'unk_token',
5
+ SEP_TOKEN: 'sep_token',
6
+ PAD_TOKEN: 'pad_token',
7
+ CLS_TOKEN: 'cls_token',
8
+ MASK_TOKEN: 'mask_token',
9
+ };
@@ -0,0 +1,104 @@
1
+ export interface Bbox {
2
+ x1: number;
3
+ x2: number;
4
+ y1: number;
5
+ y2: number;
6
+ }
7
+ export interface Detection {
8
+ bbox: Bbox;
9
+ label: keyof typeof CocoLabel;
10
+ score: number;
11
+ }
12
+ declare enum CocoLabel {
13
+ PERSON = 1,
14
+ BICYCLE = 2,
15
+ CAR = 3,
16
+ MOTORCYCLE = 4,
17
+ AIRPLANE = 5,
18
+ BUS = 6,
19
+ TRAIN = 7,
20
+ TRUCK = 8,
21
+ BOAT = 9,
22
+ TRAFFIC_LIGHT = 10,
23
+ FIRE_HYDRANT = 11,
24
+ STREET_SIGN = 12,
25
+ STOP_SIGN = 13,
26
+ PARKING = 14,
27
+ BENCH = 15,
28
+ BIRD = 16,
29
+ CAT = 17,
30
+ DOG = 18,
31
+ HORSE = 19,
32
+ SHEEP = 20,
33
+ COW = 21,
34
+ ELEPHANT = 22,
35
+ BEAR = 23,
36
+ ZEBRA = 24,
37
+ GIRAFFE = 25,
38
+ HAT = 26,
39
+ BACKPACK = 27,
40
+ UMBRELLA = 28,
41
+ SHOE = 29,
42
+ EYE = 30,
43
+ HANDBAG = 31,
44
+ TIE = 32,
45
+ SUITCASE = 33,
46
+ FRISBEE = 34,
47
+ SKIS = 35,
48
+ SNOWBOARD = 36,
49
+ SPORTS = 37,
50
+ KITE = 38,
51
+ BASEBALL = 39,
52
+ SKATEBOARD = 41,
53
+ SURFBOARD = 42,
54
+ TENNIS_RACKET = 43,
55
+ BOTTLE = 44,
56
+ PLATE = 45,
57
+ WINE_GLASS = 46,
58
+ CUP = 47,
59
+ FORK = 48,
60
+ KNIFE = 49,
61
+ SPOON = 50,
62
+ BOWL = 51,
63
+ BANANA = 52,
64
+ APPLE = 53,
65
+ SANDWICH = 54,
66
+ ORANGE = 55,
67
+ BROCCOLI = 56,
68
+ CARROT = 57,
69
+ HOT_DOG = 58,
70
+ PIZZA = 59,
71
+ DONUT = 60,
72
+ CAKE = 61,
73
+ CHAIR = 62,
74
+ COUCH = 63,
75
+ POTTED_PLANT = 64,
76
+ BED = 65,
77
+ MIRROR = 66,
78
+ DINING_TABLE = 67,
79
+ WINDOW = 68,
80
+ DESK = 69,
81
+ TOILET = 70,
82
+ DOOR = 71,
83
+ TV = 72,
84
+ LAPTOP = 73,
85
+ MOUSE = 74,
86
+ REMOTE = 75,
87
+ KEYBOARD = 76,
88
+ CELL_PHONE = 77,
89
+ MICROWAVE = 78,
90
+ OVEN = 79,
91
+ TOASTER = 80,
92
+ SINK = 81,
93
+ REFRIGERATOR = 82,
94
+ BLENDER = 83,
95
+ BOOK = 84,
96
+ CLOCK = 85,
97
+ VASE = 86,
98
+ SCISSORS = 87,
99
+ TEDDY_BEAR = 88,
100
+ HAIR_DRIER = 89,
101
+ TOOTHBRUSH = 90,
102
+ HAIR_BRUSH = 91
103
+ }
104
+ export {};