react-native-executorch 0.4.8 → 0.5.1-rc.0

package/src/constants/ocr/symbols.ts

@@ -19,130 +19,130 @@ export const alphabets = {
 };
 
 export const symbols = {
-  //Abaza
+  // Abaza
   abq: alphabets.cyrillic,
-  //Adyghe
+  // Adyghe
   ady: alphabets.cyrillic,
-  //Africans
+  // Africans
   af: alphabets.latin,
-  //Avar
+  // Avar
   ava: alphabets.cyrillic,
-  //Azerbaijani
+  // Azerbaijani
   az: alphabets.latin,
-  //Belarusian
+  // Belarusian
   be: alphabets.cyrillic,
-  //Bulgarian
+  // Bulgarian
   bg: alphabets.cyrillic,
-  //Bosnian
+  // Bosnian
   bs: alphabets.latin,
-  //Simplified Chinese
+  // Simplified Chinese
   chSim: alphabets.zhSim,
-  //Chechen
+  // Chechen
   che: alphabets.cyrillic,
-  //Czech
+  // Czech
   cs: alphabets.latin,
-  //Welsh
+  // Welsh
   cy: alphabets.latin,
-  //Danish
+  // Danish
   da: alphabets.latin,
-  //Dargwa
+  // Dargwa
   dar: alphabets.cyrillic,
-  //German
+  // German
   de: alphabets.latin,
-  //English
+  // English
   en: alphabets.english,
-  //Spanish
+  // Spanish
   es: alphabets.latin,
-  //Estonian
+  // Estonian
   et: alphabets.latin,
-  //French
+  // French
   fr: alphabets.latin,
-  //Irish
+  // Irish
   ga: alphabets.latin,
-  //Croatian
+  // Croatian
   hr: alphabets.latin,
-  //Hungarian
+  // Hungarian
   hu: alphabets.latin,
-  //Indonesian
+  // Indonesian
   id: alphabets.latin,
-  //Ingush
+  // Ingush
   inh: alphabets.cyrillic,
-  //Icelandic
+  // Icelandic
   ic: alphabets.latin,
-  //Italian
+  // Italian
   it: alphabets.latin,
-  //Japanese
+  // Japanese
   ja: alphabets.japanese,
-  //Karbadian
+  // Karbadian
   kbd: alphabets.cyrillic,
-  //Kannada
+  // Kannada
   kn: alphabets.kannada,
-  //Korean
+  // Korean
   ko: alphabets.korean,
-  //Kurdish
+  // Kurdish
   ku: alphabets.latin,
-  //Latin
+  // Latin
   la: alphabets.latin,
-  //Lak
+  // Lak
   lbe: alphabets.cyrillic,
-  //Lezghian
+  // Lezghian
   lez: alphabets.cyrillic,
-  //Lithuanian
+  // Lithuanian
   lt: alphabets.latin,
-  //Latvian
+  // Latvian
   lv: alphabets.latin,
-  //Maori
+  // Maori
   mi: alphabets.latin,
-  //Mongolian
+  // Mongolian
   mn: alphabets.cyrillic,
-  //Malay
+  // Malay
   ms: alphabets.latin,
-  //Maltese
+  // Maltese
   mt: alphabets.latin,
-  //Dutch
+  // Dutch
   nl: alphabets.latin,
-  //Norwegian
+  // Norwegian
   no: alphabets.latin,
-  //Occitan
+  // Occitan
   oc: alphabets.latin,
-  //Pali
+  // Pali
   pi: alphabets.latin,
-  //Polish
+  // Polish
   pl: alphabets.latin,
-  //Portuguese
+  // Portuguese
   pt: alphabets.latin,
-  //Romanian
+  // Romanian
   ro: alphabets.latin,
-  //Russian
+  // Russian
   ru: alphabets.cyrillic,
-  //Serbian (cyrillic)
+  // Serbian (cyrillic)
   rsCyrillic: alphabets.cyrillic,
-  //Serbian (latin)
+  // Serbian (latin)
   rsLatin: alphabets.latin,
-  //Slovak
+  // Slovak
   sk: alphabets.latin,
-  //Slovenian
+  // Slovenian
   sl: alphabets.latin,
-  //Albanian
+  // Albanian
   sq: alphabets.latin,
-  //Swedish
+  // Swedish
   sv: alphabets.latin,
-  //Swahili
+  // Swahili
   sw: alphabets.latin,
-  //Tabassaran
+  // Tabassaran
   tab: alphabets.cyrillic,
-  //Telugu
+  // Telugu
   te: alphabets.telugu,
-  //Tajik
+  // Tajik
   tjk: alphabets.cyrillic,
-  //Tagalog
+  // Tagalog
   tl: alphabets.latin,
-  //Turkish:
+  // Turkish
   tr: alphabets.latin,
-  //Ukrainian
+  // Ukrainian
   uk: alphabets.cyrillic,
-  //Uzbek
+  // Uzbek
   uz: alphabets.latin,
-  //Vietnamese
+  // Vietnamese
   vi: alphabets.latin,
 };

package/src/constants/sttDefaults.ts

@@ -1,13 +1,7 @@
 import {
-  MOONSHINE_TINY_ENCODER,
-  MOONSHINE_TINY_DECODER,
-  MOONSHINE_TOKENIZER,
-  WHISPER_TINY_ENCODER,
-  WHISPER_TINY_DECODER,
-  WHISPER_TOKENIZER,
-  WHISPER_TINY_MULTILINGUAL_ENCODER,
-  WHISPER_TINY_MULTILINGUAL_DECODER,
-  WHISPER_TINY_MULTILINGUAL_TOKENIZER,
+  MOONSHINE_TINY,
+  WHISPER_TINY,
+  WHISPER_TINY_MULTILINGUAL,
 } from './modelUrls';
 import { AvailableModels, ModelConfig } from '../types/stt';
 
@@ -17,11 +11,11 @@ export const HAMMING_DIST_THRESHOLD = 1;
 
 const whisperTinyModelConfig = {
   sources: {
-    encoder: WHISPER_TINY_ENCODER,
-    decoder: WHISPER_TINY_DECODER,
+    encoder: WHISPER_TINY.encoderSource,
+    decoder: WHISPER_TINY.decoderSource,
   },
   tokenizer: {
-    source: WHISPER_TOKENIZER,
+    source: WHISPER_TINY.tokenizerSource,
     bos: 50257, // FIXME: this is a placeholder and needs to be changed
     eos: 50256, // FIXME: this is a placeholder and needs to be changed
   },
@@ -30,11 +24,11 @@ const whisperTinyModelConfig = {
 
 const moonshineTinyModelConfig = {
   sources: {
-    encoder: MOONSHINE_TINY_ENCODER,
-    decoder: MOONSHINE_TINY_DECODER,
+    encoder: MOONSHINE_TINY.encoderSource,
+    decoder: MOONSHINE_TINY.decoderSource,
   },
   tokenizer: {
-    source: MOONSHINE_TOKENIZER,
+    source: MOONSHINE_TINY.tokenizerSource,
     bos: 1, // FIXME: this is a placeholder and needs to be changed
     eos: 2, // FIXME: this is a placeholder and needs to be changed
   },
@@ -43,11 +37,11 @@ const moonshineTinyModelConfig = {
 
 const whisperTinyMultilingualModelConfig = {
   sources: {
-    encoder: WHISPER_TINY_MULTILINGUAL_ENCODER,
-    decoder: WHISPER_TINY_MULTILINGUAL_DECODER,
+    encoder: WHISPER_TINY_MULTILINGUAL.encoderSource,
+    decoder: WHISPER_TINY_MULTILINGUAL.decoderSource,
   },
   tokenizer: {
-    source: WHISPER_TINY_MULTILINGUAL_TOKENIZER,
+    source: WHISPER_TINY_MULTILINGUAL.tokenizerSource,
     bos: 50258, // FIXME: this is a placeholder and needs to be changed
     eos: 50257, // FIXME: this is a placeholder and needs to be changed
   },
@@ -84,3 +78,5 @@ export enum STREAMING_ACTION {
   DATA,
   STOP,
 }
+
+export { AvailableModels };

package/src/controllers/LLMController.ts

@@ -14,6 +14,7 @@ import {
 } from '../types/llm';
 import { LLMNativeModule } from '../native/RnExecutorchModules';
 import { parseToolCall } from '../utils/llm';
+import { Logger } from '../common/Logger';
 
 export class LLMController {
   private nativeModule: typeof LLMNativeModule;
@@ -32,9 +33,6 @@ export class LLMController {
   private messageHistoryCallback: (messageHistory: Message[]) => void;
   private isReadyCallback: (isReady: boolean) => void;
   private isGeneratingCallback: (isGenerating: boolean) => void;
-  private onDownloadProgressCallback:
-    | ((downloadProgress: number) => void)
-    | undefined;
 
   constructor({
     tokenCallback,
@@ -42,17 +40,15 @@ export class LLMController {
     messageHistoryCallback,
     isReadyCallback,
     isGeneratingCallback,
-    onDownloadProgressCallback,
   }: {
     tokenCallback?: (token: string) => void;
     responseCallback?: (response: string) => void;
     messageHistoryCallback?: (messageHistory: Message[]) => void;
     isReadyCallback?: (isReady: boolean) => void;
     isGeneratingCallback?: (isGenerating: boolean) => void;
-    onDownloadProgressCallback?: (downloadProgress: number) => void;
   }) {
     if (responseCallback !== undefined) {
-      console.warn(
+      Logger.warn(
         'Passing response callback is deprecated and will be removed in 0.6.0'
       );
     }
@@ -75,9 +71,6 @@ export class LLMController {
       this._isGenerating = isGenerating;
       isGeneratingCallback?.(isGenerating);
     };
-
-    this.onDownloadProgressCallback = onDownloadProgressCallback;
-
     this.nativeModule = LLMNativeModule;
   }
 
@@ -98,10 +91,12 @@ export class LLMController {
     modelSource,
     tokenizerSource,
     tokenizerConfigSource,
+    onDownloadProgressCallback,
   }: {
     modelSource: ResourceSource;
     tokenizerSource: ResourceSource;
     tokenizerConfigSource: ResourceSource;
+    onDownloadProgressCallback?: (downloadProgress: number) => void;
   }) {
     // reset inner state when loading new model
     this.responseCallback('');
@@ -110,20 +105,35 @@ export class LLMController {
     this.isReadyCallback(false);
 
     try {
-      const tokenizerFileUri = await ResourceFetcher.fetch(tokenizerSource);
-      const tokenizerConfigFileUri = await ResourceFetcher.fetch(
+      const tokenizersPromise = ResourceFetcher.fetch(
+        undefined,
+        tokenizerSource,
         tokenizerConfigSource
       );
-      this.tokenizerConfig = JSON.parse(
-        await readAsStringAsync('file://' + tokenizerConfigFileUri)
+
+      const modelPromise = ResourceFetcher.fetch(
+        onDownloadProgressCallback,
+        modelSource
       );
 
-      const modelFileUri = await ResourceFetcher.fetch(
-        modelSource,
-        this.onDownloadProgressCallback
+      const [tokenizersResults, modelResult] = await Promise.all([
+        tokenizersPromise,
+        modelPromise,
+      ]);
+
+      const tokenizerPath = tokenizersResults?.[0];
+      const tokenizerConfigPath = tokenizersResults?.[1];
+      const modelPath = modelResult?.[0];
+
+      if (!tokenizerPath || !tokenizerConfigPath || !modelPath) {
+        throw new Error('Download interrupted!');
+      }
+
+      this.tokenizerConfig = JSON.parse(
+        await readAsStringAsync('file://' + tokenizerConfigPath!)
       );
 
-      await this.nativeModule.loadLLM(modelFileUri, tokenizerFileUri);
+      await this.nativeModule.loadLLM(modelPath, tokenizerPath);
       this.isReadyCallback(true);
       this.onToken = this.nativeModule.onToken((data: string) => {
         if (
@@ -209,7 +219,7 @@ export class LLMController {
       throw new Error(`Empty 'messages' array!`);
     }
     if (messages[0] && messages[0].role !== 'system') {
-      console.warn(
+      Logger.warn(
         `You are not providing system prompt. You can pass it in the first message using { role: 'system', content: YOUR_PROMPT }. Otherwise prompt from your model's chat template will be used.`
       );
     }

package/src/controllers/OCRController.ts

@@ -1,41 +1,37 @@
 import { symbols } from '../constants/ocr/symbols';
 import { ETError, getError } from '../Error';
-import { OCRNativeModule } from '../native/RnExecutorchModules';
 import { ResourceSource } from '../types/common';
 import { OCRLanguage } from '../types/ocr';
 import { ResourceFetcher } from '../utils/ResourceFetcher';
 
 export class OCRController {
-  private nativeModule: typeof OCRNativeModule;
+  private nativeModule: any;
   public isReady: boolean = false;
   public isGenerating: boolean = false;
   public error: string | null = null;
-  private modelDownloadProgressCallback: (downloadProgress: number) => void;
   private isReadyCallback: (isReady: boolean) => void;
   private isGeneratingCallback: (isGenerating: boolean) => void;
   private errorCallback: (error: string) => void;
 
   constructor({
-    modelDownloadProgressCallback = (_downloadProgress: number) => {},
     isReadyCallback = (_isReady: boolean) => {},
     isGeneratingCallback = (_isGenerating: boolean) => {},
     errorCallback = (_error: string) => {},
-  }) {
-    this.nativeModule = OCRNativeModule;
-    this.modelDownloadProgressCallback = modelDownloadProgressCallback;
+  } = {}) {
     this.isReadyCallback = isReadyCallback;
     this.isGeneratingCallback = isGeneratingCallback;
     this.errorCallback = errorCallback;
   }
 
-  public loadModel = async (
+  public load = async (
     detectorSource: ResourceSource,
     recognizerSources: {
       recognizerLarge: ResourceSource;
       recognizerMedium: ResourceSource;
       recognizerSmall: ResourceSource;
     },
-    language: OCRLanguage
+    language: OCRLanguage,
+    onDownloadProgressCallback?: (downloadProgress: number) => void
   ) => {
     try {
       if (!detectorSource || Object.keys(recognizerSources).length !== 3)
@@ -48,22 +44,23 @@ export class OCRController {
       this.isReady = false;
       this.isReadyCallback(false);
 
-      const paths = await ResourceFetcher.fetchMultipleResources(
-        this.modelDownloadProgressCallback,
+      const paths = await ResourceFetcher.fetch(
+        onDownloadProgressCallback,
         detectorSource,
         recognizerSources.recognizerLarge,
         recognizerSources.recognizerMedium,
         recognizerSources.recognizerSmall
       );
-
-      await this.nativeModule.loadModule(
+      if (paths === null || paths?.length < 4) {
+        throw new Error('Download interrupted!');
+      }
+      this.nativeModule = global.loadOCR(
         paths[0]!,
         paths[1]!,
         paths[2]!,
         paths[3]!,
         symbols[language]
       );
-
       this.isReady = true;
       this.isReadyCallback(this.isReady);
     } catch (e) {
@@ -86,7 +83,7 @@ export class OCRController {
     try {
       this.isGenerating = true;
       this.isGeneratingCallback(this.isGenerating);
-      return await this.nativeModule.forward(input);
+      return await this.nativeModule.generate(input);
     } catch (e) {
       throw new Error(getError(e));
     } finally {
@@ -94,4 +91,16 @@ export class OCRController {
       this.isGeneratingCallback(this.isGenerating);
     }
   };
+
+  public delete() {
+    if (this.isGenerating) {
+      throw new Error(
+        getError(ETError.ModelGenerating) +
+          'You cannot delete the model. You must wait until the generating is finished.'
+      );
+    }
+    this.nativeModule.unload();
+    this.isReadyCallback(false);
+    this.isGeneratingCallback(false);
+  }
 }

package/src/controllers/SpeechToTextController.ts

@@ -7,21 +7,22 @@ import {
   STREAMING_ACTION,
 } from '../constants/sttDefaults';
 import { AvailableModels, ModelConfig } from '../types/stt';
-import { SpeechToTextNativeModule } from '../native/RnExecutorchModules';
 import { TokenizerModule } from '../modules/natural_language_processing/TokenizerModule';
 import { ResourceSource } from '../types/common';
 import { ResourceFetcher } from '../utils/ResourceFetcher';
 import { longCommonInfPref } from '../utils/stt';
 import { SpeechToTextLanguage } from '../types/stt';
 import { ETError, getError } from '../Error';
+import { Logger } from '../common/Logger';
 
 export class SpeechToTextController {
-  private speechToTextNativeModule = SpeechToTextNativeModule;
+  private speechToTextNativeModule: any;
 
   public sequence: number[] = [];
   public isReady = false;
   public isGenerating = false;
 
+  private tokenizerModule: TokenizerModule;
   private overlapSeconds!: number;
   private windowSize!: number;
   private chunks: number[][] = [];
@@ -33,9 +34,6 @@ export class SpeechToTextController {
 
   // User callbacks
   private decodedTranscribeCallback: (sequence: number[]) => void;
-  private modelDownloadProgressCallback:
-    | ((downloadProgress: number) => void)
-    | undefined;
   private isReadyCallback: (isReady: boolean) => void;
   private isGeneratingCallback: (isGenerating: boolean) => void;
   private onErrorCallback: (error: any) => void;
@@ -43,7 +41,6 @@ export class SpeechToTextController {
 
   constructor({
     transcribeCallback,
-    modelDownloadProgressCallback,
     isReadyCallback,
     isGeneratingCallback,
     onErrorCallback,
@@ -52,7 +49,6 @@ export class SpeechToTextController {
     streamingConfig,
   }: {
     transcribeCallback: (sequence: string) => void;
-    modelDownloadProgressCallback?: (downloadProgress: number) => void;
     isReadyCallback?: (isReady: boolean) => void;
     isGeneratingCallback?: (isGenerating: boolean) => void;
     onErrorCallback?: (error: Error | undefined) => void;
@@ -60,9 +56,9 @@ export class SpeechToTextController {
     windowSize?: number;
     streamingConfig?: keyof typeof MODES;
   }) {
+    this.tokenizerModule = new TokenizerModule();
     this.decodedTranscribeCallback = async (seq) =>
       transcribeCallback(await this.tokenIdsToText(seq));
-    this.modelDownloadProgressCallback = modelDownloadProgressCallback;
     this.isReadyCallback = (isReady) => {
       this.isReady = isReady;
       isReadyCallback?.(isReady);
@@ -86,27 +82,41 @@ export class SpeechToTextController {
     );
   }
 
-  public async loadModel(
-    modelName: AvailableModels,
-    encoderSource?: ResourceSource,
-    decoderSource?: ResourceSource,
-    tokenizerSource?: ResourceSource
-  ) {
+  public async load({
+    modelName,
+    encoderSource,
+    decoderSource,
+    tokenizerSource,
+    onDownloadProgressCallback,
+  }: {
+    modelName: AvailableModels;
+    encoderSource?: ResourceSource;
+    decoderSource?: ResourceSource;
+    tokenizerSource?: ResourceSource;
+    onDownloadProgressCallback?: (downloadProgress: number) => void;
+  }) {
     this.onErrorCallback(undefined);
     this.isReadyCallback(false);
     this.config = MODEL_CONFIGS[modelName];
 
     try {
-      await TokenizerModule.load(
-        tokenizerSource || this.config.tokenizer.source
+      const tokenizerLoadPromise = this.tokenizerModule.load({
+        tokenizerSource: tokenizerSource || this.config.tokenizer.source,
+      });
+      const pathsPromise = ResourceFetcher.fetch(
+        onDownloadProgressCallback,
+        encoderSource || this.config.sources.encoder,
+        decoderSource || this.config.sources.decoder
       );
-
-      [encoderSource, decoderSource] =
-        await ResourceFetcher.fetchMultipleResources(
-          this.modelDownloadProgressCallback,
-          encoderSource || this.config.sources.encoder,
-          decoderSource || this.config.sources.decoder
-        );
+      const [_, encoderDecoderResults] = await Promise.all([
+        tokenizerLoadPromise,
+        pathsPromise,
+      ]);
+      encoderSource = encoderDecoderResults?.[0];
+      decoderSource = encoderDecoderResults?.[1];
+      if (!encoderSource || !decoderSource) {
+        throw new Error('Download interrupted.');
+      }
     } catch (e) {
       this.onErrorCallback(e);
       return;
@@ -117,15 +127,16 @@ export class SpeechToTextController {
       // create a separate class for multilingual version of Whisper, since it is the same. We just need
       // the distinction here, in TS, for start tokens and such. If we introduce
       // more versions of Whisper, such as the small one, this should be refactored.
-      modelName = 'whisper';
+      modelName = AvailableModels.WHISPER;
     }
 
     try {
-      await this.speechToTextNativeModule.loadModule(modelName, [
-        encoderSource!,
-        decoderSource!,
-      ]);
-      this.modelDownloadProgressCallback?.(1);
+      const nativeSpeechToText = await global.loadSpeechToText(
+        encoderSource,
+        decoderSource,
+        modelName
+      );
+      this.speechToTextNativeModule = nativeSpeechToText;
       this.isReadyCallback(true);
     } catch (e) {
       this.onErrorCallback(e);
@@ -142,14 +153,14 @@ export class SpeechToTextController {
       this.overlapSeconds = MODES[streamingConfig].overlapSeconds * SECOND;
     }
     if (streamingConfig && (windowSize || overlapSeconds)) {
-      console.warn(
+      Logger.warn(
         `windowSize and overlapSeconds overrides values from streamingConfig ${streamingConfig}.`
       );
     }
     this.windowSize = (windowSize || 0) * SECOND || this.windowSize;
     this.overlapSeconds = (overlapSeconds || 0) * SECOND || this.overlapSeconds;
     if (2 * this.overlapSeconds + this.windowSize >= 30 * SECOND) {
-      console.warn(
+      Logger.warn(
         `Invalid values for overlapSeconds and/or windowSize provided. Expected windowSize + 2 * overlapSeconds (== ${this.windowSize + 2 * this.overlapSeconds}) <= 30. Setting windowSize to ${30 * SECOND - 2 * this.overlapSeconds}.`
       );
       this.windowSize = 30 * SECOND - 2 * this.overlapSeconds;
@@ -196,10 +207,13 @@ export class SpeechToTextController {
       return [this.config.tokenizer.bos];
     }
     // FIXME: I should use .getTokenId for the BOS as well, should remove it from config
-    const langTokenId = await TokenizerModule.tokenToId(`<|${audioLanguage}|>`);
-    const transcribeTokenId = await TokenizerModule.tokenToId('<|transcribe|>');
+    const langTokenId = await this.tokenizerModule.tokenToId(
+      `<|${audioLanguage}|>`
+    );
+    const transcribeTokenId =
+      await this.tokenizerModule.tokenToId('<|transcribe|>');
     const noTimestampsTokenId =
-      await TokenizerModule.tokenToId('<|notimestamps|>');
+      await this.tokenizerModule.tokenToId('<|notimestamps|>');
     const startingTokenIds = [
       this.config.tokenizer.bos,
       langTokenId,
@@ -217,7 +231,7 @@ export class SpeechToTextController {
     let prevSeqTokenIdx = 0;
     this.prevSeq = this.sequence.slice();
     try {
-      await this.encode(chunk);
+      await this.encode(new Float32Array(chunk));
     } catch (error) {
       this.onErrorCallback(new Error(getError(error) + ' encoding error'));
       return [];
@@ -294,7 +308,7 @@ export class SpeechToTextController {
 
   private async tokenIdsToText(tokenIds: number[]): Promise<string> {
     try {
-      return TokenizerModule.decode(tokenIds, true);
+      return await this.tokenizerModule.decode(tokenIds, true);
     } catch (e) {
       this.onErrorCallback(
         new Error(`An error has occurred when decoding the token ids: ${e}`)
@@ -320,7 +334,6 @@ export class SpeechToTextController {
 
     // Making sure that the error is not set when we get there
     this.isGeneratingCallback(true);
-
     this.resetState();
     this.waveform = waveform;
     this.chunkWaveform();
@@ -448,11 +461,11 @@ export class SpeechToTextController {
     return decodedText;
   }
 
-  public async encode(waveform: number[]) {
+  public async encode(waveform: Float32Array): Promise<null> {
     return await this.speechToTextNativeModule.encode(waveform);
   }
 
-  public async decode(seq: number[], encodings?: number[]) {
-    return await this.speechToTextNativeModule.decode(seq, encodings || []);
+  public async decode(seq: number[]): Promise<number> {
+    return await this.speechToTextNativeModule.decode(seq);
   }
 }