react-native-executorch 0.4.10 → 0.5.0-nightly-6f4dd53-20251211

package/src/hooks/natural_language_processing/useSpeechToText.ts

@@ -1,108 +1,116 @@
-import { useEffect, useMemo, useState } from 'react';
-import { SpeechToTextController } from '../../controllers/SpeechToTextController';
-import { ResourceSource } from '../../types/common';
-import { STREAMING_ACTION } from '../../constants/sttDefaults';
-import { AvailableModels, SpeechToTextLanguage } from '../../types/stt';
-
-interface SpeechToTextModule {
-  isReady: boolean;
-  isGenerating: boolean;
-  sequence: string;
-  downloadProgress: number;
-  configureStreaming: SpeechToTextController['configureStreaming'];
-  error: Error | undefined;
-  transcribe: (
-    input: number[],
-    audioLanguage?: SpeechToTextLanguage
-  ) => ReturnType<SpeechToTextController['transcribe']>;
-  streamingTranscribe: (
-    streamAction: STREAMING_ACTION,
-    input?: number[],
-    audioLanguage?: SpeechToTextLanguage
-  ) => ReturnType<SpeechToTextController['streamingTranscribe']>;
-}
+import { useEffect, useCallback, useState } from 'react';
+import { ETError, getError } from '../../Error';
+import { SpeechToTextModule } from '../../modules/natural_language_processing/SpeechToTextModule';
+import { DecodingOptions, SpeechToTextModelConfig } from '../../types/stt';
 
 export const useSpeechToText = ({
-  modelName,
-  encoderSource,
-  decoderSource,
-  tokenizerSource,
-  overlapSeconds,
-  windowSize,
-  streamingConfig,
+  model,
   preventLoad = false,
 }: {
-  modelName: AvailableModels;
-  encoderSource?: ResourceSource;
-  decoderSource?: ResourceSource;
-  tokenizerSource?: ResourceSource;
-  overlapSeconds?: ConstructorParameters<
-    typeof SpeechToTextController
-  >['0']['overlapSeconds'];
-  windowSize?: ConstructorParameters<
-    typeof SpeechToTextController
-  >['0']['windowSize'];
-  streamingConfig?: ConstructorParameters<
-    typeof SpeechToTextController
-  >['0']['streamingConfig'];
+  model: SpeechToTextModelConfig;
   preventLoad?: boolean;
-}): SpeechToTextModule => {
-  const [sequence, setSequence] = useState<string>('');
+}) => {
+  const [error, setError] = useState<null | string>(null);
   const [isReady, setIsReady] = useState(false);
-  const [downloadProgress, setDownloadProgress] = useState(0);
   const [isGenerating, setIsGenerating] = useState(false);
-  const [error, setError] = useState<Error | undefined>();
-
-  const model = useMemo(
-    () =>
-      new SpeechToTextController({
-        transcribeCallback: setSequence,
-        isReadyCallback: setIsReady,
-        isGeneratingCallback: setIsGenerating,
-        onErrorCallback: setError,
-        modelDownloadProgressCallback: setDownloadProgress,
-      }),
-    []
-  );
+  const [downloadProgress, setDownloadProgress] = useState(0);
 
-  useEffect(() => {
-    model.configureStreaming(overlapSeconds, windowSize, streamingConfig);
-  }, [model, overlapSeconds, windowSize, streamingConfig]);
+  const [modelInstance] = useState(() => new SpeechToTextModule());
+  const [committedTranscription, setCommittedTranscription] = useState('');
+  const [nonCommittedTranscription, setNonCommittedTranscription] =
+    useState('');
 
   useEffect(() => {
-    const loadModel = async () => {
-      await model.loadModel(
-        modelName,
-        encoderSource,
-        decoderSource,
-        tokenizerSource
-      );
-    };
-    if (!preventLoad) {
-      loadModel();
-    }
+    if (preventLoad) return;
+    (async () => {
+      setDownloadProgress(0);
+      setError(null);
+      try {
+        setIsReady(false);
+        await modelInstance.load(
+          {
+            isMultilingual: model.isMultilingual,
+            encoderSource: model.encoderSource,
+            decoderSource: model.decoderSource,
+            tokenizerSource: model.tokenizerSource,
+          },
+          setDownloadProgress
+        );
+        setIsReady(true);
+      } catch (err) {
+        setError((err as Error).message);
+      }
+    })();
   }, [
-    model,
-    modelName,
-    encoderSource,
-    decoderSource,
-    tokenizerSource,
+    modelInstance,
+    model.isMultilingual,
+    model.encoderSource,
+    model.decoderSource,
+    model.tokenizerSource,
     preventLoad,
   ]);
 
+  const stateWrapper = useCallback(
+    <T extends (...args: any[]) => Promise<any>>(fn: T) =>
+      async (...args: Parameters<T>): Promise<Awaited<ReturnType<T>>> => {
+        if (!isReady) throw new Error(getError(ETError.ModuleNotLoaded));
+        if (isGenerating) throw new Error(getError(ETError.ModelGenerating));
+        setIsGenerating(true);
+        try {
+          return await fn.apply(modelInstance, args);
+        } finally {
+          setIsGenerating(false);
+        }
+      },
+    [isReady, isGenerating, modelInstance]
+  );
+
+  const stream = useCallback(
+    async (options?: DecodingOptions) => {
+      if (!isReady) throw new Error(getError(ETError.ModuleNotLoaded));
+      if (isGenerating) throw new Error(getError(ETError.ModelGenerating));
+      setIsGenerating(true);
+      setCommittedTranscription('');
+      setNonCommittedTranscription('');
+      let transcription = '';
+      try {
+        for await (const { committed, nonCommitted } of modelInstance.stream(
+          options
+        )) {
+          setCommittedTranscription((prev) => prev + committed);
+          setNonCommittedTranscription(nonCommitted);
+          transcription += committed;
+        }
+      } finally {
+        setIsGenerating(false);
+      }
+      return transcription;
+    },
+    [isReady, isGenerating, modelInstance]
+  );
+
+  const wrapper = useCallback(
+    <T extends (...args: any[]) => any>(fn: T) => {
+      return (...args: Parameters<T>): ReturnType<T> => {
+        if (!isReady) throw new Error(getError(ETError.ModuleNotLoaded));
+        return fn.apply(modelInstance, args);
+      };
+    },
+    [isReady, modelInstance]
+  );
+
   return {
+    error,
     isReady,
     isGenerating,
     downloadProgress,
-    configureStreaming: model.configureStreaming,
-    sequence,
-    error,
-    transcribe: (waveform: number[], audioLanguage?: SpeechToTextLanguage) =>
-      model.transcribe(waveform, audioLanguage),
-    streamingTranscribe: (
-      streamAction: STREAMING_ACTION,
-      waveform?: number[],
-      audioLanguage?: SpeechToTextLanguage
-    ) => model.streamingTranscribe(streamAction, waveform, audioLanguage),
+    committedTranscription,
+    nonCommittedTranscription,
+    encode: stateWrapper(SpeechToTextModule.prototype.encode),
+    decode: stateWrapper(SpeechToTextModule.prototype.decode),
+    transcribe: stateWrapper(SpeechToTextModule.prototype.transcribe),
+    stream,
+    streamStop: wrapper(SpeechToTextModule.prototype.streamStop),
+    streamInsert: wrapper(SpeechToTextModule.prototype.streamInsert),
   };
 };

package/src/hooks/natural_language_processing/useTextEmbeddings.ts

@@ -2,17 +2,17 @@ import { TextEmbeddingsModule } from '../../modules/natural_language_processing/
 import { ResourceSource } from '../../types/common';
 import { useModule } from '../useModule';
 
-export const useTextEmbeddings = ({
-  modelSource,
-  tokenizerSource,
-  preventLoad = false,
-}: {
-  modelSource: ResourceSource;
-  tokenizerSource: ResourceSource;
+interface Props {
+  model: {
+    modelSource: ResourceSource;
+    tokenizerSource: ResourceSource;
+  };
   preventLoad?: boolean;
-}) =>
+}
+
+export const useTextEmbeddings = ({ model, preventLoad = false }: Props) =>
   useModule({
     module: TextEmbeddingsModule,
-    loadArgs: [modelSource, tokenizerSource],
+    model,
     preventLoad,
   });

package/src/hooks/natural_language_processing/useTokenizer.ts

@@ -4,43 +4,43 @@ import { ResourceSource } from '../../types/common';
 import { ETError, getError } from '../../Error';
 
 export const useTokenizer = ({
-  tokenizerSource,
+  tokenizer,
   preventLoad = false,
 }: {
-  tokenizerSource: ResourceSource;
+  tokenizer: { tokenizerSource: ResourceSource };
   preventLoad?: boolean;
 }) => {
   const [error, setError] = useState<null | string>(null);
   const [isReady, setIsReady] = useState(false);
   const [isGenerating, setIsGenerating] = useState(false);
   const [downloadProgress, setDownloadProgress] = useState(0);
+  const [tokenizerInstance] = useState(() => new TokenizerModule());
 
   useEffect(() => {
-    const loadModule = async () => {
+    if (preventLoad) return;
+    (async () => {
+      setDownloadProgress(0);
+      setError(null);
       try {
         setIsReady(false);
-        TokenizerModule.onDownloadProgress(setDownloadProgress);
-        await TokenizerModule.load(tokenizerSource);
+        await tokenizerInstance.load(
+          { tokenizerSource: tokenizer.tokenizerSource },
+          setDownloadProgress
+        );
         setIsReady(true);
       } catch (err) {
         setError((err as Error).message);
       }
-    };
-    if (!preventLoad) {
-      loadModule();
-    }
-  }, [tokenizerSource, preventLoad]);
+    })();
+  }, [tokenizerInstance, tokenizer.tokenizerSource, preventLoad]);
 
   const stateWrapper = <T extends (...args: any[]) => Promise<any>>(fn: T) => {
-    const boundFn = fn.bind(TokenizerModule);
-
-    return async (...args: Parameters<T>): Promise<ReturnType<T>> => {
+    return (...args: Parameters<T>): Promise<ReturnType<T>> => {
       if (!isReady) throw new Error(getError(ETError.ModuleNotLoaded));
       if (isGenerating) throw new Error(getError(ETError.ModelGenerating));
-
-      setIsGenerating(true);
       try {
-        return await boundFn(...args);
+        setIsGenerating(true);
+        return fn.apply(tokenizerInstance, args);
       } finally {
         setIsGenerating(false);
       }
@@ -52,10 +52,10 @@ export const useTokenizer = ({
     isReady,
     isGenerating,
     downloadProgress,
-    decode: stateWrapper(TokenizerModule.decode),
-    encode: stateWrapper(TokenizerModule.encode),
-    getVocabSize: stateWrapper(TokenizerModule.getVocabSize),
-    idToToken: stateWrapper(TokenizerModule.idToToken),
-    tokenToId: stateWrapper(TokenizerModule.tokenToId),
+    decode: stateWrapper(TokenizerModule.prototype.decode),
+    encode: stateWrapper(TokenizerModule.prototype.encode),
+    getVocabSize: stateWrapper(TokenizerModule.prototype.getVocabSize),
+    idToToken: stateWrapper(TokenizerModule.prototype.idToToken),
+    tokenToId: stateWrapper(TokenizerModule.prototype.tokenToId),
   };
 };

package/src/hooks/natural_language_processing/useVAD.ts

@@ -0,0 +1,15 @@
+import { ResourceSource } from '../../types/common';
+import { useModule } from '../useModule';
+import { VADModule } from '../../modules/natural_language_processing/VADModule';
+
+interface Props {
+  model: { modelSource: ResourceSource };
+  preventLoad?: boolean;
+}
+
+export const useVAD = ({ model, preventLoad = false }: Props) =>
+  useModule({
+    module: VADModule,
+    model,
+    preventLoad: preventLoad,
+  });

package/src/hooks/useModule.ts

@@ -3,8 +3,12 @@ import { ETError, getError } from '../Error';
 
 interface Module {
   load: (...args: any[]) => Promise<void>;
-  forward: (...input: any[]) => Promise<any>;
-  onDownloadProgress: (cb: (progress: number) => void) => void;
+  forward: (...args: any[]) => Promise<any>;
+  delete: () => void;
+}
+
+interface ModuleConstructor<M extends Module> {
+  new (): M;
 }
 
 export const useModule = <
@@ -14,41 +18,47 @@ export const useModule = <
   ForwardReturn extends Awaited<ReturnType<M['forward']>>,
 >({
   module,
-  loadArgs,
+  model,
   preventLoad = false,
 }: {
-  module: M;
-  loadArgs: LoadArgs;
+  module: ModuleConstructor<M>;
+  model: LoadArgs[0];
   preventLoad?: boolean;
 }) => {
   const [error, setError] = useState<null | string>(null);
   const [isReady, setIsReady] = useState(false);
   const [isGenerating, setIsGenerating] = useState(false);
   const [downloadProgress, setDownloadProgress] = useState(0);
+  const [moduleInstance] = useState(() => new module());
 
   useEffect(() => {
-    const loadModule = async () => {
+    if (preventLoad) return;
+
+    (async () => {
+      setDownloadProgress(0);
+      setError(null);
       try {
         setIsReady(false);
-        module.onDownloadProgress(setDownloadProgress);
-        await module.load(...loadArgs);
+        await moduleInstance.load(model, setDownloadProgress);
         setIsReady(true);
       } catch (err) {
         setError((err as Error).message);
       }
+    })();
+
+    return () => {
+      moduleInstance.delete();
     };
-    if (!preventLoad) {
-      loadModule();
-    }
+
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [...loadArgs, preventLoad]);
+  }, [moduleInstance, ...Object.values(model), preventLoad]);
 
   const forward = async (...input: ForwardArgs): Promise<ForwardReturn> => {
     if (!isReady) throw new Error(getError(ETError.ModuleNotLoaded));
     if (isGenerating) throw new Error(getError(ETError.ModelGenerating));
     try {
       setIsGenerating(true);
-      return await module.forward(...input);
+      return await moduleInstance.forward(...input);
     } finally {
       setIsGenerating(false);
     }

package/src/index.ts

@@ -0,0 +1,126 @@
+import { ETInstallerNativeModule } from './native/RnExecutorchModules';
+
+// eslint-disable no-var
+declare global {
+  var loadStyleTransfer: (source: string) => any;
+  var loadImageSegmentation: (source: string) => any;
+  var loadClassification: (source: string) => any;
+  var loadObjectDetection: (source: string) => any;
+  var loadExecutorchModule: (source: string) => any;
+  var loadTokenizerModule: (source: string) => any;
+  var loadImageEmbeddings: (source: string) => any;
+  var loadVAD: (source: string) => any;
+  var loadTextEmbeddings: (modelSource: string, tokenizerSource: string) => any;
+  var loadLLM: (modelSource: string, tokenizerSource: string) => any;
+  var loadTextToImage: (
+    tokenizerSource: string,
+    encoderSource: string,
+    unetSource: string,
+    decoderSource: string,
+    schedulerBetaStart: number,
+    schedulerBetaEnd: number,
+    schedulerNumTrainTimesteps: number,
+    schedulerStepsOffset: number
+  ) => any;
+  var loadSpeechToText: (
+    encoderSource: string,
+    decoderSource: string,
+    modelName: string
+  ) => any;
+  var loadOCR: (
+    detectorSource: string,
+    recognizerLarge: string,
+    recognizerMedium: string,
+    recognizerSmall: string,
+    symbols: string
+  ) => any;
+  var loadVerticalOCR: (
+    detectorLarge: string,
+    detectorNarrow: string,
+    recognizer: string,
+    symbols: string,
+    independentCharacters?: boolean
+  ) => any;
+}
+// eslint-disable no-var
+if (
+  global.loadStyleTransfer == null ||
+  global.loadImageSegmentation == null ||
+  global.loadTextToImage == null ||
+  global.loadExecutorchModule == null ||
+  global.loadClassification == null ||
+  global.loadObjectDetection == null ||
+  global.loadTokenizerModule == null ||
+  global.loadTextEmbeddings == null ||
+  global.loadImageEmbeddings == null ||
+  global.loadVAD == null ||
+  global.loadLLM == null ||
+  global.loadSpeechToText == null ||
+  global.loadOCR == null ||
+  global.loadVerticalOCR == null
+) {
+  if (!ETInstallerNativeModule) {
+    throw new Error(
+      `Failed to install react-native-executorch: The native module could not be found.`
+    );
+  }
+  ETInstallerNativeModule.install();
+}
+
+// hooks
+export * from './hooks/computer_vision/useClassification';
+export * from './hooks/computer_vision/useObjectDetection';
+export * from './hooks/computer_vision/useStyleTransfer';
+export * from './hooks/computer_vision/useImageSegmentation';
+export * from './hooks/computer_vision/useOCR';
+export * from './hooks/computer_vision/useVerticalOCR';
+export * from './hooks/computer_vision/useImageEmbeddings';
+export * from './hooks/computer_vision/useTextToImage';
+
+export * from './hooks/natural_language_processing/useLLM';
+export * from './hooks/natural_language_processing/useSpeechToText';
+export * from './hooks/natural_language_processing/useTextEmbeddings';
+export * from './hooks/natural_language_processing/useTokenizer';
+export * from './hooks/natural_language_processing/useVAD';
+
+export * from './hooks/general/useExecutorchModule';
+
+// modules
+export * from './modules/computer_vision/ClassificationModule';
+export * from './modules/computer_vision/ObjectDetectionModule';
+export * from './modules/computer_vision/StyleTransferModule';
+export * from './modules/computer_vision/ImageSegmentationModule';
+export * from './modules/computer_vision/OCRModule';
+export * from './modules/computer_vision/VerticalOCRModule';
+export * from './modules/computer_vision/ImageEmbeddingsModule';
+export * from './modules/computer_vision/TextToImageModule';
+
+export * from './modules/natural_language_processing/LLMModule';
+export * from './modules/natural_language_processing/SpeechToTextModule';
+export * from './modules/natural_language_processing/TextEmbeddingsModule';
+export * from './modules/natural_language_processing/TokenizerModule';
+export * from './modules/natural_language_processing/VADModule';
+
+export * from './modules/general/ExecutorchModule';
+
+// utils
+export * from './utils/ResourceFetcher';
+export * from './utils/llm';
+
+// types
+export * from './types/objectDetection';
+export * from './types/ocr';
+export * from './types/imageSegmentation';
+export * from './types/llm';
+export * from './types/vad';
+export * from './types/common';
+export {
+  SpeechToTextLanguage,
+  SpeechToTextModelConfig,
+  DecodingOptions,
+} from './types/stt';
+
+// constants
+export * from './constants/modelUrls';
+export * from './constants/ocr/models';
+export * from './constants/llmDefaults';

package/src/modules/BaseModule.ts

@@ -1,31 +1,26 @@
-import { ResourceFetcher } from '../utils/ResourceFetcher';
-import { getError } from '../Error';
 import { ResourceSource } from '../types/common';
+import { TensorPtr } from '../types/common';
 
-export class BaseModule {
-  protected static nativeModule: any;
-  static onDownloadProgressCallback: (downloadProgress: number) => void =
-    () => {};
+export abstract class BaseModule {
+  nativeModule: any = null;
 
-  static async load(...sources: ResourceSource[]): Promise<void> {
-    try {
-      const paths = await ResourceFetcher.fetchMultipleResources(
-        this.onDownloadProgressCallback,
-        ...sources
-      );
-      await this.nativeModule.loadModule(...paths);
-    } catch (error) {
-      throw new Error(getError(error));
-    }
+  abstract load(
+    modelSource: ResourceSource,
+    onDownloadProgressCallback: (_: number) => void,
+    ...args: any[]
+  ): Promise<void>;
+
+  protected async forwardET(inputTensor: TensorPtr[]): Promise<TensorPtr[]> {
+    return await this.nativeModule.forward(inputTensor);
   }
 
-  protected static async forward(..._args: any[]): Promise<any> {
-    throw new Error(
-      'forward method is not implemented in the BaseModule class. Please implement it in the derived class.'
-    );
+  async getInputShape(methodName: string, index: number): Promise<number[]> {
+    return this.nativeModule.getInputShape(methodName, index);
   }
 
-  static onDownloadProgress(callback: (downloadProgress: number) => void) {
-    this.onDownloadProgressCallback = callback;
+  delete() {
+    if (this.nativeModule !== null) {
+      this.nativeModule.unload();
+    }
   }
 }

package/src/modules/computer_vision/ClassificationModule.ts

@@ -1,17 +1,26 @@
-import { ClassificationNativeModule } from '../../native/RnExecutorchModules';
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
 import { ResourceSource } from '../../types/common';
+import { ETError, getError } from '../../Error';
 import { BaseModule } from '../BaseModule';
 
 export class ClassificationModule extends BaseModule {
-  protected static override nativeModule = ClassificationNativeModule;
-
-  static override async load(modelSource: ResourceSource) {
-    await super.load(modelSource);
+  async load(
+    model: { modelSource: ResourceSource },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.modelSource
+    );
+    if (paths === null || paths.length < 1) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = global.loadClassification(paths[0] || '');
   }
 
-  static override async forward(
-    input: string
-  ): ReturnType<typeof ClassificationNativeModule.forward> {
-    return await this.nativeModule.forward(input);
+  async forward(imageSource: string) {
+    if (this.nativeModule == null)
+      throw new Error(getError(ETError.ModuleNotLoaded));
+    return await this.nativeModule.generate(imageSource);
   }
 }

package/src/modules/computer_vision/ImageEmbeddingsModule.ts

@@ -0,0 +1,26 @@
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
+import { ResourceSource } from '../../types/common';
+import { ETError, getError } from '../../Error';
+import { BaseModule } from '../BaseModule';
+
+export class ImageEmbeddingsModule extends BaseModule {
+  async load(
+    model: { modelSource: ResourceSource },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.modelSource
+    );
+    if (paths === null || paths.length < 1) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = global.loadImageEmbeddings(paths[0] || '');
+  }
+
+  async forward(imageSource: string): Promise<Float32Array> {
+    if (this.nativeModule == null)
+      throw new Error(getError(ETError.ModuleNotLoaded));
+    return new Float32Array(await this.nativeModule.generate(imageSource));
+  }
+}