react-native-executorch 0.4.9 → 0.5.0-nightly-6f4dd53-20251211

package/src/modules/natural_language_processing/SpeechToTextModule.ts

@@ -1,81 +1,179 @@
-import { ResourceSource } from '../../types/common';
-import { SpeechToTextController } from '../../controllers/SpeechToTextController';
-import { AvailableModels, SpeechToTextLanguage } from '../../types/stt';
-import { STREAMING_ACTION } from '../../constants/sttDefaults';
-
-export class SpeechToText {
-  static module: SpeechToTextController;
-
-  static onDownloadProgressCallback = (_downloadProgress: number) => {};
-
-  static async load(
-    modelName: AvailableModels,
-    transcribeCallback: (sequence: string) => void,
-    modelDownloadProgressCallback?: (downloadProgress: number) => void,
-    encoderSource?: ResourceSource,
-    decoderSource?: ResourceSource,
-    tokenizerSource?: ResourceSource,
-    overlapSeconds?: ConstructorParameters<
-      typeof SpeechToTextController
-    >['0']['overlapSeconds'],
-    windowSize?: ConstructorParameters<
-      typeof SpeechToTextController
-    >['0']['windowSize'],
-    streamingConfig?: ConstructorParameters<
-      typeof SpeechToTextController
-    >['0']['streamingConfig']
-  ) {
-    this.module = new SpeechToTextController({
-      transcribeCallback: transcribeCallback,
-      modelDownloadProgressCallback: modelDownloadProgressCallback,
-      overlapSeconds: overlapSeconds,
-      windowSize: windowSize,
-      streamingConfig: streamingConfig,
-    });
-    await this.module.loadModel(
-      (modelName = modelName),
-      (encoderSource = encoderSource),
-      (decoderSource = decoderSource),
-      (tokenizerSource = tokenizerSource)
-    );
-  }
+import { Logger } from '../../common/Logger';
+import { DecodingOptions, SpeechToTextModelConfig } from '../../types/stt';
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
+
+export class SpeechToTextModule {
+  private nativeModule: any;
+
+  private modelConfig!: SpeechToTextModelConfig;
+
+  private textDecoder = new TextDecoder('utf-8', {
+    fatal: false,
+    ignoreBOM: true,
+  });
 
-  static configureStreaming(
-    overlapSeconds: Parameters<SpeechToTextController['configureStreaming']>[0],
-    windowSize: Parameters<SpeechToTextController['configureStreaming']>[1],
-    streamingConfig: Parameters<SpeechToTextController['configureStreaming']>[2]
+  public async load(
+    model: SpeechToTextModelConfig,
+    onDownloadProgressCallback: (progress: number) => void = () => {}
   ) {
-    this.module?.configureStreaming(
-      overlapSeconds,
-      windowSize,
-      streamingConfig
+    this.modelConfig = model;
+
+    const tokenizerLoadPromise = ResourceFetcher.fetch(
+      undefined,
+      model.tokenizerSource
+    );
+    const encoderDecoderPromise = ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.encoderSource,
+      model.decoderSource
+    );
+    const [tokenizerSources, encoderDecoderResults] = await Promise.all([
+      tokenizerLoadPromise,
+      encoderDecoderPromise,
+    ]);
+    const encoderSource = encoderDecoderResults?.[0];
+    const decoderSource = encoderDecoderResults?.[1];
+    if (!encoderSource || !decoderSource || !tokenizerSources) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = await global.loadSpeechToText(
+      encoderSource,
+      decoderSource,
+      tokenizerSources[0]!
     );
   }
 
-  static async encode(waveform: number[]) {
-    return await this.module.encode(waveform);
+  public delete(): void {
+    this.nativeModule.unload();
   }
 
-  static async decode(seq: number[], encodings: number[]) {
-    return await this.module.decode(seq, encodings);
+  public async encode(
+    waveform: Float32Array | number[]
+  ): Promise<Float32Array> {
+    if (Array.isArray(waveform)) {
+      Logger.info(
+        'Passing waveform as number[] is deprecated, use Float32Array instead'
+      );
+      waveform = new Float32Array(waveform);
+    }
+    return new Float32Array(await this.nativeModule.encode(waveform));
   }
 
-  static async transcribe(
-    waveform: number[],
-    audioLanguage?: SpeechToTextLanguage
-  ): ReturnType<SpeechToTextController['transcribe']> {
-    return await this.module.transcribe(waveform, audioLanguage);
+  public async decode(
+    tokens: Int32Array | number[],
+    encoderOutput: Float32Array | number[]
+  ): Promise<Float32Array> {
+    if (Array.isArray(tokens)) {
+      Logger.info(
+        'Passing tokens as number[] is deprecated, use Int32Array instead'
+      );
+      tokens = new Int32Array(tokens);
+    }
+    if (Array.isArray(encoderOutput)) {
+      Logger.info(
+        'Passing encoderOutput as number[] is deprecated, use Float32Array instead'
+      );
+      encoderOutput = new Float32Array(encoderOutput);
+    }
+    return new Float32Array(
+      await this.nativeModule.decode(tokens, encoderOutput)
+    );
   }
 
-  static async streamingTranscribe(
-    streamAction: STREAMING_ACTION,
-    waveform?: number[],
-    audioLanguage?: SpeechToTextLanguage
-  ): ReturnType<SpeechToTextController['streamingTranscribe']> {
-    return await this.module.streamingTranscribe(
-      streamAction,
+  public async transcribe(
+    waveform: Float32Array | number[],
+    options: DecodingOptions = {}
+  ): Promise<string> {
+    this.validateOptions(options);
+
+    if (Array.isArray(waveform)) {
+      Logger.info(
+        'Passing waveform as number[] is deprecated, use Float32Array instead'
+      );
+      waveform = new Float32Array(waveform);
+    }
+    const transcriptionBytes = await this.nativeModule.transcribe(
       waveform,
-      audioLanguage
+      options.language || ''
     );
+    return this.textDecoder.decode(new Uint8Array(transcriptionBytes));
+  }
+
+  public async *stream(
+    options: DecodingOptions = {}
+  ): AsyncGenerator<{ committed: string; nonCommitted: string }> {
+    this.validateOptions(options);
+
+    const queue: { committed: string; nonCommitted: string }[] = [];
+    let waiter: (() => void) | null = null;
+    let finished = false;
+    let error: unknown;
+
+    const wake = () => {
+      waiter?.();
+      waiter = null;
+    };
+
+    (async () => {
+      try {
+        await this.nativeModule.stream(
+          (committed: number[], nonCommitted: number[], isDone: boolean) => {
+            queue.push({
+              committed: this.textDecoder.decode(new Uint8Array(committed)),
+              nonCommitted: this.textDecoder.decode(
+                new Uint8Array(nonCommitted)
+              ),
+            });
+            if (isDone) {
+              finished = true;
+            }
+            wake();
+          },
+          options.language || ''
+        );
+        finished = true;
+        wake();
+      } catch (e) {
+        error = e;
+        finished = true;
+        wake();
+      }
+    })();
+
+    while (true) {
+      if (queue.length > 0) {
+        yield queue.shift()!;
+        if (finished && queue.length === 0) {
+          return;
+        }
+        continue;
+      }
+      if (error) throw error;
+      if (finished) return;
+      await new Promise<void>((r) => (waiter = r));
+    }
+  }
+
+  public streamInsert(waveform: Float32Array | number[]): void {
+    if (Array.isArray(waveform)) {
+      Logger.info(
+        'Passing waveform as number[] is deprecated, use Float32Array instead'
+      );
+      waveform = new Float32Array(waveform);
+    }
+    this.nativeModule.streamInsert(waveform);
+  }
+
+  public streamStop(): void {
+    this.nativeModule.streamStop();
+  }
+
+  private validateOptions(options: DecodingOptions) {
+    if (!this.modelConfig.isMultilingual && options.language) {
+      throw new Error('Model is not multilingual, cannot set language');
+    }
+    if (this.modelConfig.isMultilingual && !options.language) {
+      throw new Error('Model is multilingual, provide a language');
+    }
   }
 }

package/src/modules/natural_language_processing/TextEmbeddingsModule.ts

@@ -1,18 +1,33 @@
-import { TextEmbeddingsNativeModule } from '../../native/RnExecutorchModules';
 import { ResourceSource } from '../../types/common';
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
 import { BaseModule } from '../BaseModule';
 
 export class TextEmbeddingsModule extends BaseModule {
-  protected static override nativeModule = TextEmbeddingsNativeModule;
-
-  static override async load(
-    modelSource: ResourceSource,
-    tokenizerSource: ResourceSource
-  ) {
-    await super.load(modelSource, tokenizerSource);
+  async load(
+    model: { modelSource: ResourceSource; tokenizerSource: ResourceSource },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const modelPromise = ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.modelSource
+    );
+    const tokenizerPromise = ResourceFetcher.fetch(
+      undefined,
+      model.tokenizerSource
+    );
+    const [modelResult, tokenizerResult] = await Promise.all([
+      modelPromise,
+      tokenizerPromise,
+    ]);
+    const modelPath = modelResult?.[0];
+    const tokenizerPath = tokenizerResult?.[0];
+    if (!modelPath || !tokenizerPath) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = global.loadTextEmbeddings(modelPath, tokenizerPath);
   }
 
-  static override async forward(input: string): Promise<number[]> {
-    return this.nativeModule.forward(input);
+  async forward(input: string): Promise<Float32Array> {
+    return new Float32Array(await this.nativeModule.generate(input));
   }
 }

package/src/modules/natural_language_processing/TokenizerModule.ts

@@ -1,34 +1,44 @@
-import { TokenizerNativeModule } from '../../native/RnExecutorchModules';
 import { ResourceSource } from '../../types/common';
-import { BaseModule } from '../BaseModule';
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
 
-export class TokenizerModule extends BaseModule {
-  protected static override nativeModule = TokenizerNativeModule;
+export class TokenizerModule {
+  nativeModule: any;
 
-  static override async load(tokenizerSource: ResourceSource) {
-    await super.load(tokenizerSource);
+  async load(
+    tokenizer: { tokenizerSource: ResourceSource },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      tokenizer.tokenizerSource
+    );
+    const path = paths?.[0];
+    if (!path) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = global.loadTokenizerModule(path);
   }
 
-  static async decode(
-    input: number[],
-    skipSpecialTokens = false
-  ): Promise<string> {
-    return await this.nativeModule.decode(input, skipSpecialTokens);
+  async encode(s: string) {
+    return await this.nativeModule.encode(s);
   }
 
-  static async encode(input: string): Promise<number[]> {
-    return await this.nativeModule.encode(input);
+  async decode(tokens: number[], skipSpecialTokens: boolean = true) {
+    if (tokens.length === 0) {
+      return '';
+    }
+    return await this.nativeModule.decode(tokens, skipSpecialTokens);
   }
 
-  static async getVocabSize(): Promise<number> {
+  async getVocabSize(): Promise<number> {
     return await this.nativeModule.getVocabSize();
   }
 
-  static async idToToken(tokenId: number): Promise<string> {
-    return await this.nativeModule.idToToken(tokenId);
+  async idToToken(tokenId: number): Promise<string> {
+    return this.nativeModule.idToToken(tokenId);
   }
 
-  static async tokenToId(token: string): Promise<number> {
+  async tokenToId(token: string): Promise<number> {
     return await this.nativeModule.tokenToId(token);
   }
 }

package/src/modules/natural_language_processing/VADModule.ts

@@ -0,0 +1,27 @@
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
+import { ResourceSource } from '../../types/common';
+import { Segment } from '../../types/vad';
+import { ETError, getError } from '../../Error';
+import { BaseModule } from '../BaseModule';
+
+export class VADModule extends BaseModule {
+  async load(
+    model: { modelSource: ResourceSource },
+    onDownloadProgressCallback: (progress: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetch(
+      onDownloadProgressCallback,
+      model.modelSource
+    );
+    if (paths === null || paths.length < 1) {
+      throw new Error('Download interrupted.');
+    }
+    this.nativeModule = global.loadVAD(paths[0] || '');
+  }
+
+  async forward(waveform: Float32Array): Promise<Segment[]> {
+    if (this.nativeModule == null)
+      throw new Error(getError(ETError.ModuleNotLoaded));
+    return await this.nativeModule.generate(waveform);
+  }
+}

package/src/native/NativeETInstaller.ts

@@ -0,0 +1,8 @@
+import type { TurboModule } from 'react-native';
+import { TurboModuleRegistry } from 'react-native';
+
+export interface Spec extends TurboModule {
+  install(): boolean;
+}
+
+export default TurboModuleRegistry.get<Spec>('ETInstaller');

package/src/native/RnExecutorchModules.ts

@@ -1,15 +1,5 @@
 import { Platform } from 'react-native';
-import { Spec as ObjectDetectionInterface } from './NativeObjectDetection';
-import { Spec as StyleTransferInterface } from './NativeStyleTransfer';
-import { Spec as ImageSegmentationInterface } from './NativeImageSegmentation';
-import { Spec as ETModuleInterface } from './NativeETModule';
-import { Spec as OCRInterface } from './NativeOCR';
-import { Spec as VerticalOCRInterface } from './NativeVerticalOCR';
-import { Spec as SpeechToTextInterface } from './NativeSpeechToText';
-import { Spec as TextEmbeddingsInterface } from './NativeTextEmbeddings';
-import { Spec as LLMInterface } from './NativeLLM';
-import { Spec as ClassificationInterface } from './NativeClassification';
-import { Spec as TokenizerInterface } from './NativeTokenizer';
+import { Spec as ETInstallerInterface } from './NativeETInstaller';
 
 const LINKING_ERROR =
   `The package 'react-native-executorch' doesn't seem to be linked. Make sure: \n\n` +
@@ -30,43 +20,7 @@ function returnSpecOrThrowLinkingError(spec: any) {
       );
 }
 
-const LLMNativeModule: LLMInterface = returnSpecOrThrowLinkingError(
-  require('./NativeLLM').default
-);
-const ETModuleNativeModule: ETModuleInterface = returnSpecOrThrowLinkingError(
-  require('./NativeETModule').default
-);
-const ClassificationNativeModule: ClassificationInterface =
-  returnSpecOrThrowLinkingError(require('./NativeClassification').default);
-const ImageSegmentationNativeModule: ImageSegmentationInterface =
-  returnSpecOrThrowLinkingError(require('./NativeImageSegmentation').default);
-const ObjectDetectionNativeModule: ObjectDetectionInterface =
-  returnSpecOrThrowLinkingError(require('./NativeObjectDetection').default);
-const StyleTransferNativeModule: StyleTransferInterface =
-  returnSpecOrThrowLinkingError(require('./NativeStyleTransfer').default);
-const SpeechToTextNativeModule: SpeechToTextInterface =
-  returnSpecOrThrowLinkingError(require('./NativeSpeechToText').default);
-const OCRNativeModule: OCRInterface = returnSpecOrThrowLinkingError(
-  require('./NativeOCR').default
-);
-const VerticalOCRNativeModule: VerticalOCRInterface =
-  returnSpecOrThrowLinkingError(require('./NativeVerticalOCR').default);
-const TokenizerNativeModule: TokenizerInterface = returnSpecOrThrowLinkingError(
-  require('./NativeTokenizer').default
-);
-const TextEmbeddingsNativeModule: TextEmbeddingsInterface =
-  returnSpecOrThrowLinkingError(require('./NativeTextEmbeddings').default);
+const ETInstallerNativeModule: ETInstallerInterface =
+  returnSpecOrThrowLinkingError(require('./NativeETInstaller').default);
 
-export {
-  LLMNativeModule,
-  ETModuleNativeModule,
-  ClassificationNativeModule,
-  ObjectDetectionNativeModule,
-  StyleTransferNativeModule,
-  ImageSegmentationNativeModule,
-  SpeechToTextNativeModule,
-  OCRNativeModule,
-  VerticalOCRNativeModule,
-  TextEmbeddingsNativeModule,
-  TokenizerNativeModule,
-};
+export { ETInstallerNativeModule };

package/src/types/common.ts

@@ -1,17 +1,45 @@
-export const getTypeIdentifier = (input: ETInput): number => {
-  if (input instanceof Int8Array) return 1;
-  if (input instanceof Int32Array) return 3;
-  if (input instanceof BigInt64Array) return 4;
-  if (input instanceof Float32Array) return 6;
-  if (input instanceof Float64Array) return 7;
-  return -1;
-};
-
 export type ResourceSource = string | number | object;
 
-export type ETInput =
+export enum ScalarType {
+  BYTE = 0,
+  CHAR = 1,
+  SHORT = 2,
+  INT = 3,
+  LONG = 4,
+  HALF = 5,
+  FLOAT = 6,
+  DOUBLE = 7,
+  BOOL = 11,
+  QINT8 = 12,
+  QUINT8 = 13,
+  QINT32 = 14,
+  QUINT4X2 = 16,
+  QUINT2X4 = 17,
+  BITS16 = 22,
+  FLOAT8E5M2 = 23,
+  FLOAT8E4M3FN = 24,
+  FLOAT8E5M2FNUZ = 25,
+  FLOAT8E4M3FNUZ = 26,
+  UINT16 = 27,
+  UINT32 = 28,
+  UINT64 = 29,
+}
+
+export type TensorBuffer =
+  | ArrayBuffer
+  | Float32Array
+  | Float64Array
   | Int8Array
+  | Int16Array
   | Int32Array
+  | Uint8Array
+  | Uint16Array
+  | Uint32Array
   | BigInt64Array
-  | Float32Array
-  | Float64Array;
+  | BigUint64Array;
+
+export interface TensorPtr {
+  dataPtr: TensorBuffer;
+  sizes: number[];
+  scalarType: ScalarType;
+}

package/src/types/llm.ts

@@ -9,10 +9,13 @@ export interface LLMType {
   configure: ({
     chatConfig,
     toolsConfig,
+    generationConfig,
   }: {
     chatConfig?: Partial<ChatConfig>;
     toolsConfig?: ToolsConfig;
+    generationConfig?: GenerationConfig;
   }) => void;
+  getGeneratedTokenCount: () => number;
   generate: (messages: Message[], tools?: LLMTool[]) => Promise<void>;
   sendMessage: (message: string) => Promise<void>;
   deleteMessage: (index: number) => void;
@@ -47,6 +50,13 @@ export interface ToolsConfig {
   displayToolCalls?: boolean;
 }
 
+export interface GenerationConfig {
+  temperature?: number;
+  topp?: number;
+  outputTokenBatchSize?: number;
+  batchTimeInterval?: number;
+}
+
 export const SPECIAL_TOKENS = {
   BOS_TOKEN: 'bos_token',
   EOS_TOKEN: 'eos_token',