react-native-executorch 0.7.0-nightly-b4770df-20260205 → 0.7.0

package/src/modules/natural_language_processing/LLMModule.ts

@@ -1,22 +1,33 @@
 import { LLMController } from '../../controllers/LLMController';
 import { ResourceSource } from '../../types/common';
-import {
-  ChatConfig,
-  GenerationConfig,
-  LLMTool,
-  Message,
-  ToolsConfig,
-} from '../../types/llm';
+import { LLMConfig, LLMTool, Message } from '../../types/llm';
 
+/**
+ * Module for managing a Large Language Model (LLM) instance.
+ *
+ * @category Typescript API
+ */
 export class LLMModule {
   private controller: LLMController;
 
+  /**
+   * Creates a new instance of `LLMModule` with optional callbacks.
+   * @param optionalCallbacks - Object containing optional callbacks.
+   *
+   * @returns A new LLMModule instance.
+   */
   constructor({
     tokenCallback,
     messageHistoryCallback,
   }: {
+    /**
+     * An optional function that will be called on every generated token (`string`) with that token as its only argument.
+     */
     tokenCallback?: (token: string) => void;
-    responseCallback?: (response: string) => void;
+    /**
+     * An optional function called on every finished message (`Message[]`).
+     * Returns the entire message history.
+     */
     messageHistoryCallback?: (messageHistory: Message[]) => void;
   } = {}) {
     this.controller = new LLMController({
@@ -25,6 +36,15 @@ export class LLMModule {
     });
   }
 
+  /**
+   * Loads the LLM model and tokenizer.
+   *
+   * @param model - Object containing model, tokenizer, and tokenizer config sources.
+   * @param model.modelSource - `ResourceSource` that specifies the location of the model binary.
+   * @param model.tokenizerSource - `ResourceSource` pointing to the JSON file which contains the tokenizer.
+   * @param model.tokenizerConfigSource - `ResourceSource` pointing to the JSON file which contains the tokenizer config.
+   * @param onDownloadProgressCallback - Optional callback to track download progress (value between 0 and 1).
+   */
   async load(
     model: {
       modelSource: ResourceSource;
@@ -39,6 +59,11 @@ export class LLMModule {
     });
   }
 
+  /**
+   * Sets new token callback invoked on every token batch.
+   *
+   * @param tokenCallback - Callback function to handle new tokens.
+   */
   setTokenCallback({
     tokenCallback,
   }: {
@@ -47,44 +72,105 @@ export class LLMModule {
     this.controller.setTokenCallback(tokenCallback);
   }
 
-  configure({
-    chatConfig,
-    toolsConfig,
-    generationConfig,
-  }: {
-    chatConfig?: Partial<ChatConfig>;
-    toolsConfig?: ToolsConfig;
-    generationConfig?: GenerationConfig;
-  }) {
+  /**
+   * Configures chat and tool calling and generation settings.
+   * See [Configuring the model](https://docs.swmansion.com/react-native-executorch/docs/hooks/natural-language-processing/useLLM#configuring-the-model) for details.
+   *
+   * @param configuration - Configuration object containing `chatConfig`, `toolsConfig`, and `generationConfig`.
+   */
+  configure({ chatConfig, toolsConfig, generationConfig }: LLMConfig) {
     this.controller.configure({ chatConfig, toolsConfig, generationConfig });
   }
 
+  /**
+   * Runs model inference with raw input string.
+   * You need to provide entire conversation and prompt (in correct format and with special tokens!) in input string to this method.
+   * It doesn't manage conversation context. It is intended for users that need access to the model itself without any wrapper.
+   * If you want a simple chat with model the consider using `sendMessage`
+   *
+   * @param input - Raw input string containing the prompt and conversation history.
+   * @returns The generated response as a string.
+   */
   async forward(input: string): Promise<string> {
     return await this.controller.forward(input);
   }
 
+  /**
+   * Runs model to complete chat passed in `messages` argument. It doesn't manage conversation context.
+   *
+   * @param messages - Array of messages representing the chat history.
+   * @param tools - Optional array of tools that can be used during generation.
+   * @returns The generated response as a string.
+   */
   async generate(messages: Message[], tools?: LLMTool[]): Promise<string> {
     return await this.controller.generate(messages, tools);
   }
 
+  /**
+   * Method to add user message to conversation.
+   * After model responds it will call `messageHistoryCallback()` containing both user message and model response.
+   * It also returns them.
+   *
+   * @param message - The message string to send.
+   * @returns - Updated message history including the new user message and model response.
+   */
   async sendMessage(message: string): Promise<Message[]> {
     await this.controller.sendMessage(message);
     return this.controller.messageHistory;
   }
 
+  /**
+   * Deletes all messages starting with message on `index` position.
+   * After deletion it will call `messageHistoryCallback()` containing new history.
+   * It also returns it.
+   *
+   * @param index - The index of the message to delete from history.
+   * @returns - Updated message history after deletion.
+   */
   deleteMessage(index: number): Message[] {
     this.controller.deleteMessage(index);
     return this.controller.messageHistory;
   }
 
+  /**
+   * Interrupts model generation. It may return one more token after interrupt.
+   */
   interrupt() {
     this.controller.interrupt();
   }
 
-  getGeneratedTokenCount() {
+  /**
+   * Returns the number of tokens generated in the last response.
+   *
+   * @returns The count of generated tokens.
+   */
+  getGeneratedTokenCount(): number {
     return this.controller.getGeneratedTokenCount();
   }
 
+  /**
+   * Returns the number of prompt tokens in the last message.
+   *
+   * @returns The count of prompt token.
+   */
+  getPromptTokensCount() {
+    return this.controller.getPromptTokenCount();
+  }
+
+  /**
+   * Returns the number of total tokens from the previous generation. This is a sum of prompt tokens and generated tokens.
+   *
+   * @returns The count of prompt and generated tokens.
+   */
+  getTotalTokensCount() {
+    return this.controller.getTotalTokenCount();
+  }
+
+  /**
+   * Method to delete the model from memory.
+   * Note you cannot delete model while it's generating.
+   * You need to interrupt it first and make sure model stopped generation.
+   */
   delete() {
     this.controller.delete();
   }

package/src/modules/natural_language_processing/SpeechToTextModule.ts

@@ -1,9 +1,13 @@
-import { Logger } from '../../common/Logger';
 import { DecodingOptions, SpeechToTextModelConfig } from '../../types/stt';
 import { ResourceFetcher } from '../../utils/ResourceFetcher';
 import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
 import { RnExecutorchError, parseUnknownError } from '../../errors/errorUtils';
 
+/**
+ * Module for Speech to Text (STT) functionalities.
+ *
+ * @category Typescript API
+ */
 export class SpeechToTextModule {
   private nativeModule: any;
 
@@ -14,6 +18,13 @@ export class SpeechToTextModule {
     ignoreBOM: true,
   });
 
+  /**
+   * Loads the model specified by the config object.
+   * `onDownloadProgressCallback` allows you to monitor the current progress of the model download.
+   *
+   * @param model - Configuration object containing model sources.
+   * @param onDownloadProgressCallback - Optional callback to monitor download progress.
+   */
   public async load(
     model: SpeechToTextModelConfig,
     onDownloadProgressCallback: (progress: number) => void = () => {}
@@ -48,55 +59,54 @@ export class SpeechToTextModule {
     );
   }
 
+  /**
+   * Unloads the model from memory.
+   */
   public delete(): void {
     this.nativeModule.unload();
   }
 
-  public async encode(
-    waveform: Float32Array | number[]
-  ): Promise<Float32Array> {
-    if (Array.isArray(waveform)) {
-      Logger.info(
-        'Passing waveform as number[] is deprecated, use Float32Array instead'
-      );
-      waveform = new Float32Array(waveform);
-    }
+  /**
+   * Runs the encoding part of the model on the provided waveform.
+   * Returns the encoded waveform as a Float32Array.
+   *
+   * @param waveform - The input audio waveform.
+   * @returns The encoded output.
+   */
+  public async encode(waveform: Float32Array): Promise<Float32Array> {
     return new Float32Array(await this.nativeModule.encode(waveform));
   }
 
+  /**
+   * Runs the decoder of the model.
+   *
+   * @param tokens - The input tokens.
+   * @param encoderOutput - The encoder output.
+   * @returns Decoded output.
+   */
   public async decode(
-    tokens: Int32Array | number[],
-    encoderOutput: Float32Array | number[]
+    tokens: Int32Array,
+    encoderOutput: Float32Array
   ): Promise<Float32Array> {
-    if (Array.isArray(tokens)) {
-      Logger.info(
-        'Passing tokens as number[] is deprecated, use Int32Array instead'
-      );
-      tokens = new Int32Array(tokens);
-    }
-    if (Array.isArray(encoderOutput)) {
-      Logger.info(
-        'Passing encoderOutput as number[] is deprecated, use Float32Array instead'
-      );
-      encoderOutput = new Float32Array(encoderOutput);
-    }
     return new Float32Array(
       await this.nativeModule.decode(tokens, encoderOutput)
     );
   }
 
+  /**
+   * Starts a transcription process for a given input array (16kHz waveform).
+   * For multilingual models, specify the language in `options`.
+   * Returns the transcription as a string. Passing `number[]` is deprecated.
+   *
+   * @param waveform - The Float32Array audio data.
+   * @param options - Decoding options including language.
+   * @returns The transcription string.
+   */
   public async transcribe(
-    waveform: Float32Array | number[],
+    waveform: Float32Array,
     options: DecodingOptions = {}
   ): Promise<string> {
     this.validateOptions(options);
-
-    if (Array.isArray(waveform)) {
-      Logger.info(
-        'Passing waveform as number[] is deprecated, use Float32Array instead'
-      );
-      waveform = new Float32Array(waveform);
-    }
     const transcriptionBytes = await this.nativeModule.transcribe(
       waveform,
       options.language || ''
@@ -104,6 +114,18 @@ export class SpeechToTextModule {
     return this.textDecoder.decode(new Uint8Array(transcriptionBytes));
   }
 
+  /**
+   * Starts a streaming transcription session.
+   * Yields objects with `committed` and `nonCommitted` transcriptions.
+   * Committed transcription contains the part of the transcription that is finalized and will not change.
+   * Useful for displaying stable results during streaming.
+   * Non-committed transcription contains the part of the transcription that is still being processed and may change.
+   * Useful for displaying live, partial results during streaming.
+   * Use with `streamInsert` and `streamStop` to control the stream.
+   *
+   * @param options - Decoding options including language.
+   * @returns An async generator yielding transcription updates.
+   */
   public async *stream(
     options: DecodingOptions = {}
   ): AsyncGenerator<{ committed: string; nonCommitted: string }> {
@@ -159,16 +181,18 @@ export class SpeechToTextModule {
     }
   }
 
-  public streamInsert(waveform: Float32Array | number[]): void {
-    if (Array.isArray(waveform)) {
-      Logger.info(
-        'Passing waveform as number[] is deprecated, use Float32Array instead'
-      );
-      waveform = new Float32Array(waveform);
-    }
+  /**
+   * Inserts a new audio chunk into the streaming transcription session.
+   *
+   * @param waveform - The audio chunk to insert.
+   */
+  public streamInsert(waveform: Float32Array): void {
     this.nativeModule.streamInsert(waveform);
   }
 
+  /**
+   * Stops the current streaming transcription session.
+   */
   public streamStop(): void {
     this.nativeModule.streamStop();
   }

package/src/modules/natural_language_processing/TextEmbeddingsModule.ts

@@ -4,7 +4,20 @@ import { BaseModule } from '../BaseModule';
 import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
 import { RnExecutorchError } from '../../errors/errorUtils';
 
+/**
+ * Module for generating text embeddings from input text.
+ *
+ * @category Typescript API
+ */
 export class TextEmbeddingsModule extends BaseModule {
+  /**
+   * Loads the model and tokenizer specified by the config object.
+   *
+   * @param model - Object containing model and tokenizer sources.
+   * @param model.modelSource - `ResourceSource` that specifies the location of the text embeddings model binary.
+   * @param model.tokenizerSource - `ResourceSource` that specifies the location of the tokenizer JSON file.
+   * @param onDownloadProgressCallback - Optional callback to track download progress (value between 0 and 1).
+   */
   async load(
     model: { modelSource: ResourceSource; tokenizerSource: ResourceSource },
     onDownloadProgressCallback: (progress: number) => void = () => {}
@@ -32,6 +45,12 @@ export class TextEmbeddingsModule extends BaseModule {
     this.nativeModule = global.loadTextEmbeddings(modelPath, tokenizerPath);
   }
 
+  /**
+   * Executes the model's forward pass, where `input` is a text that will be embedded.
+   *
+   * @param input - The text string to embed.
+   * @returns A Float32Array containing the vector embeddings.
+   */
   async forward(input: string): Promise<Float32Array> {
     return new Float32Array(await this.nativeModule.generate(input));
   }

package/src/modules/natural_language_processing/TextToSpeechModule.ts

@@ -8,9 +8,24 @@ import {
   VoiceConfig,
 } from '../../types/tts';
 
+/**
+ * Module for Text to Speech (TTS) functionalities.
+ *
+ * @category Typescript API
+ */
 export class TextToSpeechModule {
+  /**
+   * Native module instance
+   */
   nativeModule: any = null;
 
+  /**
+   * Loads the model and voice assets specified by the config object.
+   * `onDownloadProgressCallback` allows you to monitor the current progress.
+   *
+   * @param config - Configuration object containing `model` source and `voice`.
+   * @param onDownloadProgressCallback - Optional callback to monitor download progress.
+   */
   public async load(
     config: TextToSpeechConfig,
     onDownloadProgressCallback: (progress: number) => void = () => {}
@@ -73,7 +88,18 @@ export class TextToSpeechModule {
     );
   }
 
-  public async forward(text: string, speed: number = 1.0) {
+  /**
+   * Synthesizes the provided text into speech.
+   * Returns a promise that resolves to the full audio waveform as a `Float32Array`.
+   *
+   * @param text The input text to be synthesized.
+   * @param speed Optional speed multiplier for the speech synthesis (default is 1.0).
+   * @returns A promise resolving to the synthesized audio waveform.
+   */
+  public async forward(
+    text: string,
+    speed: number = 1.0
+  ): Promise<Float32Array> {
     if (this.nativeModule == null)
       throw new RnExecutorchError(
         RnExecutorchErrorCode.ModuleNotLoaded,
@@ -82,7 +108,16 @@ export class TextToSpeechModule {
     return await this.nativeModule.generate(text, speed);
   }
 
-  public async *stream({ text, speed }: TextToSpeechStreamingInput) {
+  /**
+   * Starts a streaming synthesis session. Yields audio chunks as they are generated.
+   *
+   * @param input - Input object containing text and optional speed.
+   * @returns An async generator yielding Float32Array audio chunks.
+   */
+  public async *stream({
+    text,
+    speed,
+  }: TextToSpeechStreamingInput): AsyncGenerator<Float32Array> {
     // Stores computed audio segments
     const queue: Float32Array[] = [];
 
@@ -124,10 +159,16 @@ export class TextToSpeechModule {
     }
   }
 
+  /**
+   * Stops the streaming process if there is any ongoing.
+   */
   public streamStop(): void {
     this.nativeModule.streamStop();
   }
 
+  /**
+   * Unloads the model from memory.
+   */
   delete() {
     if (this.nativeModule !== null) {
       this.nativeModule.unload();

package/src/modules/natural_language_processing/TokenizerModule.ts

@@ -3,9 +3,24 @@ import { ResourceFetcher } from '../../utils/ResourceFetcher';
 import { RnExecutorchError } from '../../errors/errorUtils';
 import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
 
+/**
+ * Module for Tokenizer functionalities.
+ *
+ * @category Typescript API
+ */
 export class TokenizerModule {
+  /**
+   * Native module instance
+   */
   nativeModule: any;
 
+  /**
+   * Loads the tokenizer from the specified source.
+   * `tokenizerSource` is a string that points to the location of the tokenizer JSON file.
+   *
+   * @param tokenizer - Object containing `tokenizerSource`.
+   * @param onDownloadProgressCallback - Optional callback to monitor download progress.
+   */
   async load(
     tokenizer: { tokenizerSource: ResourceSource },
     onDownloadProgressCallback: (progress: number) => void = () => {}
@@ -24,25 +39,58 @@ export class TokenizerModule {
     this.nativeModule = global.loadTokenizerModule(path);
   }
 
-  async encode(s: string) {
-    return await this.nativeModule.encode(s);
+  /**
+   * Converts a string into an array of token IDs.
+   *
+   * @param input - The input string to be tokenized.
+   * @returns An array of token IDs.
+   */
+  async encode(input: string): Promise<number[]> {
+    return await this.nativeModule.encode(input);
   }
 
-  async decode(tokens: number[], skipSpecialTokens: boolean = true) {
+  /**
+   * Converts an array of token IDs into a string.
+   *
+   * @param tokens - Array of token IDs to be decoded.
+   * @param skipSpecialTokens - Whether to skip special tokens during decoding (default: true).
+   * @returns The decoded string.
+   */
+  async decode(
+    tokens: number[],
+    skipSpecialTokens: boolean = true
+  ): Promise<string> {
     if (tokens.length === 0) {
       return '';
     }
     return await this.nativeModule.decode(tokens, skipSpecialTokens);
   }
 
+  /**
+   * Returns the size of the tokenizer's vocabulary.
+   *
+   * @returns The vocabulary size.
+   */
   async getVocabSize(): Promise<number> {
     return await this.nativeModule.getVocabSize();
   }
 
+  /**
+   * Returns the token associated to the ID.
+   *
+   * @param tokenId - ID of the token.
+   * @returns The token string associated to ID.
+   */
   async idToToken(tokenId: number): Promise<string> {
     return this.nativeModule.idToToken(tokenId);
   }
 
+  /**
+   * Returns the ID associated to the token.
+   *
+   * @param token - The token string.
+   * @returns The ID associated to the token.
+   */
   async tokenToId(token: string): Promise<number> {
     return await this.nativeModule.tokenToId(token);
   }

package/src/modules/natural_language_processing/VADModule.ts

@@ -5,7 +5,19 @@ import { BaseModule } from '../BaseModule';
 import { RnExecutorchErrorCode } from '../../errors/ErrorCodes';
 import { RnExecutorchError } from '../../errors/errorUtils';
 
+/**
+ * Module for Voice Activity Detection (VAD) functionalities.
+ *
+ * @category Typescript API
+ */
 export class VADModule extends BaseModule {
+  /**
+   * Loads the model, where `modelSource` is a string that specifies the location of the model binary.
+   * To track the download progress, supply a callback function `onDownloadProgressCallback`.
+   *
+   * @param model - Object containing `modelSource`.
+   * @param onDownloadProgressCallback - Optional callback to monitor download progress.
+   */
   async load(
     model: { modelSource: ResourceSource },
     onDownloadProgressCallback: (progress: number) => void = () => {}
@@ -23,6 +35,12 @@ export class VADModule extends BaseModule {
     this.nativeModule = global.loadVAD(paths[0] || '');
   }
 
+  /**
+   * Executes the model's forward pass, where `waveform` is a Float32Array representing the audio signal (16kHz).
+   *
+   * @param waveform - The input audio waveform as a Float32Array. It must represent a mono audio signal sampled at 16kHz.
+   * @returns A promise resolving to an array of detected speech segments.
+   */
   async forward(waveform: Float32Array): Promise<Segment[]> {
     if (this.nativeModule == null)
       throw new RnExecutorchError(

package/src/types/classification.ts

@@ -0,0 +1,51 @@
+import { RnExecutorchError } from '../errors/errorUtils';
+import { ResourceSource } from './common';
+
+/**
+ * Props for the `useClassification` hook.
+ *
+ * @category Types
+ * @property {Object} model - An object containing the model source.
+ * @property {ResourceSource} model.modelSource - The source of the classification model binary.
+ * @property {boolean} [preventLoad] - Boolean that can prevent automatic model loading (and downloading the data if you load it for the first time) after running the hook.
+ */
+export interface ClassificationProps {
+  model: { modelSource: ResourceSource };
+  preventLoad?: boolean;
+}
+
+/**
+ * Return type for the `useClassification` hook.
+ * Manages the state and operations for Computer Vision image classification.
+ *
+ * @category Types
+ */
+export interface ClassificationType {
+  /**
+   * Contains the error object if the model failed to load, download, or encountered a runtime error during classification.
+   */
+  error: RnExecutorchError | null;
+
+  /**
+   * Indicates whether the classification model is loaded and ready to process images.
+   */
+  isReady: boolean;
+
+  /**
+   * Indicates whether the model is currently processing an image.
+   */
+  isGenerating: boolean;
+
+  /**
+   * Represents the download progress of the model binary as a value between 0 and 1.
+   */
+  downloadProgress: number;
+
+  /**
+   * Executes the model's forward pass to classify the provided image.
+   * @param imageSource - A string representing the image source (e.g., a file path, URI, or base64 string) to be classified.
+   * @returns A Promise that resolves to the classification result (typically containing labels and confidence scores).
+   * @throws {RnExecutorchError} If the model is not loaded or is currently processing another image.
+   */
+  forward: (imageSource: string) => Promise<{ [category: string]: number }>;
+}