react-native-executorch 0.7.0-nightly-b4770df-20260205 → 0.7.0

package/src/types/common.ts

@@ -1,30 +1,115 @@
+/**
+ * Common types used across the React Native Executorch package.
+ */
+
+/**
+ * Represents a source of a resource, which can be a string (e.g., URL or file path), a number (e.g., resource ID), or an object (e.g., binary data).
+ *
+ * @category Types
+ */
 export type ResourceSource = string | number | object;
 
+/**
+ * Enum representing the scalar types of tensors.
+ *
+ * @category Types
+ */
 export enum ScalarType {
+  /**
+   * Byte type (8-bit unsigned integer).
+   */
   BYTE = 0,
+  /**
+   * Character type (8-bit signed integer).
+   */
   CHAR = 1,
+  /**
+   * Short integer type (16-bit signed integer).
+   */
   SHORT = 2,
+  /**
+   * Integer type (32-bit signed integer).
+   */
   INT = 3,
+  /**
+   * Long integer type (64-bit signed integer).
+   */
   LONG = 4,
+  /**
+   * Half-precision floating point type (16-bit).
+   */
   HALF = 5,
+  /**
+   * Single-precision floating point type (32-bit).
+   */
   FLOAT = 6,
+  /**
+   * Double-precision floating point type (64-bit).
+   */
   DOUBLE = 7,
+  /**
+   * Boolean type.
+   */
   BOOL = 11,
+  /**
+   * Quantized 8-bit signed integer type.
+   */
   QINT8 = 12,
+  /**
+   * Quantized 8-bit unsigned integer type.
+   */
   QUINT8 = 13,
+  /**
+   * Quantized 32-bit signed integer type.
+   */
   QINT32 = 14,
+  /**
+   * Packed Quantized Unsigned 4-bit Integers type (2 number in 1 byte).
+   */
   QUINT4X2 = 16,
+  /**
+   * Packed Quantized Unsigned 2-bit Integer type (4 numbers in 1 byte).
+   */
   QUINT2X4 = 17,
+  /**
+   * Raw Bits type.
+   */
   BITS16 = 22,
+  /**
+   * Quantized 8-bit floating point type: Sign bit, 5 Exponent bits, 2 Mantissa bits.
+   */
   FLOAT8E5M2 = 23,
+  /**
+   * Quantized 8-bit floating point type: Sign bit, 4 Exponent bits, 3 Mantissa bits.
+   */
   FLOAT8E4M3FN = 24,
+  /**
+   * Quantized 8-bit floating point type with No Unsigned Zero (NUZ): Sign bit, 5 Exponent bits, 2 Mantissa bits.
+   */
   FLOAT8E5M2FNUZ = 25,
+  /**
+   * Quantized 8-bit floating point type with No Unsigned Zero (NUZ): Sign bit, 4 Exponent bits, 3 Mantissa bits.
+   */
   FLOAT8E4M3FNUZ = 26,
+  /**
+   * Unsigned 16-bit integer type.
+   */
   UINT16 = 27,
+  /**
+   * Unsigned 32-bit integer type.
+   */
   UINT32 = 28,
+  /**
+   * Unsigned 64-bit integer type.
+   */
   UINT64 = 29,
 }
 
+/**
+ * Represents the data buffer of a tensor, which can be one of several typed array formats.
+ *
+ * @category Types
+ */
 export type TensorBuffer =
   | ArrayBuffer
   | Float32Array
@@ -38,6 +123,14 @@ export type TensorBuffer =
   | BigInt64Array
   | BigUint64Array;
 
+/**
+ * Represents a pointer to a tensor, including its data buffer, size dimensions, and scalar type.
+ *
+ * @category Types
+ * @property {TensorBuffer} dataPtr - The data buffer of the tensor.
+ * @property {number[]} sizes - An array representing the size of each dimension of the tensor.
+ * @property {ScalarType} scalarType - The scalar type of the tensor, as defined in the `ScalarType` enum.
+ */
 export interface TensorPtr {
   dataPtr: TensorBuffer;
   sizes: number[];

package/src/types/executorchModule.ts

@@ -0,0 +1,50 @@
+import { ResourceSource, TensorPtr } from '../types/common';
+import { RnExecutorchError } from '../errors/errorUtils';
+
+/**
+ * Props for the `useExecutorchModule` hook.
+ *
+ * @category Types
+ * @property {ResourceSource} modelSource - The source of the ExecuTorch model binary.
+ * @property {boolean} [preventLoad] - Boolean that can prevent automatic model loading (and downloading the data if you load it for the first time) after running the hook.
+ */
+export interface ExecutorchModuleProps {
+  modelSource: ResourceSource;
+  preventLoad?: boolean;
+}
+
+/**
+ * Return type for the `useExecutorchModule` hook.
+ * Manages the state and core execution methods for a general ExecuTorch model.
+ *
+ * @category Types
+ */
+export interface ExecutorchModuleType {
+  /**
+   * Contains the error object if the model failed to load, download, or encountered a runtime error.
+   */
+  error: RnExecutorchError | null;
+
+  /**
+   * Indicates whether the ExecuTorch model binary has successfully loaded into memory and is ready for inference.
+   */
+  isReady: boolean;
+
+  /**
+   * Indicates whether the model is currently processing a forward pass.
+   */
+  isGenerating: boolean;
+
+  /**
+   * Represents the download progress of the model binary as a value between 0 and 1.
+   */
+  downloadProgress: number;
+
+  /**
+   * Executes the model's forward pass with the provided input tensors.
+   * @param inputTensor - An array of `TensorPtr` objects representing the input tensors required by the model.
+   * @returns A Promise that resolves to an array of output `TensorPtr` objects resulting from the model's inference.
+   * @throws {RnExecutorchError} If the model is not loaded or is currently processing another request.
+   */
+  forward: (inputTensor: TensorPtr[]) => Promise<TensorPtr[]>;
+}

package/src/types/imageEmbeddings.ts

@@ -0,0 +1,51 @@
+import { RnExecutorchError } from '../errors/errorUtils';
+import { ResourceSource } from './common';
+
+/**
+ * Props for the `useImageEmbeddings` hook.
+ *
+ * @category Types
+ * @property {Object} model - An object containing the model source.
+ * @property {ResourceSource} model.modelSource - The source of the image embeddings model binary.
+ * @property {boolean} [preventLoad] - Boolean that can prevent automatic model loading (and downloading the data if you load it for the first time) after running the hook.
+ */
+export interface ImageEmbeddingsProps {
+  model: { modelSource: ResourceSource };
+  preventLoad?: boolean;
+}
+
+/**
+ * Return type for the `useImageEmbeddings` hook.
+ * Manages the state and operations for generating image embeddings (feature vectors) used in Computer Vision tasks.
+ *
+ * @category Types
+ */
+export interface ImageEmbeddingsType {
+  /**
+   * Contains the error object if the model failed to load, download, or encountered a runtime error during embedding generation.
+   */
+  error: RnExecutorchError | null;
+
+  /**
+   * Indicates whether the image embeddings model is loaded and ready to process images.
+   */
+  isReady: boolean;
+
+  /**
+   * Indicates whether the model is currently generating embeddings for an image.
+   */
+  isGenerating: boolean;
+
+  /**
+   * Represents the download progress of the model binary as a value between 0 and 1.
+   */
+  downloadProgress: number;
+
+  /**
+   * Executes the model's forward pass to generate embeddings (a feature vector) for the provided image.
+   * @param imageSource - A string representing the image source (e.g., a file path, URI, or base64 string) to be processed.
+   * @returns A Promise that resolves to a `Float32Array` containing the generated embedding vector.
+   * @throws {RnExecutorchError} If the model is not loaded or is currently processing another image.
+   */
+  forward: (imageSource: string) => Promise<Float32Array>;
+}

package/src/types/imageSegmentation.ts

@@ -1,4 +1,11 @@
-/* eslint-disable @cspell/spellchecker */
+import { RnExecutorchError } from '../errors/errorUtils';
+import { ResourceSource } from './common';
+
+/**
+ * Labels used in the DeepLab image segmentation model.
+ *
+ * @category Types
+ */
 export enum DeeplabLabel {
   BACKGROUND,
   AEROPLANE,
@@ -23,3 +30,59 @@ export enum DeeplabLabel {
   TVMONITOR,
   ARGMAX, // Additional label not present in the model
 }
+
+/**
+ * Props for the `useImageSegmentation` hook.
+ *
+ * @property {Object} model - An object containing the model source.
+ * @property {ResourceSource} model.modelSource - The source of the image segmentation model binary.
+ * @property {boolean} [preventLoad] - Boolean that can prevent automatic model loading (and downloading the data if you load it for the first time) after running the hook.
+ *
+ * @category Types
+ */
+export interface ImageSegmentationProps {
+  model: { modelSource: ResourceSource };
+  preventLoad?: boolean;
+}
+
+/**
+ * Return type for the `useImageSegmentation` hook.
+ * Manages the state and operations for Computer Vision image segmentation (e.g., DeepLab).
+ *
+ * @category Types
+ */
+export interface ImageSegmentationType {
+  /**
+   * Contains the error object if the model failed to load, download, or encountered a runtime error during segmentation.
+   */
+  error: RnExecutorchError | null;
+
+  /**
+   * Indicates whether the segmentation model is loaded and ready to process images.
+   */
+  isReady: boolean;
+
+  /**
+   * Indicates whether the model is currently processing an image.
+   */
+  isGenerating: boolean;
+
+  /**
+   * Represents the download progress of the model binary as a value between 0 and 1.
+   */
+  downloadProgress: number;
+
+  /**
+   * Executes the model's forward pass to perform semantic segmentation on the provided image.
+   * @param imageSource - A string representing the image source (e.g., a file path, URI, or base64 string) to be processed.
+   * @param classesOfInterest - An optional array of `DeeplabLabel` enums. If provided, the model will only return segmentation masks for these specific classes.
+   * @param resize - An optional boolean indicating whether the output segmentation masks should be resized to match the original image dimensions. Defaults to standard model behavior if undefined.
+   * @returns A Promise that resolves to an object mapping each detected `DeeplabLabel` to its corresponding segmentation mask (represented as a flattened array of numbers).
+   * @throws {RnExecutorchError} If the model is not loaded or is currently processing another image.
+   */
+  forward: (
+    imageSource: string,
+    classesOfInterest?: DeeplabLabel[],
+    resize?: boolean
+  ) => Promise<Partial<Record<DeeplabLabel, number[]>>>;
+}

package/src/types/llm.ts

@@ -1,57 +1,249 @@
 import { RnExecutorchError } from '../errors/errorUtils';
+import { ResourceSource } from './common';
 
+/**
+ * Properties for initializing and configuring a Large Language Model (LLM) instance.
+ *
+ * @category Types
+ */
+export interface LLMProps {
+  model: {
+    /**
+     * `ResourceSource` that specifies the location of the model binary.
+     */
+    modelSource: ResourceSource;
+    /**
+     * `ResourceSource` pointing to the JSON file which contains the tokenizer.
+     */
+    tokenizerSource: ResourceSource;
+    /**
+     * `ResourceSource` pointing to the JSON file which contains the tokenizer config.
+     */
+    tokenizerConfigSource?: ResourceSource;
+  };
+  /**
+   * Boolean that can prevent automatic model loading (and downloading the data if you load it for the first time) after running the hook.
+   */
+  preventLoad?: boolean;
+}
+
+/**
+ * React hook for managing a Large Language Model (LLM) instance.
+ *
+ * @category Types
+ */
 export interface LLMType {
+  /**
+   * History containing all messages in conversation. This field is updated after model responds to sendMessage.
+   */
   messageHistory: Message[];
+
+  /**
+   * State of the generated response. This field is updated with each token generated by the model.
+   */
   response: string;
+
+  /**
+   * The most recently generated token.
+   */
   token: string;
+
+  /**
+   * Indicates whether the model is ready.
+   */
   isReady: boolean;
+
+  /**
+   * Indicates whether the model is currently generating a response.
+   */
   isGenerating: boolean;
+
+  /**
+   * Represents the download progress as a value between 0 and 1, indicating the extent of the model file retrieval.
+   */
   downloadProgress: number;
+
+  /**
+   * Contains the error message if the model failed to load.
+   */
   error: RnExecutorchError | null;
-  configure: ({
-    chatConfig,
-    toolsConfig,
-    generationConfig,
-  }: {
-    chatConfig?: Partial<ChatConfig>;
-    toolsConfig?: ToolsConfig;
-    generationConfig?: GenerationConfig;
-  }) => void;
+
+  /**
+   * Configures chat and tool calling.
+   * See [Configuring the model](https://docs.swmansion.com/react-native-executorch/docs/hooks/natural-language-processing/useLLM#configuring-the-model) for details.
+   *
+   * @param {LLMConfig} configuration - Configuration object containing `chatConfig`, `toolsConfig`, and `generationConfig`.
+   */
+  configure: ({ chatConfig, toolsConfig, generationConfig }: LLMConfig) => void;
+
+  /**
+   * Returns the number of tokens generated so far in the current generation.
+   *
+   * @returns The count of generated tokens.
+   */
   getGeneratedTokenCount: () => number;
+  /**
+   * Runs model to complete chat passed in `messages` argument. It doesn't manage conversation context.
+   *
+   * @param messages - Array of messages representing the chat history.
+   * @param tools - Optional array of tools that can be used during generation.
+   * @returns The generated tokens as `string`.
+   */
   generate: (messages: Message[], tools?: LLMTool[]) => Promise<string>;
+  /**
+   * Returns the number of total tokens from the previous generation.This is a sum of prompt tokens and generated tokens.
+   *
+   * @returns The count of prompt and generated tokens.
+   */
+  getTotalTokenCount: () => number;
+  /**
+   * Returns the number of prompt tokens in the last message.
+   *
+   * @returns The count of prompt token.
+   */
+  getPromptTokenCount: () => number;
+
+  /**
+   * Function to add user message to conversation.
+   * After model responds, `messageHistory` will be updated with both user message and model response.
+   *
+   * @param message - The message string to send.
+   * @returns The model's response as a `string`.
+   */
   sendMessage: (message: string) => Promise<string>;
+
+  /**
+   * Deletes all messages starting with message on `index` position. After deletion `messageHistory` will be updated.
+   *
+   * @param index - The index of the message to delete from history.
+   */
   deleteMessage: (index: number) => void;
+
+  /**
+   * Function to interrupt the current inference.
+   */
   interrupt: () => void;
 }
 
+/**
+ * Configuration object for initializing and customizing a Large Language Model (LLM) instance.
+ *
+ * @category Types
+ */
+export interface LLMConfig {
+  /**
+   * Object configuring chat management, contains following properties:
+   *
+   * `systemPrompt` - Often used to tell the model what is its purpose, for example - "Be a helpful translator".
+   *
+   * `initialMessageHistory` - An array of `Message` objects that represent the conversation history. This can be used to provide initial context to the model.
+   *
+   * `contextWindowLength` - The number of messages from the current conversation that the model will use to generate a response. The higher the number, the more context the model will have. Keep in mind that using larger context windows will result in longer inference time and higher memory usage.
+   */
+  chatConfig?: Partial<ChatConfig>;
+
+  /**
+   * Object configuring options for enabling and managing tool use. **It will only have effect if your model's chat template support it**. Contains following properties:
+   *
+   * `tools` - List of objects defining tools.
+   *
+   * `executeToolCallback` - Function that accepts `ToolCall`, executes tool and returns the string to model.
+   *
+   * `displayToolCalls` - If set to true, JSON tool calls will be displayed in chat. If false, only answers will be displayed.
+   */
+  toolsConfig?: ToolsConfig;
+
+  /**
+   * Object configuring generation settings.
+   *
+   * `outputTokenBatchSize` - Soft upper limit on the number of tokens in each token batch (in certain cases there can be more tokens in given batch, i.e. when the batch would end with special emoji join character).
+   *
+   * `batchTimeInterval` - Upper limit on the time interval between consecutive token batches.
+   *
+   * `temperature` - Scales output logits by the inverse of temperature. Controls the randomness / creativity of text generation.
+   *
+   * `topp` - Only samples from the smallest set of tokens whose cumulative probability exceeds topp.
+   */
+  generationConfig?: GenerationConfig;
+}
+
+/**
+ * Roles that a message sender can have.
+ *
+ * @category Types
+ */
 export type MessageRole = 'user' | 'assistant' | 'system';
 
+/**
+ * Represents a message in the conversation.
+ *
+ * @category Types
+ * @property {MessageRole} role - Role of the message sender of type `MessageRole`.
+ * @property {string} content - Content of the message.
+ */
 export interface Message {
   role: MessageRole;
   content: string;
 }
 
+/**
+ * Represents a tool call made by the model.
+ *
+ * @category Types
+ * @property {string} toolName - The name of the tool being called.
+ * @property {Object} arguments - The arguments passed to the tool.
+ */
 export interface ToolCall {
   toolName: string;
   arguments: Object;
 }
 
-// usually tool is represented with dictionary (Object), but fields depend on the model
-// unfortunately there's no one standard so it's hard to type it better
+/**
+ * Represents a tool that can be used by the model.
+ * Usually tool is represented with dictionary (Object), but fields depend on the model.
+ * Unfortunately there's no one standard so it's hard to type it better.
+ *
+ * @category Types
+ */
 export type LLMTool = Object;
 
+/**
+ * Object configuring chat management.
+ *
+ * @category Types
+ * @property {Message[]} initialMessageHistory - An array of `Message` objects that represent the conversation history. This can be used to provide initial context to the model.
+ * @property {number} contextWindowLength - The number of messages from the current conversation that the model will use to generate a response. The higher the number, the more context the model will have. Keep in mind that using larger context windows will result in longer inference time and higher memory usage.
+ * @property {string} systemPrompt - Often used to tell the model what is its purpose, for example - "Be a helpful translator".
+ */
 export interface ChatConfig {
   initialMessageHistory: Message[];
   contextWindowLength: number;
   systemPrompt: string;
 }
 
+/**
+ * Object configuring options for enabling and managing tool use. **It will only have effect if your model's chat template support it**.
+ *
+ * @category Types
+ * @property {LLMTool[]} tools - List of objects defining tools.
+ * @property {(call: ToolCall) => Promise<string | null>} executeToolCallback - Function that accepts `ToolCall`, executes tool and returns the string to model.
+ * @property {boolean} [displayToolCalls] - If set to true, JSON tool calls will be displayed in chat. If false, only answers will be displayed.
+ */
 export interface ToolsConfig {
   tools: LLMTool[];
   executeToolCallback: (call: ToolCall) => Promise<string | null>;
   displayToolCalls?: boolean;
 }
 
+/**
+ * Object configuring generation settings.
+ *
+ * @category Types
+ * @property {number} [temperature] - Scales output logits by the inverse of temperature. Controls the randomness / creativity of text generation.
+ * @property {number} [topp] - Only samples from the smallest set of tokens whose cumulative probability exceeds topp.
+ * @property {number} [outputTokenBatchSize] - Soft upper limit on the number of tokens in each token batch (in certain cases there can be more tokens in given batch, i.e. when the batch would end with special emoji join character).
+ * @property {number} [batchTimeInterval] - Upper limit on the time interval between consecutive token batches.
+ */
 export interface GenerationConfig {
   temperature?: number;
   topp?: number;
@@ -59,6 +251,11 @@ export interface GenerationConfig {
   batchTimeInterval?: number;
 }
 
+/**
+ * Special tokens used in Large Language Models (LLMs).
+ *
+ * @category Types
+ */
 export const SPECIAL_TOKENS = {
   BOS_TOKEN: 'bos_token',
   EOS_TOKEN: 'eos_token',

package/src/types/objectDetection.ts

@@ -1,3 +1,15 @@
+import { RnExecutorchError } from '../errors/errorUtils';
+import { ResourceSource } from './common';
+
+/**
+ * Represents a bounding box for a detected object in an image.
+ *
+ * @category Types
+ * @property {number} x1 - The x-coordinate of the bottom-left corner of the bounding box.
+ * @property {number} y1 - The y-coordinate of the bottom-left corner of the bounding box.
+ * @property {number} x2 - The x-coordinate of the top-right corner of the bounding box.
+ * @property {number} y2 - The y-coordinate of the top-right corner of the bounding box.
+ */
 export interface Bbox {
   x1: number;
   x2: number;
@@ -5,13 +17,26 @@ export interface Bbox {
   y2: number;
 }
 
+/**
+ * Represents a detected object within an image, including its bounding box, label, and confidence score.
+ *
+ * @category Types
+ * @property {Bbox} bbox - The bounding box of the detected object, defined by its top-left (x1, y1) and bottom-right (x2, y2) coordinates.
+ * @property {keyof typeof CocoLabel} label - The class label of the detected object, represented as a key from the `CocoLabel` enum.
+ * @property {number} score - The confidence score of the detection, typically ranging from 0 to 1.
+ */
 export interface Detection {
   bbox: Bbox;
   label: keyof typeof CocoLabel;
   score: number;
 }
 
-enum CocoLabel {
+/**
+ * COCO dataset class labels used for object detection.
+ *
+ * @category Types
+ */
+export enum CocoLabel {
   PERSON = 1,
   BICYCLE = 2,
   CAR = 3,
@@ -103,3 +128,56 @@ enum CocoLabel {
   TOOTHBRUSH = 90,
   HAIR_BRUSH = 91,
 }
+
+/**
+ * Props for the `useObjectDetection` hook.
+ *
+ * @category Types
+ * @property {Object} model - An object containing the model source.
+ * @property {ResourceSource} model.modelSource - The source of the object detection model binary.
+ * @property {boolean} [preventLoad] - Boolean that can prevent automatic model loading (and downloading the data if you load it for the first time) after running the hook.
+ */
+export interface ObjectDetectionProps {
+  model: { modelSource: ResourceSource };
+  preventLoad?: boolean;
+}
+
+/**
+ * Return type for the `useObjectDetection` hook.
+ * Manages the state and operations for Computer Vision object detection tasks.
+ *
+ * @category Types
+ */
+export interface ObjectDetectionType {
+  /**
+   * Contains the error object if the model failed to load, download, or encountered a runtime error during detection.
+   */
+  error: RnExecutorchError | null;
+
+  /**
+   * Indicates whether the object detection model is loaded and ready to process images.
+   */
+  isReady: boolean;
+
+  /**
+   * Indicates whether the model is currently processing an image.
+   */
+  isGenerating: boolean;
+
+  /**
+   * Represents the download progress of the model binary as a value between 0 and 1.
+   */
+  downloadProgress: number;
+
+  /**
+   * Executes the model's forward pass to detect objects within the provided image.
+   * @param imageSource - A string representing the image source (e.g., a file path, URI, or base64 string) to be processed.
+   * @param detectionThreshold - An optional number between 0 and 1 representing the minimum confidence score required for an object to be included in the results. Default is 0.7.
+   * @returns A Promise that resolves to an array of `Detection` objects, where each object typically contains bounding box coordinates, a class label, and a confidence score.
+   * @throws {RnExecutorchError} If the model is not loaded or is currently processing another image.
+   */
+  forward: (
+    imageSource: string,
+    detectionThreshold?: number
+  ) => Promise<Detection[]>;
+}