react-native-executorch 0.9.0 → 0.9.1

package/lib/typescript/types/llm.d.ts

@@ -4,56 +4,76 @@ import { ResourceSource } from './common';
  * Capabilities a multimodal LLM can have.
  * @category Types
  */
-export type LLMCapability = 'vision';
+export type LLMCapability = 'vision' | 'audio';
 /**
  * Derives the media argument shape for `sendMessage` from a capabilities tuple.
  * @category Types
  */
-export type MediaArg<C extends readonly LLMCapability[]> = 'vision' extends C[number] ? {
+export type MediaArg<C extends readonly LLMCapability[]> = ('vision' extends C[number] ? {
     imagePath?: string;
-} : object;
+} : object) & ('audio' extends C[number] ? {
+    audioBuffer?: Float32Array;
+} : object);
 /**
  * Union of all built-in LLM model names.
  * @category Types
  */
-export type LLMModelName = 'llama-3.2-3b' | 'llama-3.2-3b-qlora' | 'llama-3.2-3b-spinquant' | 'llama-3.2-1b' | 'llama-3.2-1b-qlora' | 'llama-3.2-1b-spinquant' | 'qwen3-0.6b' | 'qwen3-0.6b-quantized' | 'qwen3-1.7b' | 'qwen3-1.7b-quantized' | 'qwen3-4b' | 'qwen3-4b-quantized' | 'hammer2.1-0.5b' | 'hammer2.1-0.5b-quantized' | 'hammer2.1-1.5b' | 'hammer2.1-1.5b-quantized' | 'hammer2.1-3b' | 'hammer2.1-3b-quantized' | 'smollm2.1-135m' | 'smollm2.1-135m-quantized' | 'smollm2.1-360m' | 'smollm2.1-360m-quantized' | 'smollm2.1-1.7b' | 'smollm2.1-1.7b-quantized' | 'qwen2.5-0.5b' | 'qwen2.5-0.5b-quantized' | 'qwen2.5-1.5b' | 'qwen2.5-1.5b-quantized' | 'qwen2.5-3b' | 'qwen2.5-3b-quantized' | 'phi-4-mini-4b' | 'phi-4-mini-4b-quantized' | 'lfm2.5-350m' | 'lfm2.5-350m-quantized' | 'lfm2.5-1.2b-instruct' | 'lfm2.5-1.2b-instruct-quantized' | 'lfm2.5-vl-1.6b-quantized' | 'lfm2.5-vl-450m-quantized' | 'qwen3.5-0.8b-quantized' | 'qwen3.5-2b-quantized' | 'bielik-v3.0-1.5b' | 'bielik-v3.0-1.5b-quantized';
+export type LLMModelName = 'gemma4-e2b' | 'gemma4-e2b-multimodal' | 'llama-3.2-3b' | 'llama-3.2-3b-qlora' | 'llama-3.2-3b-spinquant' | 'llama-3.2-1b' | 'llama-3.2-1b-qlora' | 'llama-3.2-1b-spinquant' | 'qwen3-0.6b' | 'qwen3-0.6b-quantized' | 'qwen3-1.7b' | 'qwen3-1.7b-quantized' | 'qwen3-4b' | 'qwen3-4b-quantized' | 'hammer2.1-0.5b' | 'hammer2.1-0.5b-quantized' | 'hammer2.1-1.5b' | 'hammer2.1-1.5b-quantized' | 'hammer2.1-3b' | 'hammer2.1-3b-quantized' | 'smollm2.1-135m' | 'smollm2.1-135m-quantized' | 'smollm2.1-360m' | 'smollm2.1-360m-quantized' | 'smollm2.1-1.7b' | 'smollm2.1-1.7b-quantized' | 'qwen2.5-0.5b' | 'qwen2.5-0.5b-quantized' | 'qwen2.5-1.5b' | 'qwen2.5-1.5b-quantized' | 'qwen2.5-3b' | 'qwen2.5-3b-quantized' | 'phi-4-mini-4b' | 'phi-4-mini-4b-quantized' | 'lfm2.5-350m' | 'lfm2.5-350m-quantized' | 'lfm2.5-1.2b-instruct' | 'lfm2.5-1.2b-instruct-quantized' | 'lfm2.5-vl-1.6b-quantized' | 'lfm2.5-vl-450m-quantized' | 'qwen3.5-0.8b-quantized' | 'qwen3.5-2b-quantized' | 'bielik-v3.0-1.5b' | 'bielik-v3.0-1.5b-quantized';
+/**
+ * Audio soft-token expansion constants for audio_encoder.
+ * @category Types
+ */
+export interface AudioConfig {
+    samplesPerBlock: number;
+    tokensPerBlock: number;
+}
+/**
+ * Properties defining LLMModel.
+ * @category Types
+ */
+export interface LLMModel {
+    /**
+     * The built-in model name (e.g. `'llama-3.2-3b'`). Used for telemetry and hook reload triggers.
+     * Pass one of the pre-built LLM constants (e.g. `LLAMA3_2_3B`) to populate all required fields.
+     */
+    modelName: LLMModelName;
+    /**
+     * `ResourceSource` that specifies the location of the model binary.
+     */
+    modelSource: ResourceSource;
+    /**
+     * `ResourceSource` pointing to the JSON file which contains the tokenizer.
+     */
+    tokenizerSource: ResourceSource;
+    /**
+     * `ResourceSource` pointing to the JSON file which contains the tokenizer config.
+     */
+    tokenizerConfigSource: ResourceSource;
+    /**
+     * Optional list of modality capabilities the model supports.
+     * Determines the type of the `media` argument in `sendMessage`.
+     * Example: `['vision']` enables `sendMessage(text, { imagePath })`.
+     */
+    capabilities?: readonly LLMCapability[];
+    /**
+     * Recommended default generation settings, typically copied from the
+     * upstream `generation_config.json` or the model card. Applied automatically
+     * after the native module loads and before any user `configure()` call,
+     * so callers only need to override the values they want to change.
+     */
+    generationConfig?: GenerationConfig;
+    /**
+     * Defines config for audio input modality for multimodal LLMs.
+     * `capabilities` must include 'audio'.
+     */
+    audioConfig?: AudioConfig;
+}
 /**
  * Properties for initializing and configuring a Large Language Model (LLM) instance.
  * @category Types
  */
 export interface LLMProps {
-    model: {
-        /**
-         * The built-in model name (e.g. `'llama-3.2-3b'`). Used for telemetry and hook reload triggers.
-         * Pass one of the pre-built LLM constants (e.g. `LLAMA3_2_3B`) to populate all required fields.
-         */
-        modelName: LLMModelName;
-        /**
-         * `ResourceSource` that specifies the location of the model binary.
-         */
-        modelSource: ResourceSource;
-        /**
-         * `ResourceSource` pointing to the JSON file which contains the tokenizer.
-         */
-        tokenizerSource: ResourceSource;
-        /**
-         * `ResourceSource` pointing to the JSON file which contains the tokenizer config.
-         */
-        tokenizerConfigSource: ResourceSource;
-        /**
-         * Optional list of modality capabilities the model supports.
-         * Determines the type of the `media` argument in `sendMessage`.
-         * Example: `['vision']` enables `sendMessage(text, { imagePath })`.
-         */
-        capabilities?: readonly LLMCapability[];
-        /**
-         * Recommended default generation settings, typically copied from the
-         * upstream `generation_config.json` or the model card. Applied automatically
-         * after the native module loads and before any user `configure()` call,
-         * so callers only need to override the values they want to change.
-         */
-        generationConfig?: GenerationConfig;
-    };
+    model: LLMModel;
     /**
      * Boolean that can prevent automatic model loading (and downloading the data if you load it for the first time) after running the hook.
      */
@@ -224,6 +244,12 @@ export interface Message {
      * controller normalizes the path before passing it to native code.
      */
     mediaPath?: string;
+    /**
+     * Optional fp32 mono 16 kHz PCM buffer. Only valid on `user` messages for
+     * models with the `'audio'` capability. The controller forwards it to the
+     * native `generateMultimodal` path.
+     */
+    audioWaveform?: Float32Array;
 }
 /**
  * Represents a tool call made by the model.
@@ -309,6 +335,7 @@ export interface ContextStrategy {
 export declare const SPECIAL_TOKENS: {
     BOS_TOKEN: string;
     EOS_TOKEN: string;
+    EOT_TOKEN: string;
     UNK_TOKEN: string;
     SEP_TOKEN: string;
     PAD_TOKEN: string;

package/lib/typescript/types/llm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../../src/types/llm.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE1C;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,QAAQ,CAAC;AAErC;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,SAAS,SAAS,aAAa,EAAE,IACrD,QAAQ,SAAS,CAAC,CAAC,MAAM,CAAC,GAAG;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAAC;AAE/D;;;GAGG;AACH,MAAM,MAAM,YAAY,GACpB,cAAc,GACd,oBAAoB,GACpB,wBAAwB,GACxB,cAAc,GACd,oBAAoB,GACpB,wBAAwB,GACxB,YAAY,GACZ,sBAAsB,GACtB,YAAY,GACZ,sBAAsB,GACtB,UAAU,GACV,oBAAoB,GACpB,gBAAgB,GAChB,0BAA0B,GAC1B,gBAAgB,GAChB,0BAA0B,GAC1B,cAAc,GACd,wBAAwB,GACxB,gBAAgB,GAChB,0BAA0B,GAC1B,gBAAgB,GAChB,0BAA0B,GAC1B,gBAAgB,GAChB,0BAA0B,GAC1B,cAAc,GACd,wBAAwB,GACxB,cAAc,GACd,wBAAwB,GACxB,YAAY,GACZ,sBAAsB,GACtB,eAAe,GACf,yBAAyB,GACzB,aAAa,GACb,uBAAuB,GACvB,sBAAsB,GACtB,gCAAgC,GAChC,0BAA0B,GAC1B,0BAA0B,GAC1B,wBAAwB,GACxB,sBAAsB,GACtB,kBAAkB,GAClB,4BAA4B,CAAC;AAEjC;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE;QACL;;;WAGG;QACH,SAAS,EAAE,YAAY,CAAC;QACxB;;WAEG;QACH,WAAW,EAAE,cAAc,CAAC;QAC5B;;WAEG;QACH,eAAe,EAAE,cAAc,CAAC;QAChC;;WAEG;QACH,qBAAqB,EAAE,cAAc,CAAC;QACtC;;;;WAIG;QACH,YAAY,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;QACxC;;;;;WAKG;QACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;KACrC,CAAC;IACF;;OAEG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,cAAc,EAAE,OAAO,EAAE,CAAC;IAE1B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,YAAY,EAAE,OAAO,CAAC;IAEtB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IAEzB;;OAEG;IACH,KAAK,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAEhC;;;;OAIG;IACH,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,gBAAgB,EAAE,EAAE,SAAS,KAAK,IAAI,CAAC;IAE9E;;;OAGG;IACH,sBAAsB,EAAE,MAAM,MAAM,CAAC;IACrC;;;;;;OAMG;IACH,QAAQ,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACtE;;;OAGG;IACH,kBAAkB,EAAE,MAAM,MAAM,CAAC;IACjC;;;OAGG;IACH,mBAAmB,EAAE,MAAM,MAAM,CAAC;IAElC;;;OAGG;IACH,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAEvC;;OAEG;IACH,SAAS,EAAE,MAAM,IAAI,CAAC;CACvB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB,CAChC,CAAC,SAAS,SAAS,aAAa,EAAE,GAAG,SAAS,aAAa,EAAE,CAC7D,SAAQ,WAAW;IACnB;;;;;;;OAOG;IACH,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACxE;AAED;;;;GAIG;AACH,MAAM,WAAW,OAAQ,SAAQ,WAAW;IAC1C;;;;;OAKG;IACH,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACnD;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAEjC;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;;;;;;;;;;OAcG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC;AAE7B;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB,EAAE,OAAO,EAAE,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,eAAe,CAAC;CAClC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,mBAAmB,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,gBAAgB;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sCAAsC;IACtC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;;;;OAOG;IACH,YAAY,CACV,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,OAAO,EAAE,EAClB,gBAAgB,EAAE,MAAM,EACxB,aAAa,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,MAAM,GAC7C,OAAO,EAAE,CAAC;CACd;AAED;;;GAGG;AACH,eAAO,MAAM,cAAc;;;;;;;;CAQ1B,CAAC"}
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../../src/types/llm.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE1C;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/C;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,SAAS,SAAS,aAAa,EAAE,IACrD,CAAC,QAAQ,SAAS,CAAC,CAAC,MAAM,CAAC,GAAG;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAAC,GAC5D,CAAC,OAAO,SAAS,CAAC,CAAC,MAAM,CAAC,GAAG;IAAE,WAAW,CAAC,EAAE,YAAY,CAAA;CAAE,GAAG,MAAM,CAAC,CAAC;AAE1E;;;GAGG;AACH,MAAM,MAAM,YAAY,GACpB,YAAY,GACZ,uBAAuB,GACvB,cAAc,GACd,oBAAoB,GACpB,wBAAwB,GACxB,cAAc,GACd,oBAAoB,GACpB,wBAAwB,GACxB,YAAY,GACZ,sBAAsB,GACtB,YAAY,GACZ,sBAAsB,GACtB,UAAU,GACV,oBAAoB,GACpB,gBAAgB,GAChB,0BAA0B,GAC1B,gBAAgB,GAChB,0BAA0B,GAC1B,cAAc,GACd,wBAAwB,GACxB,gBAAgB,GAChB,0BAA0B,GAC1B,gBAAgB,GAChB,0BAA0B,GAC1B,gBAAgB,GAChB,0BAA0B,GAC1B,cAAc,GACd,wBAAwB,GACxB,cAAc,GACd,wBAAwB,GACxB,YAAY,GACZ,sBAAsB,GACtB,eAAe,GACf,yBAAyB,GACzB,aAAa,GACb,uBAAuB,GACvB,sBAAsB,GACtB,gCAAgC,GAChC,0BAA0B,GAC1B,0BAA0B,GAC1B,wBAAwB,GACxB,sBAAsB,GACtB,kBAAkB,GAClB,4BAA4B,CAAC;AAEjC;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACvB;;;OAGG;IACH,SAAS,EAAE,YAAY,CAAC;IACxB;;OAEG;IACH,WAAW,EAAE,cAAc,CAAC;IAC5B;;OAEG;IACH,eAAe,EAAE,cAAc,CAAC;IAChC;;OAEG;IACH,qBAAqB,EAAE,cAAc,CAAC;IACtC;;;;OAIG;IACH,YAAY,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACxC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,QAAQ,CAAC;IAChB;;OAEG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,cAAc,EAAE,OAAO,EAAE,CAAC;IAE1B;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;IAEjB;;OAEG;IACH,YAAY,EAAE,OAAO,CAAC;IAEtB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IAEzB;;OAEG;IACH,KAAK,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAEhC;;;;OAIG;IACH,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,gBAAgB,EAAE,EAAE,SAAS,KAAK,IAAI,CAAC;IAE9E;;;OAGG;IACH,sBAAsB,EAAE,MAAM,MAAM,CAAC;IACrC;;;;;;OAMG;IACH,QAAQ,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACtE;;;OAGG;IACH,kBAAkB,EAAE,MAAM,MAAM,CAAC;IACjC;;;OAGG;IACH,mBAAmB,EAAE,MAAM,MAAM,CAAC;IAElC;;;OAGG;IACH,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAEvC;;OAEG;IACH,SAAS,EAAE,MAAM,IAAI,CAAC;CACvB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB,CAChC,CAAC,SAAS,SAAS,aAAa,EAAE,GAAG,SAAS,aAAa,EAAE,CAC7D,SAAQ,WAAW;IACnB;;;;;;;OAOG;IACH,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACxE;AAED;;;;GAIG;AACH,MAAM,WAAW,OAAQ,SAAQ,WAAW;IAC1C;;;;;OAKG;IACH,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACnD;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAEjC;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;;;;;;;;;;OAcG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,aAAa,CAAC,EAAE,YAAY,CAAC;CAC9B;AAED;;;;;GAKG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC;AAE7B;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB,EAAE,OAAO,EAAE,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,eAAe,CAAC;CAClC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,mBAAmB,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,gBAAgB;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sCAAsC;IACtC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B;;;;;;;OAOG;IACH,YAAY,CACV,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,OAAO,EAAE,EAClB,gBAAgB,EAAE,MAAM,EACxB,aAAa,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,MAAM,GAC7C,OAAO,EAAE,CAAC;CACd;AAED;;;GAGG;AACH,eAAO,MAAM,cAAc;;;;;;;;;CAS1B,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-executorch",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "description": "An easy way to run AI models in React Native with ExecuTorch",
   "source": "./src/index.ts",
   "main": "./lib/module/index.js",

package/react-native-executorch.podspec

@@ -62,6 +62,12 @@ Pod::Spec.new do |s|
 
   s.libraries = "z"
   s.ios.vendored_frameworks = "third-party/ios/ExecutorchLib.xcframework"
+
+  # NOTE: mlx.metallib (the MLX GPU kernels) is bundled INSIDE
+  # ExecutorchLib.framework, colocated with the binary that contains the MLX
+  # code. MLX's runtime loader resolves the metallib relative to that binary
+  # (via dladdr), so it must live next to it in the framework — not at the app
+  # bundle root.
   # Exclude file with tests to not introduce gtest dependency.
   # Do not include the headers from common/rnexecutorch/jsi/ as source files.
   # Xcode/Cocoapods leaks them to other pods that an app also depends on, so if

package/src/constants/llmDefaults.ts

@@ -6,7 +6,7 @@ import { SlidingWindowContextStrategy } from '../utils/llms/context_strategy';
  * @category Utilities - LLM
  */
 export const DEFAULT_SYSTEM_PROMPT =
-  "You are a knowledgeable, efficient, and direct AI assistant. Provide concise answers, focusing on the key information needed. Offer suggestions tactfully when appropriate to improve outcomes. Engage in productive collaboration with the user. Don't return too much text.";
+  "You are a knowledgeable, efficient, and direct AI assistant. Provide concise answers, focusing on the key information needed. Offer suggestions tactfully when appropriate to improve outcomes. Engage in productive collaboration with the user. Don't return too much text. If provided with audio samples treat it with at most importance";
 
 /**
  * Generates a default structured output prompt based on the provided JSON schema.

package/src/constants/modelRegistry.ts

@@ -38,7 +38,7 @@ import { RnExecutorchErrorCode } from '../errors/ErrorCodes';
  * compile-time error.
  * @category Utils
  */
-export type Backend = 'xnnpack' | 'coreml' | 'vulkan' | 'qnn';
+export type Backend = 'xnnpack' | 'coreml' | 'vulkan' | 'qnn' | 'mlx';
 
 /**
  * Options for a `models` accessor call.
@@ -78,7 +78,7 @@ type ConfigOf<V> = Extract<
 >;
 type BackendsOf<V> = Extract<keyof V, Backend>;
 
-const BACKEND_ORDER: Backend[] = ['xnnpack', 'coreml', 'vulkan', 'qnn'];
+const BACKEND_ORDER: Backend[] = ['xnnpack', 'coreml', 'mlx', 'vulkan', 'qnn'];
 
 function firstBackend(variants: AnyVariantMap): Backend {
   for (const b of BACKEND_ORDER) {
@@ -181,6 +181,33 @@ function tts<C extends TextToSpeechModelConfig>(c: C): () => C {
 // Per-backend variant maps for models that ship more than one backend.
 // ─────────────────────────────────────────────────────────────────────────────
 
+const GEMMA4_E2B_VARIANTS = {
+  mlx: {
+    base: {
+      modelName: 'gemma4-e2b' as const,
+      modelSource: M.GEMMA4_E2B_MLX_MODEL,
+      tokenizerSource: M.GEMMA4_E2B_TOKENIZER,
+      tokenizerConfigSource: M.GEMMA4_E2B_TOKENIZER_CONFIG,
+    },
+  },
+  xnnpack: {
+    base: {
+      modelName: 'gemma4-e2b' as const,
+      modelSource: M.GEMMA4_E2B_XNNPACK_MODEL,
+      tokenizerSource: M.GEMMA4_E2B_TOKENIZER,
+      tokenizerConfigSource: M.GEMMA4_E2B_TOKENIZER_CONFIG,
+    },
+  },
+  vulkan: {
+    base: {
+      modelName: 'gemma4-e2b' as const,
+      modelSource: M.GEMMA4_E2B_VULKAN_MODEL,
+      tokenizerSource: M.GEMMA4_E2B_TOKENIZER,
+      tokenizerConfigSource: M.GEMMA4_E2B_TOKENIZER_CONFIG,
+    },
+  },
+};
+
 const EFFICIENTNET_V2_S_VARIANTS = {
   xnnpack: {
     base: {
@@ -496,10 +523,15 @@ export const models = {
       M.LFM2_5_1_2B_INSTRUCT_QUANTIZED
     ),
     bielik_v3_0_1_5b: pair(M.BIELIK_V3_0_1_5B, M.BIELIK_V3_0_1_5B_QUANTIZED),
+    gemma4_e2b: variant(GEMMA4_E2B_VARIANTS, {
+      ios: 'mlx',
+      android: 'vulkan',
+    }),
     // Multimodal LLMs — same hook/module as plain LLMs, listed here so users
     // pick a model by capability ("LLM") rather than by modality.
     lfm2_5_vl_1_6b: base(M.LFM2_5_VL_1_6B_QUANTIZED),
     lfm2_5_vl_450m: base(M.LFM2_5_VL_450M_QUANTIZED),
+    gemma4_e2b_multimodal: base(M.GEMMA4_E2B_MM),
   },
   classification: {
     efficientnet_v2_s: variant(EFFICIENTNET_V2_S_VARIANTS),

package/src/constants/modelUrls.ts

@@ -125,6 +125,47 @@ export const QWEN3_0_6B_QUANTIZED = {
   generationConfig: QWEN3_GENERATION_CONFIG,
 } as const;
 
+// GEMMA 4 — separate HF repo; tokenizer files live at the e2b root and are
+// shared by all backend variants.
+const GEMMA4_E2B_PREFIX = `${URL_PREFIX}-gemma-4/${VERSION_TAG}/e2b`;
+export const GEMMA4_E2B_MLX_MODEL = `${GEMMA4_E2B_PREFIX}/mlx/gemma4_e2b_mlx_int4.pte`;
+export const GEMMA4_E2B_XNNPACK_MODEL = `${GEMMA4_E2B_PREFIX}/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`;
+export const GEMMA4_E2B_VULKAN_MODEL = `${GEMMA4_E2B_PREFIX}/vulkan/gemma_4_e2b_vulkan_8da4w.pte`;
+export const GEMMA4_E2B_TOKENIZER = `${GEMMA4_E2B_PREFIX}/tokenizer.json`;
+export const GEMMA4_E2B_TOKENIZER_CONFIG = `${GEMMA4_E2B_PREFIX}/tokenizer_config.json`;
+
+const GEMMA4_E2B_MODEL =
+  Platform.OS === `android` ? GEMMA4_E2B_VULKAN_MODEL : GEMMA4_E2B_MLX_MODEL;
+
+const GEMMA4_E2B_MLX_MM = `${URL_PREFIX}-gemma-4-multimodal/${VERSION_TAG}/e2b/mlx/gemma4_e2b_mlx_int4.pte`;
+const GEMMA4_E2B_VULKAN_MM = `${URL_PREFIX}-gemma-4-multimodal/${VERSION_TAG}/e2b/vulkan/gemma_4_e2b_vulkan_8da4w.pte`;
+
+/**
+ * @category Models - LLM
+ */
+export const GEMMA4_E2B = {
+  modelName: 'gemma4-e2b',
+  modelSource: GEMMA4_E2B_MODEL,
+  tokenizerSource: GEMMA4_E2B_TOKENIZER,
+  tokenizerConfigSource: GEMMA4_E2B_TOKENIZER_CONFIG,
+} as const;
+
+/**
+ * @category Models - LLM Multimodal
+ */
+export const GEMMA4_E2B_MM = {
+  modelName: 'gemma4-e2b-multimodal',
+  modelSource:
+    Platform.OS === `android` ? GEMMA4_E2B_VULKAN_MM : GEMMA4_E2B_MLX_MM,
+  tokenizerSource: GEMMA4_E2B_TOKENIZER,
+  tokenizerConfigSource: GEMMA4_E2B_TOKENIZER_CONFIG,
+  capabilities: ['vision', 'audio'],
+  audioConfig: {
+    samplesPerBlock: 7680,
+    tokensPerBlock: 12,
+  },
+} as const;
+
 /**
  * @category Models - LLM
  */
@@ -816,27 +857,27 @@ export const STYLE_TRANSFER_UDNIE_QUANTIZED = {
 // S2T
 export const WHISPER_TINY_EN_TOKENIZER = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/tokenizer.json`;
 export const WHISPER_TINY_EN_MODEL_XNNPACK = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/xnnpack/whisper_tiny_en_xnnpack_fp32.pte`;
-export const WHISPER_TINY_EN_MODEL_COREML = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/coreml/whisper_tiny_en_coreml_fp32.pte`;
+export const WHISPER_TINY_EN_MODEL_COREML = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/coreml/whisper_tiny_en_coreml_fp16.pte`;
 
 export const WHISPER_BASE_EN_TOKENIZER = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/tokenizer.json`;
 export const WHISPER_BASE_EN_MODEL_XNNPACK = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/xnnpack/whisper_base_en_xnnpack_fp32.pte`;
-export const WHISPER_BASE_EN_MODEL_COREML = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/coreml/whisper_base_en_coreml_fp32.pte`;
+export const WHISPER_BASE_EN_MODEL_COREML = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/coreml/whisper_base_en_coreml_fp16.pte`;
 
 export const WHISPER_SMALL_EN_TOKENIZER = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/tokenizer.json`;
 export const WHISPER_SMALL_EN_MODEL_XNNPACK = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/xnnpack/whisper_small_en_xnnpack_fp32.pte`;
-export const WHISPER_SMALL_EN_MODEL_COREML = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/coreml/whisper_small_en_coreml_fp32.pte`;
+export const WHISPER_SMALL_EN_MODEL_COREML = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/coreml/whisper_small_en_coreml_fp16.pte`;
 
 export const WHISPER_TINY_TOKENIZER = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/tokenizer.json`;
 export const WHISPER_TINY_MODEL_XNNPACK = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/xnnpack/whisper_tiny_xnnpack_fp32.pte`;
-export const WHISPER_TINY_MODEL_COREML = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/coreml/whisper_tiny_coreml_fp32.pte`;
+export const WHISPER_TINY_MODEL_COREML = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/coreml/whisper_tiny_coreml_fp16.pte`;
 
 export const WHISPER_BASE_TOKENIZER = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/tokenizer.json`;
 export const WHISPER_BASE_MODEL_XNNPACK = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/xnnpack/whisper_base_xnnpack_fp32.pte`;
-export const WHISPER_BASE_MODEL_COREML = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/coreml/whisper_base_coreml_fp32.pte`;
+export const WHISPER_BASE_MODEL_COREML = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/coreml/whisper_base_coreml_fp16.pte`;
 
 export const WHISPER_SMALL_TOKENIZER = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/tokenizer.json`;
 export const WHISPER_SMALL_MODEL_XNNPACK = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/xnnpack/whisper_small_xnnpack_fp32.pte`;
-export const WHISPER_SMALL_MODEL_COREML = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/coreml/whisper_small_coreml_fp32.pte`;
+export const WHISPER_SMALL_MODEL_COREML = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/coreml/whisper_small_coreml_fp16.pte`;
 
 /**
  * @category Models - Speech To Text

package/src/controllers/LLMController.ts

@@ -1,11 +1,11 @@
-import { ResourceSource } from '../types/common';
 import { ResourceFetcher } from '../utils/ResourceFetcher';
 import { Template } from '@huggingface/jinja';
 import { DEFAULT_CHAT_CONFIG } from '../constants/llmDefaults';
 import {
+  AudioConfig,
   ChatConfig,
   GenerationConfig,
-  LLMCapability,
+  LLMModel,
   LLMTool,
   Message,
   SPECIAL_TOKENS,
@@ -30,6 +30,7 @@ export class LLMController {
   private messageHistoryCallback: (messageHistory: Message[]) => void;
   private isReadyCallback: (isReady: boolean) => void;
   private isGeneratingCallback: (isGenerating: boolean) => void;
+  private audioConfig: AudioConfig | undefined;
 
   constructor({
     tokenCallback,
@@ -72,18 +73,10 @@ export class LLMController {
   }
 
   public async load({
-    modelSource,
-    tokenizerSource,
-    tokenizerConfigSource,
-    capabilities,
-    defaultGenerationConfig,
+    model,
     onDownloadProgressCallback,
   }: {
-    modelSource: ResourceSource;
-    tokenizerSource: ResourceSource;
-    tokenizerConfigSource: ResourceSource;
-    capabilities?: readonly LLMCapability[];
-    defaultGenerationConfig?: GenerationConfig;
+    model: LLMModel;
     onDownloadProgressCallback?: (downloadProgress: number) => void;
   }) {
     // reset inner state when loading new model
@@ -94,13 +87,13 @@ export class LLMController {
     try {
       const tokenizersPromise = ResourceFetcher.fetch(
         undefined,
-        tokenizerSource,
-        tokenizerConfigSource
+        model.tokenizerSource,
+        model.tokenizerConfigSource
       );
 
       const modelPromise = ResourceFetcher.fetch(
         onDownloadProgressCallback,
-        modelSource
+        model.modelSource
       );
 
       const [tokenizersResults, modelResult] = await Promise.all([
@@ -124,16 +117,18 @@ export class LLMController {
         this.nativeModule.unload();
       }
 
+      this.audioConfig = model.audioConfig;
+
       this.nativeModule = await global.loadLLM(
         modelPath,
         tokenizerPath,
-        capabilities ?? []
+        model.capabilities ?? []
       );
-      if (defaultGenerationConfig) {
+      if (model.generationConfig) {
         // Apply model-specific recommended sampling defaults before flipping
         // isReady so callers that react to it see the right config on first
         // send. User-provided `configure()` calls still override these.
-        this.applyGenerationConfig(defaultGenerationConfig);
+        this.applyGenerationConfig(model.generationConfig);
       }
       this.isReadyCallback(true);
       this.onToken = (data: string) => {
@@ -236,6 +231,17 @@ export class LLMController {
     return token;
   }
 
+  private getAudioToken(): string {
+    const token = this.tokenizerConfig.audio_token;
+    if (!token) {
+      throw new RnExecutorchError(
+        RnExecutorchErrorCode.InvalidConfig,
+        "Tokenizer config is missing 'audio_token'. Audio-capable models require tokenizerConfigSource with an 'audio_token' field."
+      );
+    }
+    return token;
+  }
+
   private filterSpecialTokens(text: string): string {
     let filtered = text;
     if (
@@ -244,6 +250,12 @@ export class LLMController {
     ) {
       filtered = filtered.replaceAll(this.tokenizerConfig.eos_token, '');
     }
+    if (
+      SPECIAL_TOKENS.EOT_TOKEN in this.tokenizerConfig &&
+      this.tokenizerConfig.eot_token
+    ) {
+      filtered = filtered.replaceAll(this.tokenizerConfig.eot_token, '');
+    }
     if (
       SPECIAL_TOKENS.PAD_TOKEN in this.tokenizerConfig &&
       this.tokenizerConfig.pad_token
@@ -269,25 +281,37 @@ export class LLMController {
     this.isGeneratingCallback(false);
   }
 
-  public async forward(input: string, imagePaths?: string[]): Promise<string> {
+  public async forward(
+    input: string,
+    imagePaths?: string[],
+    audioWaveforms?: Float32Array[]
+  ): Promise<string> {
     if (!this._isReady) {
       throw new RnExecutorchError(RnExecutorchErrorCode.ModuleNotLoaded);
     }
     if (this._isGenerating) {
       throw new RnExecutorchError(RnExecutorchErrorCode.ModelGenerating);
     }
+    const hasImages = !!imagePaths && imagePaths.length > 0;
+    const hasAudio = !!audioWaveforms && audioWaveforms.length > 0;
     try {
       this.isGeneratingCallback(true);
       this.nativeModule.reset();
-      const response =
-        imagePaths && imagePaths.length > 0
-          ? await this.nativeModule.generateMultimodal(
-              input,
-              imagePaths.map(normalizeImagePath),
-              this.getImageToken(),
-              this.onToken
-            )
-          : await this.nativeModule.generate(input, this.onToken);
+      let response: string;
+      if (hasImages || hasAudio) {
+        response = await this.nativeModule.generateMultimodal(
+          input,
+          this.onToken,
+          {
+            imagePaths: hasImages ? imagePaths!.map(normalizeImagePath) : null,
+            imageToken: hasImages ? this.getImageToken() : null,
+            audioWaveforms: hasAudio ? audioWaveforms! : null,
+            audioToken: hasAudio ? this.getAudioToken() : null,
+          }
+        );
+      } else {
+        response = await this.nativeModule.generate(input, this.onToken);
+      }
       return this.filterSpecialTokens(response);
     } catch (e) {
       throw parseUnknownError(e);
@@ -355,7 +379,9 @@ export class LLMController {
     const imagePaths = messages
       .filter((m) => m.mediaPath)
       .map((m) => m.mediaPath!);
-
+    const audioWaveforms = messages
+      .filter((m) => m.audioWaveform)
+      .map((m) => m.audioWaveform!);
     const renderedChat: string = this.applyChatTemplate(
       messages,
       this.tokenizerConfig,
@@ -365,19 +391,22 @@ export class LLMController {
 
     return await this.forward(
       renderedChat,
-      imagePaths.length > 0 ? imagePaths : undefined
+      imagePaths.length > 0 ? imagePaths : undefined,
+      audioWaveforms.length > 0 ? audioWaveforms : undefined
     );
   }
 
   public async sendMessage(
     message: string,
-    media?: { imagePath?: string }
+    media?: { imagePath?: string; audioBuffer?: Float32Array }
   ): Promise<string> {
     const mediaPath = media?.imagePath;
+    const audioBuffer = media?.audioBuffer;
     const newMessage: Message = {
       content: message,
       role: 'user',
       ...(mediaPath ? { mediaPath } : {}),
+      ...(audioBuffer ? { audioWaveform: audioBuffer } : {}),
     };
     const updatedHistory = [...this._messageHistory, newMessage];
     this.messageHistoryCallback(updatedHistory);
@@ -392,7 +421,22 @@ export class LLMController {
       );
       const textTokens = this.nativeModule.countTextTokens(rendered);
       const imageCount = messages.filter((m) => m.mediaPath).length;
-      return textTokens + imageCount * (visualTokenCount - 1);
+      // Audio soft-token expansion: audio_encoder pads samples to
+      // multiples of this.audioConfig.samplesPerBlock (7680 @ 16 kHz) and emits
+      // this.audioConfig.tokensPerBlock (~12) soft tokens per padded block. The
+      // rendered template only contributes 1 token for the audio placeholder,
+      // so add (expansion - 1) per audio message to match prefill consumption.
+      const audioTokenExpansion = messages.reduce((acc, m) => {
+        if (!m.audioWaveform) return acc;
+        const kBlocks = Math.max(
+          1,
+          Math.ceil(m.audioWaveform.length / this.audioConfig!.samplesPerBlock)
+        );
+        return acc + (this.audioConfig!.tokensPerBlock * kBlocks - 1);
+      }, 0);
+      return (
+        textTokens + imageCount * (visualTokenCount - 1) + audioTokenExpansion
+      );
     };
     const maxContextLength = this.nativeModule.getMaxContextLength();
     const messageHistoryWithPrompt =
@@ -497,12 +541,17 @@ function normalizeImagePath(path: string): string {
  * @returns Messages with image-bearing turns rewritten to structured content.
  */
 function messagesForChatTemplate(messages: Message[]): any[] {
-  return messages.map((m) =>
-    m.mediaPath && typeof m.content === 'string'
-      ? {
-          ...m,
-          content: [{ type: 'image' }, { type: 'text', text: m.content }],
-        }
-      : m
-  );
+  return messages.map((m) => {
+    if (typeof m.content !== 'string') return m;
+    const hasImage = !!m.mediaPath;
+    const hasAudio = !!m.audioWaveform;
+    if (!hasImage && !hasAudio) return m;
+    const parts: any[] = [];
+    if (hasImage) parts.push({ type: 'image' });
+    if (hasAudio) parts.push({ type: 'audio' });
+    parts.push({ type: 'text', text: m.content });
+    // Drop the Float32Array on the clone only — passing it into the Jinja
+    // template engine slows render past 3s. Don't mutate m;
+    return { ...m, content: parts, audioWaveform: undefined };
+  });
 }

package/src/hooks/natural_language_processing/useLLM.ts

@@ -58,11 +58,7 @@ export function useLLM({
     (async () => {
       try {
         await controllerInstance.load({
-          modelSource: model.modelSource,
-          tokenizerSource: model.tokenizerSource,
-          tokenizerConfigSource: model.tokenizerConfigSource!,
-          capabilities: model.capabilities,
-          defaultGenerationConfig: model.generationConfig,
+          model: model,
           onDownloadProgressCallback: setDownloadProgress,
         });
       } catch (e) {
@@ -106,7 +102,10 @@ export function useLLM({
   );
 
   const sendMessage = useCallback(
-    (message: string, media?: { imagePath?: string }) => {
+    (
+      message: string,
+      media?: { imagePath?: string; audioBuffer?: Float32Array }
+    ) => {
       setResponse('');
       return controllerInstance.sendMessage(message, media);
     },