react-native-litert-lm 0.4.0 → 0.4.2

package/scripts/framework-source.js

@@ -0,0 +1,46 @@
+/**
+ * framework-source.js
+ *
+ * Single source of truth for the prebuilt iOS framework artifact, shared by
+ * scripts/postinstall.js, scripts/download-ios-frameworks.sh, and
+ * scripts/check-framework-release.js, so the asset name, tag, and URL can never
+ * drift between the install path and the release-time guardrail.
+ *
+ * The artifact is Google's canonical LiteRT-LM release (NOT a per-version
+ * re-host of this package), pinned to the LiteRT-LM **engine** version
+ * (`litertLm.iosGitTag` in package.json), NOT this wrapper's npm version. So:
+ *   - patch releases of this wrapper reuse the same framework (no re-upload),
+ *   - there is no per-release asset for the maintainer to forget to upload,
+ *   - the guardrail checks the exact URL consumers fetch.
+ *
+ * Override the host/asset (e.g. to point at a private mirror) via env vars:
+ *   LITERT_FRAMEWORK_REPO   (default: google-ai-edge/LiteRT-LM)
+ *   LITERT_FRAMEWORK_ASSET  (default: CLiteRTLM.xcframework.zip)
+ */
+
+const path = require('path');
+const packageJson = require('../package.json');
+
+/** GitHub repo that hosts the framework release asset. */
+const GITHUB_REPO = process.env.LITERT_FRAMEWORK_REPO || 'google-ai-edge/LiteRT-LM';
+
+/** Release asset filename. Must match the file on the GitHub release. */
+const ASSET_NAME = process.env.LITERT_FRAMEWORK_ASSET || 'CLiteRTLM.xcframework.zip';
+
+/** Release tag the asset lives under — the LiteRT-LM engine git tag, e.g. "v0.12.0". */
+const FRAMEWORK_TAG = packageJson.litertLm.iosGitTag;
+
+const PACKAGE_ROOT = path.resolve(__dirname, '..');
+const FRAMEWORKS_DIR = path.join(PACKAGE_ROOT, 'ios', 'Frameworks');
+
+/** Fully-resolved download URL for the framework zip. */
+const ASSET_URL = `https://github.com/${GITHUB_REPO}/releases/download/${FRAMEWORK_TAG}/${ASSET_NAME}`;
+
+module.exports = {
+  GITHUB_REPO,
+  ASSET_NAME,
+  FRAMEWORK_TAG,
+  PACKAGE_ROOT,
+  FRAMEWORKS_DIR,
+  ASSET_URL,
+};

package/scripts/postinstall.js

@@ -5,10 +5,15 @@
  * Downloads prebuilt LiteRT-LM iOS frameworks from this package's GitHub
  * releases when consumers run `npm install react-native-litert-lm`.
  *
+ * The framework is intentionally NOT shipped inside the npm tarball (it is
+ * ~40MB and irrelevant to Android-only consumers). The asset name, tag, and
+ * URL come from scripts/framework-source.js so they stay in lockstep with the
+ * release-time guardrail (scripts/check-framework-release.js).
+ *
  * Skips download if:
  *   - Not on macOS (iOS builds require macOS)
  *   - Frameworks already exist
- *   - CI environment with SKIP_IOS_FRAMEWORK_DOWNLOAD=1
+ *   - SKIP_IOS_FRAMEWORK_DOWNLOAD=1 (e.g. Android-only / CI builds)
  */
 
 const { execSync } = require('child_process');
@@ -16,14 +21,12 @@ const fs = require('fs');
 const path = require('path');
 const https = require('https');
 
-const PACKAGE_JSON = require('../package.json');
-const PACKAGE_VERSION = PACKAGE_JSON.version;
-const GITHUB_REPO = 'hung-yueh/react-native-litert-lm';
-const ASSET_NAME = 'LiteRTLM-ios-frameworks.zip';
+const { ASSET_URL, FRAMEWORKS_DIR, FRAMEWORK_TAG } = require('./framework-source');
 
-const SCRIPT_DIR = __dirname;
-const PACKAGE_ROOT = path.resolve(SCRIPT_DIR, '..');
-const FRAMEWORKS_DIR = path.join(PACKAGE_ROOT, 'ios', 'Frameworks');
+/** ZIP local-file-header magic ("PK\x03\x04"). Guards against truncated downloads / HTML error pages. */
+const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
+/** A valid framework zip is well over this; anything smaller is certainly an error page. */
+const MIN_VALID_BYTES = 1024 * 1024; // 1 MB
 
 function log(msg) {
   console.log(`[react-native-litert-lm] ${msg}`);
@@ -74,25 +77,46 @@ function downloadFile(url, destPath, maxRedirects = 5) {
       const file = fs.createWriteStream(destPath);
       res.pipe(file);
       file.on('finish', () => {
-        file.close();
-        resolve();
+        file.close(resolve);
       });
       file.on('error', reject);
     }).on('error', reject);
   });
 }
 
+/**
+ * Verify the downloaded file is a non-trivial ZIP before we trust it.
+ * Catches GitHub HTML error pages and truncated downloads that would
+ * otherwise fail cryptically at `unzip` or, worse, link a corrupt framework.
+ */
+function assertValidZip(zipPath) {
+  const { size } = fs.statSync(zipPath);
+  if (size < MIN_VALID_BYTES) {
+    throw new Error(`downloaded asset is only ${size} bytes — expected a multi-MB framework zip (likely an error page or truncated download)`);
+  }
+
+  const header = Buffer.alloc(4);
+  const fd = fs.openSync(zipPath, 'r');
+  try {
+    fs.readSync(fd, header, 0, 4, 0);
+  } finally {
+    fs.closeSync(fd);
+  }
+  if (!header.equals(ZIP_MAGIC)) {
+    throw new Error('downloaded asset is not a valid ZIP (bad magic bytes)');
+  }
+}
+
 async function main() {
   if (shouldSkip()) return;
 
-  const releaseUrl = `https://github.com/${GITHUB_REPO}/releases/download/v${PACKAGE_VERSION}/${ASSET_NAME}`;
-
-  log(`Downloading iOS frameworks from: ${releaseUrl}`);
+  log(`Downloading iOS frameworks (engine ${FRAMEWORK_TAG}) from: ${ASSET_URL}`);
 
-  const tmpZip = path.join(PACKAGE_ROOT, '.ios-frameworks-tmp.zip');
+  const tmpZip = path.join(path.dirname(FRAMEWORKS_DIR), '.ios-frameworks-tmp.zip');
 
   try {
-    await downloadFile(releaseUrl, tmpZip);
+    await downloadFile(ASSET_URL, tmpZip);
+    assertValidZip(tmpZip);
 
     // Extract
     fs.mkdirSync(FRAMEWORKS_DIR, { recursive: true });
@@ -111,7 +135,6 @@ async function main() {
     log('Run: ./scripts/download-ios-frameworks.sh to download manually.');
 
     // Fail fast on macOS so users discover the problem now, not at Xcode link time.
-    // Skip SKIP_IOS_FRAMEWORK_DOWNLOAD is already checked above.
     if (process.platform === 'darwin') {
       log('Set SKIP_IOS_FRAMEWORK_DOWNLOAD=1 to suppress this error (e.g. Android-only builds).');
       process.exit(1);

package/src/__mocks__/react-native-nitro-modules.ts

@@ -15,6 +15,16 @@ export const mockLiteRTLM = {
     onToken("token", true);
     return Promise.resolve();
   }),
+  sendMessageWithImageAsync: jest.fn((msg, imagePath, onToken) => {
+    onToken("Mock vision ", false);
+    onToken("token", true);
+    return Promise.resolve();
+  }),
+  sendMessageWithAudioAsync: jest.fn((msg, audioPath, onToken) => {
+    onToken("Mock audio ", false);
+    onToken("token", true);
+    return Promise.resolve();
+  }),
   getHistory: jest.fn(() => []),
   resetConversation: jest.fn(),
   getStats: jest.fn(() => ({

package/src/__tests__/modelFactory.test.ts

@@ -56,6 +56,34 @@ describe('modelFactory Security & Proxy Unit Tests', () => {
     expect(mockLiteRTLM.getMemoryUsage).toHaveBeenCalled();
   });
 
+  it('should successfully proxy sendMessageWithImageAsync and record memory metrics when done', async () => {
+    const onToken = jest.fn();
+    await llm.sendMessageWithImageAsync("Vision prompt", "/path/to/image.jpg", onToken);
+
+    expect(onToken).toHaveBeenCalledWith("Mock vision ", false);
+    expect(onToken).toHaveBeenCalledWith("token", true);
+    expect(mockLiteRTLM.sendMessageWithImageAsync).toHaveBeenCalledWith(
+      "Vision prompt",
+      "/path/to/image.jpg",
+      expect.any(Function)
+    );
+    expect(mockLiteRTLM.getMemoryUsage).toHaveBeenCalled();
+  });
+
+  it('should successfully proxy sendMessageWithAudioAsync and record memory metrics when done', async () => {
+    const onToken = jest.fn();
+    await llm.sendMessageWithAudioAsync("Audio prompt", "/path/to/audio.wav", onToken);
+
+    expect(onToken).toHaveBeenCalledWith("Mock audio ", false);
+    expect(onToken).toHaveBeenCalledWith("token", true);
+    expect(mockLiteRTLM.sendMessageWithAudioAsync).toHaveBeenCalledWith(
+      "Audio prompt",
+      "/path/to/audio.wav",
+      expect.any(Function)
+    );
+    expect(mockLiteRTLM.getMemoryUsage).toHaveBeenCalled();
+  });
+
   it('should successfully access memoryTracker and getSnapshots when memory tracking is enabled', () => {
     expect(llm.memoryTracker).toBeDefined();
     expect(llm.memoryTracker?.getCapacity()).toBe(256);

package/src/hooks.ts

@@ -64,7 +64,8 @@ export function useModel(
   const [isGenerating, setIsGenerating] = useState(false);
   const [downloadProgress, setDownloadProgress] = useState(0);
   const [error, setError] = useState<string | null>(null);
-  const [memorySummary, setMemorySummary] = useState<MemoryTrackerSummary | null>(null);
+  const [memorySummary, setMemorySummary] =
+    useState<MemoryTrackerSummary | null>(null);
 
   // Destructure config into primitive values for stable dependency arrays.
   // This prevents infinite re-render loops when consumers pass inline config
@@ -78,6 +79,11 @@ export function useModel(
   const temperature = config?.temperature;
   const topK = config?.topK;
   const topP = config?.topP;
+  const validate = config?.validate;
+  const multimodal = config?.multimodal;
+  const tools = config?.tools;
+  const enableSpeculativeDecoding = config?.enableSpeculativeDecoding;
+  const toolsKey = tools ? JSON.stringify(tools) : undefined;
 
   // Build a stable config object from the destructured primitives
   const nativeConfig = useMemo<LLMConfig>(
@@ -88,8 +94,25 @@ export function useModel(
       ...(temperature !== undefined && { temperature }),
       ...(topK !== undefined && { topK }),
       ...(topP !== undefined && { topP }),
+      ...(validate !== undefined && { validate }),
+      ...(multimodal !== undefined && { multimodal }),
+      ...(tools !== undefined && { tools }),
+      ...(enableSpeculativeDecoding !== undefined && {
+        enableSpeculativeDecoding,
+      }),
     }),
-    [backend, systemPrompt, maxTokens, temperature, topK, topP],
+    [
+      backend,
+      systemPrompt,
+      maxTokens,
+      temperature,
+      topK,
+      topP,
+      validate,
+      multimodal,
+      toolsKey,
+      enableSpeculativeDecoding,
+    ],
   );
 
   /**
@@ -165,16 +188,15 @@ export function useModel(
         return new Promise<string>((resolve, reject) => {
           let fullResponse = "";
           try {
-            modelRef.current?.sendMessageAsync(
-              prompt,
-              (token: string, done: boolean) => {
+            modelRef.current
+              ?.sendMessageAsync(prompt, (token: string, done: boolean) => {
                 fullResponse += token;
                 if (done) {
                   refreshMemorySummary();
                   resolve(fullResponse);
                 }
-              },
-            ).catch(reject);
+              })
+              .catch(reject);
           } catch (e: any) {
             reject(e);
           }

package/src/index.ts

@@ -151,6 +151,12 @@ export function checkBackendSupport(backend: Backend): string | undefined {
  * Check if multimodal features (image/audio) are supported on the current platform.
  * Returns an error message if not supported, undefined if OK.
  *
+ * Both iOS (v0.12.0 CLiteRTLM xcframework) and Android (LiteRT-LM SDK) ship the
+ * vision/audio executor ops, so there is no platform-level block. Whether a
+ * given call succeeds depends on the **loaded model**: only multimodal models
+ * (e.g. Gemma 3n) bundle the vision/audio executors. Pass `multimodal: true` to
+ * `loadModel` for such models, or rely on filename sniffing ("3n"/"gemma3").
+ *
  * @returns Error message if multimodal is not supported, undefined if OK
  *
  * @example
@@ -165,9 +171,7 @@ export function checkBackendSupport(backend: Backend): string | undefined {
  * ```
  */
 export function checkMultimodalSupport(): string | undefined {
-  if (Platform.OS === "ios") {
-    return "Multimodal (image/audio) is not available on iOS. The XCFramework lacks compiled vision and audio executor ops.";
-  }
+  // Supported on both platforms with a multimodal model loaded.
   return undefined;
 }
 

package/src/modelFactory.ts

@@ -132,6 +132,28 @@ export function createLLM(options?: {
         };
       }
 
+      if (prop === "sendMessageWithImageAsync") {
+        return (message: string, imagePath: string, onToken: (token: string, done: boolean) => void) => {
+          return original.call(target, message, imagePath, (token: string, done: boolean) => {
+            onToken(token, done);
+            if (done) {
+              recordMemorySnapshot();
+            }
+          });
+        };
+      }
+
+      if (prop === "sendMessageWithAudioAsync") {
+        return (message: string, audioPath: string, onToken: (token: string, done: boolean) => void) => {
+          return original.call(target, message, audioPath, (token: string, done: boolean) => {
+            onToken(token, done);
+            if (done) {
+              recordMemorySnapshot();
+            }
+          });
+        };
+      }
+
       if (SNAPSHOT_TRIGGERS.has(prop as string)) {
         return async (...args: any[]) => {
           const result = await original.apply(target, args);

package/src/specs/LiteRTLM.nitro.ts

@@ -226,6 +226,19 @@ export interface LiteRTLM extends HybridObject<{
    */
   sendMessageWithImage(message: string, imagePath: string): Promise<string>;
 
+  /**
+   * Send a text message with an image and get a streaming response.
+   * Tokens are delivered via callback as they are generated.
+   * @param message User message text.
+   * @param imagePath Absolute path to an image file.
+   * @param onToken Callback invoked for each token (token, isDone).
+   */
+  sendMessageWithImageAsync(
+    message: string,
+    imagePath: string,
+    onToken: (token: string, done: boolean) => void,
+  ): Promise<void>;
+
   /**
    * Download a model file from a URL.
    * @param url URL to download from.
@@ -253,6 +266,19 @@ export interface LiteRTLM extends HybridObject<{
    */
   sendMessageWithAudio(message: string, audioPath: string): Promise<string>;
 
+  /**
+   * Send a text message with audio and get a streaming response.
+   * Tokens are delivered via callback as they are generated.
+   * @param message User message text.
+   * @param audioPath Absolute path to an audio file (WAV).
+   * @param onToken Callback invoked for each token (token, isDone).
+   */
+  sendMessageWithAudioAsync(
+    message: string,
+    audioPath: string,
+    onToken: (token: string, done: boolean) => void,
+  ): Promise<void>;
+
   /**
    * Send a unified multimodal message containing text and/or zero-copy binary buffers.
    * @param parts The message content parts (text, image, and/or audio).