react-native-litert-lm 0.3.6 → 0.4.0

package/src/modelFactory.ts

@@ -54,92 +54,94 @@ export function createLLM(options?: {
     }
   };
 
-  return {
-    ...native,
-    memoryTracker: tracker,
-    loadModel: async (
-      pathOrUrl: string,
-      config?: LLMConfig,
-      onDownloadProgress?: (progress: number) => void,
-    ) => {
-      let modelPath = pathOrUrl;
-
-      // Check if it's a URL — enforce HTTPS for model downloads
-      if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) {
-        if (pathOrUrl.startsWith("http://")) {
-          throw new Error(
-            "Insecure HTTP URLs are not allowed for model downloads. " +
-              "Use HTTPS instead: " +
-              pathOrUrl.replace("http://", "https://"),
-          );
-        }
-
-        // Extract filename from URL
-        const fileName = pathOrUrl.split("/").pop();
-        if (!fileName) {
-          throw new Error(`Invalid model URL: ${pathOrUrl}`);
-        }
-
-        console.log(`Checking model at ${pathOrUrl}...`);
-        modelPath = await native.downloadModel(
-          pathOrUrl,
-          fileName,
-          (progress) => {
-            onDownloadProgress?.(progress);
-          },
+  const augmentedLoadModel = async (
+    pathOrUrl: string,
+    config?: LLMConfig,
+    onDownloadProgress?: (progress: number) => void,
+  ) => {
+    let modelPath = pathOrUrl;
+
+    // Check if it's a URL — enforce HTTPS for model downloads
+    if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) {
+      if (pathOrUrl.startsWith("http://")) {
+        throw new Error(
+          "Insecure HTTP URLs are not allowed for model downloads. " +
+            "Use HTTPS instead: " +
+            pathOrUrl.replace("http://", "https://"),
         );
-        console.log(`Model downloaded to: ${modelPath}`);
       }
 
-      const result = await native.loadModel(modelPath, config);
-
-      // Record initial memory snapshot after model load
-      if (tracker) {
-        tracker.reset();
-        recordMemorySnapshot();
+      // Extract filename from URL, stripping query parameters
+      const urlWithoutQuery = pathOrUrl.split("?")[0];
+      const fileName = urlWithoutQuery.split("/").pop();
+      if (!fileName) {
+        throw new Error(`Invalid model URL: ${pathOrUrl}`);
       }
 
-      return result;
-    },
-    sendMessage: async (...args: Parameters<typeof native.sendMessage>) => {
-      const result = await native.sendMessage(...args);
+      console.log(`Checking model at ${pathOrUrl}...`);
+      modelPath = await native.downloadModel(
+        pathOrUrl,
+        fileName,
+        (progress) => {
+          onDownloadProgress?.(progress);
+        },
+      );
+      console.log(`Model downloaded to: ${modelPath}`);
+    }
+
+    const result = await native.loadModel(modelPath, config);
+
+    // Record initial memory snapshot after model load
+    if (tracker) {
+      tracker.reset();
       recordMemorySnapshot();
-      return result;
-    },
-    sendMessageAsync: (...args: Parameters<typeof native.sendMessageAsync>) => {
-      const [message, onToken] = args;
-      native.sendMessageAsync(message, (token, done) => {
-        onToken(token, done);
-        if (done) {
+    }
+
+    return result;
+  };
+
+  const SNAPSHOT_TRIGGERS = new Set([
+    "sendMessage",
+    "sendMessageWithImage",
+    "sendMessageWithAudio",
+    "resetConversation",
+  ]);
+
+  return new Proxy(native, {
+    get(target, prop, receiver) {
+      if (prop === "memoryTracker") {
+        return tracker;
+      }
+      if (prop === "loadModel") {
+        return augmentedLoadModel;
+      }
+
+      const original = Reflect.get(target, prop, receiver);
+      if (typeof original !== "function") {
+        return original;
+      }
+
+      if (prop === "sendMessageAsync") {
+        return (message: string, onToken: (token: string, done: boolean) => void) => {
+          return original.call(target, message, (token: string, done: boolean) => {
+            onToken(token, done);
+            if (done) {
+              recordMemorySnapshot();
+            }
+          });
+        };
+      }
+
+      if (SNAPSHOT_TRIGGERS.has(prop as string)) {
+        return async (...args: any[]) => {
+          const result = await original.apply(target, args);
           recordMemorySnapshot();
-        }
-      });
-    },
-    sendMessageWithImage: async (
-      ...args: Parameters<typeof native.sendMessageWithImage>
-    ) => {
-      const result = await native.sendMessageWithImage(...args);
-      recordMemorySnapshot();
-      return result;
-    },
-    sendMessageWithAudio: async (
-      ...args: Parameters<typeof native.sendMessageWithAudio>
-    ) => {
-      const result = await native.sendMessageWithAudio(...args);
-      recordMemorySnapshot();
-      return result;
-    },
-    getHistory: native.getHistory.bind(native),
-    resetConversation: () => {
-      native.resetConversation();
-      // KV cache is cleared on reset, record the drop
-      recordMemorySnapshot();
+          return result;
+        };
+      }
+
+      return original.bind(target);
     },
-    isReady: native.isReady.bind(native),
-    getStats: native.getStats.bind(native),
-    getMemoryUsage: native.getMemoryUsage.bind(native),
-    close: native.close.bind(native),
-    downloadModel: native.downloadModel.bind(native),
-    deleteModel: native.deleteModel.bind(native),
-  };
+  }) as unknown as LiteRTLMInstance;
 }
+

package/src/specs/LiteRTLM.nitro.ts

@@ -17,6 +17,37 @@ export type Backend = "cpu" | "gpu" | "npu";
  */
 export type Role = "user" | "model" | "system";
 
+/**
+ * Definition for a function/tool that the model can request to execute.
+ */
+export interface ToolDefinition {
+  /** Name of the function/tool */
+  name: string;
+  /** Human-readable description of what the function/tool does */
+  description: string;
+  /** JSON schema defining parameter names and types (stringified) */
+  parametersJson: string;
+}
+
+/**
+ * The part type for a multimodal message content part.
+ */
+export type PartType = "text" | "image" | "audio";
+
+/**
+ * A part of a unified multimodal message payload.
+ */
+export interface MultimodalPart {
+  /** The part type: 'text', 'image', or 'audio' */
+  type: PartType;
+  /** The plain text content, if type is 'text' */
+  text?: string;
+  /** Raw image binary data, if type is 'image' (zero-copy ArrayBuffer mapping) */
+  imageBuffer?: ArrayBuffer;
+  /** Raw audio binary data, if type is 'audio' (zero-copy ArrayBuffer mapping) */
+  audioBuffer?: ArrayBuffer;
+}
+
 /**
  * Configuration options for loading an LLM.
  */
@@ -68,6 +99,41 @@ export interface LLMConfig {
    * @default 0.95
    */
   topP?: number;
+
+  /**
+   * Whether to run engine validation after loading the model.
+   * When enabled, sends a quick test inference ("Hi") and waits up to 30s
+   * for a response to confirm the backend works. This is useful for GPU/NPU
+   * backends that may silently fail during inference (they can initialize
+   * without error but produce no tokens).
+   *
+   * Validation is **always a no-op on CPU** — the CPU backend is inherently
+   * reliable and never needs validation.
+   *
+   * Disabled by default because it adds significant latency (5-30s) to model loading.
+   * Enable only to catch GPU/NPU silent failure issues during development.
+   *
+   * @default false
+   */
+  validate?: boolean;
+
+  /**
+   * Whether this is a multimodal model.
+   * When enabled, the engine handles image/audio tokens properly.
+   * If not specified, the system will fall back to filename sniffing.
+   */
+  multimodal?: boolean;
+
+  /**
+   * List of tools/functions that the model can call.
+   */
+  tools?: ToolDefinition[];
+
+  /**
+   * Whether to enable speculative decoding (multi-token prediction) if supported by the model.
+   * @default false
+   */
+  enableSpeculativeDecoding?: boolean;
 }
 
 /**
@@ -135,7 +201,7 @@ export interface MemoryUsage {
  * ```
  */
 export interface LiteRTLM extends HybridObject<{
-  ios: "c++";
+  ios: "swift";
   android: "kotlin";
 }> {
   /**
@@ -187,6 +253,13 @@ export interface LiteRTLM extends HybridObject<{
    */
   sendMessageWithAudio(message: string, audioPath: string): Promise<string>;
 
+  /**
+   * Send a unified multimodal message containing text and/or zero-copy binary buffers.
+   * @param parts The message content parts (text, image, and/or audio).
+   * @returns The model's response text.
+   */
+  sendMultimodalMessage(parts: MultimodalPart[]): Promise<string>;
+
   /**
    * Send a message with streaming response.
    * Tokens are delivered via callback as they are generated.
@@ -196,7 +269,7 @@ export interface LiteRTLM extends HybridObject<{
   sendMessageAsync(
     message: string,
     onToken: (token: string, done: boolean) => void,
-  ): void;
+  ): Promise<void>;
 
   /**
    * Get the current conversation history.
@@ -219,6 +292,11 @@ export interface LiteRTLM extends HybridObject<{
    */
   getStats(): GenerationStats;
 
+  /**
+   * Count tokens in a text string. Returns -1 if unavailable.
+   */
+  countTokens(text: string): number;
+
   /**
    * Get real memory usage from the native runtime.
    * Uses OS-level APIs to report actual memory consumption.