react-native-litert-lm 0.2.0 → 0.2.1

package/nitrogen/generated/android/kotlin/com/margelo/nitro/dev/litert/litertlm/Func_void_double.kt

@@ -0,0 +1,80 @@
+///
+/// Func_void_double.kt
+/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
+/// https://github.com/mrousavy/nitro
+/// Copyright © Marc Rousavy @ Margelo
+///
+
+package com.margelo.nitro.dev.litert.litertlm
+
+import androidx.annotation.Keep
+import com.facebook.jni.HybridData
+import com.facebook.proguard.annotations.DoNotStrip
+import dalvik.annotation.optimization.FastNative
+
+
+/**
+ * Represents the JavaScript callback `(progress: number) => void`.
+ * This can be either implemented in C++ (in which case it might be a callback coming from JS),
+ * or in Kotlin/Java (in which case it is a native callback).
+ */
+@DoNotStrip
+@Keep
+@Suppress("ClassName", "RedundantUnitReturnType")
+fun interface Func_void_double: (Double) -> Unit {
+  /**
+   * Call the given JS callback.
+   * @throws Throwable if the JS function itself throws an error, or if the JS function/runtime has already been deleted.
+   */
+  @DoNotStrip
+  @Keep
+  override fun invoke(progress: Double): Unit
+}
+
+/**
+ * Represents the JavaScript callback `(progress: number) => void`.
+ * This is implemented in C++, via a `std::function<...>`.
+ * The callback might be coming from JS.
+ */
+@DoNotStrip
+@Keep
+@Suppress(
+  "KotlinJniMissingFunction", "unused",
+  "RedundantSuppression", "RedundantUnitReturnType", "FunctionName",
+  "ConvertSecondaryConstructorToPrimary", "ClassName", "LocalVariableName",
+)
+class Func_void_double_cxx: Func_void_double {
+  @DoNotStrip
+  @Keep
+  private val mHybridData: HybridData
+
+  @DoNotStrip
+  @Keep
+  private constructor(hybridData: HybridData) {
+    mHybridData = hybridData
+  }
+
+  @DoNotStrip
+  @Keep
+  override fun invoke(progress: Double): Unit
+    = invoke_cxx(progress)
+
+  @FastNative
+  private external fun invoke_cxx(progress: Double): Unit
+}
+
+/**
+ * Represents the JavaScript callback `(progress: number) => void`.
+ * This is implemented in Java/Kotlin, via a `(Double) -> Unit`.
+ * The callback is always coming from native.
+ */
+@DoNotStrip
+@Keep
+@Suppress("ClassName", "RedundantUnitReturnType", "unused")
+class Func_void_double_java(private val function: (Double) -> Unit): Func_void_double {
+  @DoNotStrip
+  @Keep
+  override fun invoke(progress: Double): Unit {
+    return this.function(progress)
+  }
+}

package/nitrogen/generated/android/kotlin/com/margelo/nitro/dev/litert/litertlm/HybridLiteRTLMSpec.kt

@@ -58,6 +58,19 @@ abstract class HybridLiteRTLMSpec: HybridObject() {
   @Keep
   abstract fun sendMessageWithImage(message: String, imagePath: String): Promise<String>
   
+  abstract fun downloadModel(url: String, fileName: String, onProgress: ((progress: Double) -> Unit)?): Promise<String>
+  
+  @DoNotStrip
+  @Keep
+  private fun downloadModel_cxx(url: String, fileName: String, onProgress: Func_void_double?): Promise<String> {
+    val __result = downloadModel(url, fileName, onProgress?.let { it })
+    return __result
+  }
+  
+  @DoNotStrip
+  @Keep
+  abstract fun deleteModel(fileName: String): Promise<Unit>
+  
   @DoNotStrip
   @Keep
   abstract fun sendMessageWithAudio(message: String, audioPath: String): Promise<String>

package/nitrogen/generated/android/kotlin/com/margelo/nitro/dev/litert/litertlm/LLMConfig.kt

@@ -17,6 +17,9 @@ import com.facebook.proguard.annotations.DoNotStrip
 @DoNotStrip
 @Keep
 data class LLMConfig(
+  @DoNotStrip
+  @Keep
+  val systemPrompt: String?,
   @DoNotStrip
   @Keep
   val backend: Backend?,
@@ -43,8 +46,8 @@ data class LLMConfig(
     @Keep
     @Suppress("unused")
     @JvmStatic
-    private fun fromCpp(backend: Backend?, maxTokens: Double?, temperature: Double?, topK: Double?, topP: Double?): LLMConfig {
-      return LLMConfig(backend, maxTokens, temperature, topK, topP)
+    private fun fromCpp(systemPrompt: String?, backend: Backend?, maxTokens: Double?, temperature: Double?, topK: Double?, topP: Double?): LLMConfig {
+      return LLMConfig(systemPrompt, backend, maxTokens, temperature, topK, topP)
     }
   }
 }

package/nitrogen/generated/shared/c++/HybridLiteRTLMSpec.cpp

@@ -17,6 +17,8 @@ namespace margelo::nitro::litertlm {
       prototype.registerHybridMethod("loadModel", &HybridLiteRTLMSpec::loadModel);
       prototype.registerHybridMethod("sendMessage", &HybridLiteRTLMSpec::sendMessage);
       prototype.registerHybridMethod("sendMessageWithImage", &HybridLiteRTLMSpec::sendMessageWithImage);
+      prototype.registerHybridMethod("downloadModel", &HybridLiteRTLMSpec::downloadModel);
+      prototype.registerHybridMethod("deleteModel", &HybridLiteRTLMSpec::deleteModel);
       prototype.registerHybridMethod("sendMessageWithAudio", &HybridLiteRTLMSpec::sendMessageWithAudio);
       prototype.registerHybridMethod("sendMessageAsync", &HybridLiteRTLMSpec::sendMessageAsync);
       prototype.registerHybridMethod("getHistory", &HybridLiteRTLMSpec::getHistory);

package/nitrogen/generated/shared/c++/HybridLiteRTLMSpec.hpp

@@ -63,6 +63,8 @@ namespace margelo::nitro::litertlm {
       virtual std::shared_ptr<Promise<void>> loadModel(const std::string& modelPath, const std::optional<LLMConfig>& config) = 0;
       virtual std::shared_ptr<Promise<std::string>> sendMessage(const std::string& message) = 0;
       virtual std::shared_ptr<Promise<std::string>> sendMessageWithImage(const std::string& message, const std::string& imagePath) = 0;
+      virtual std::shared_ptr<Promise<std::string>> downloadModel(const std::string& url, const std::string& fileName, const std::optional<std::function<void(double /* progress */)>>& onProgress) = 0;
+      virtual std::shared_ptr<Promise<void>> deleteModel(const std::string& fileName) = 0;
       virtual std::shared_ptr<Promise<std::string>> sendMessageWithAudio(const std::string& message, const std::string& audioPath) = 0;
       virtual void sendMessageAsync(const std::string& message, const std::function<void(const std::string& /* token */, bool /* done */)>& onToken) = 0;
       virtual std::vector<Message> getHistory() = 0;

package/nitrogen/generated/shared/c++/LLMConfig.hpp

@@ -31,8 +31,9 @@
 // Forward declaration of `Backend` to properly resolve imports.
 namespace margelo::nitro::litertlm { enum class Backend; }
 
-#include "Backend.hpp"
+#include <string>
 #include <optional>
+#include "Backend.hpp"
 
 namespace margelo::nitro::litertlm {
 
@@ -41,6 +42,7 @@ namespace margelo::nitro::litertlm {
    */
   struct LLMConfig final {
   public:
+    std::optional<std::string> systemPrompt     SWIFT_PRIVATE;
     std::optional<Backend> backend     SWIFT_PRIVATE;
     std::optional<double> maxTokens     SWIFT_PRIVATE;
     std::optional<double> temperature     SWIFT_PRIVATE;
@@ -49,7 +51,7 @@ namespace margelo::nitro::litertlm {
 
   public:
     LLMConfig() = default;
-    explicit LLMConfig(std::optional<Backend> backend, std::optional<double> maxTokens, std::optional<double> temperature, std::optional<double> topK, std::optional<double> topP): backend(backend), maxTokens(maxTokens), temperature(temperature), topK(topK), topP(topP) {}
+    explicit LLMConfig(std::optional<std::string> systemPrompt, std::optional<Backend> backend, std::optional<double> maxTokens, std::optional<double> temperature, std::optional<double> topK, std::optional<double> topP): systemPrompt(systemPrompt), backend(backend), maxTokens(maxTokens), temperature(temperature), topK(topK), topP(topP) {}
 
   public:
     friend bool operator==(const LLMConfig& lhs, const LLMConfig& rhs) = default;
@@ -65,6 +67,7 @@ namespace margelo::nitro {
     static inline margelo::nitro::litertlm::LLMConfig fromJSI(jsi::Runtime& runtime, const jsi::Value& arg) {
       jsi::Object obj = arg.asObject(runtime);
       return margelo::nitro::litertlm::LLMConfig(
+        JSIConverter<std::optional<std::string>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "systemPrompt"))),
         JSIConverter<std::optional<margelo::nitro::litertlm::Backend>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "backend"))),
         JSIConverter<std::optional<double>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "maxTokens"))),
         JSIConverter<std::optional<double>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "temperature"))),
@@ -74,6 +77,7 @@ namespace margelo::nitro {
     }
     static inline jsi::Value toJSI(jsi::Runtime& runtime, const margelo::nitro::litertlm::LLMConfig& arg) {
       jsi::Object obj(runtime);
+      obj.setProperty(runtime, PropNameIDCache::get(runtime, "systemPrompt"), JSIConverter<std::optional<std::string>>::toJSI(runtime, arg.systemPrompt));
       obj.setProperty(runtime, PropNameIDCache::get(runtime, "backend"), JSIConverter<std::optional<margelo::nitro::litertlm::Backend>>::toJSI(runtime, arg.backend));
       obj.setProperty(runtime, PropNameIDCache::get(runtime, "maxTokens"), JSIConverter<std::optional<double>>::toJSI(runtime, arg.maxTokens));
       obj.setProperty(runtime, PropNameIDCache::get(runtime, "temperature"), JSIConverter<std::optional<double>>::toJSI(runtime, arg.temperature));
@@ -89,6 +93,7 @@ namespace margelo::nitro {
       if (!nitro::isPlainObject(runtime, obj)) {
         return false;
       }
+      if (!JSIConverter<std::optional<std::string>>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "systemPrompt")))) return false;
       if (!JSIConverter<std::optional<margelo::nitro::litertlm::Backend>>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "backend")))) return false;
       if (!JSIConverter<std::optional<double>>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "maxTokens")))) return false;
       if (!JSIConverter<std::optional<double>>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "temperature")))) return false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-litert-lm",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "High-performance LLM inference for React Native using LiteRT-LM. Optimized for Gemma 3n and other on-device language models.",
   "license": "MIT",
   "author": "Hugh Chen (https://github.com/hung-yueh)",

package/src/hooks.ts

@@ -0,0 +1,152 @@
+import { useState, useEffect, useRef, useCallback } from "react";
+import { LiteRTLM, LLMConfig } from "./index";
+import { createLLM } from "./modelFactory";
+
+export interface UseModelConfig extends LLMConfig {
+  autoLoad?: boolean;
+}
+
+export interface UseModelResult {
+  model: LiteRTLM | null;
+  isReady: boolean;
+  isGenerating: boolean;
+  downloadProgress: number;
+  error: string | null;
+  generate: (prompt: string) => Promise<string>;
+  reset: () => void;
+  deleteModel: (fileName: string) => Promise<void>;
+  load: () => Promise<void>;
+}
+
+export function useModel(
+  pathOrUrl: string,
+  config?: UseModelConfig,
+): UseModelResult {
+  const modelRef = useRef<LiteRTLM | null>(null);
+  const [isReady, setIsReady] = useState(false);
+  const [isGenerating, setIsGenerating] = useState(false);
+  const [downloadProgress, setDownloadProgress] = useState(0);
+  const [error, setError] = useState<string | null>(null);
+
+  // Extract autoLoad (default true)
+  const autoLoad = config?.autoLoad ?? true;
+
+  // Initialize the model instance
+  useEffect(() => {
+    modelRef.current = createLLM();
+    let isMounted = true;
+
+    // Cleanup on unmount
+    return () => {
+      isMounted = false;
+      try {
+        modelRef.current?.close();
+      } catch (e) {
+        console.warn("Failed to close model", e);
+      }
+    };
+  }, []);
+
+  const load = useCallback(async () => {
+    setIsReady(false);
+    setError(null);
+    setDownloadProgress(0);
+
+    try {
+      let modelPath = pathOrUrl;
+
+      // Handle URL download manually to capture progress
+      if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) {
+        const fileName = pathOrUrl.split("/").pop() || "model.bin";
+
+        if (modelRef.current) {
+          modelPath = await modelRef.current.downloadModel(
+            pathOrUrl,
+            fileName,
+            (progress) => {
+              setDownloadProgress(progress);
+            },
+          );
+        }
+      }
+
+      if (modelRef.current) {
+        // Create a clean config object for native loadModel (excluding autoLoad)
+        const nativeConfig: LLMConfig = { ...config };
+        delete (nativeConfig as any).autoLoad;
+
+        await modelRef.current.loadModel(modelPath, nativeConfig);
+        setIsReady(true);
+      }
+    } catch (e: any) {
+      setError(e.message || "Failed to load model");
+      console.error(e);
+    }
+  }, [pathOrUrl, config]);
+
+  useEffect(() => {
+    if (autoLoad) {
+      load();
+    }
+  }, [autoLoad, load]);
+
+  const generate = useCallback(
+    async (prompt: string): Promise<string> => {
+      if (!modelRef.current || !isReady) {
+        throw new Error("Model not ready");
+      }
+
+      setIsGenerating(true);
+      try {
+        return new Promise<string>((resolve, reject) => {
+          let fullResponse = "";
+          try {
+            modelRef.current?.sendMessageAsync(
+              prompt,
+              (token: string, done: boolean) => {
+                fullResponse += token;
+                if (done) {
+                  resolve(fullResponse);
+                }
+              },
+            );
+          } catch (e: any) {
+            reject(e);
+          }
+        });
+      } catch (e: any) {
+        setError(e.message || "Generation failed");
+        throw e;
+      } finally {
+        setIsGenerating(false);
+      }
+    },
+    [isReady],
+  );
+
+  const reset = useCallback(() => {
+    if (modelRef.current) {
+      modelRef.current.resetConversation();
+    }
+  }, []);
+
+  const deleteModel = useCallback(async (fileName: string): Promise<void> => {
+    if (modelRef.current) {
+      await modelRef.current.deleteModel(fileName);
+      setIsReady(false);
+      setDownloadProgress(0);
+    }
+  }, []);
+
+  return {
+    model: modelRef.current,
+    isReady,
+    isGenerating,
+    downloadProgress,
+    error,
+    generate,
+    reset,
+    deleteModel,
+    load,
+  };
+}

package/src/index.ts

@@ -18,6 +18,16 @@ export type {
   GenerationStats,
 } from "./specs/LiteRTLM.nitro";
 
+// Re-export template utilities
+export type { ChatMessage } from "./templates";
+export {
+  applyGemmaTemplate,
+  applyPhiTemplate,
+  applyLlamaTemplate,
+} from "./templates";
+
+export * from "./hooks";
+
 /**
  * Creates a new LiteRT-LM inference engine instance.
  *
@@ -51,9 +61,7 @@ export type {
  * llm.close();
  * ```
  */
-export function createLLM(): LiteRTLM {
-  return NitroModules.createHybridObject<LiteRTLM>("LiteRTLM");
-}
+export { createLLM } from "./modelFactory";
 
 /**
  * Pre-defined model identifiers for common models.
@@ -123,3 +131,33 @@ export function checkBackendSupport(backend: Backend): string | undefined {
 
   return undefined;
 }
+
+/**
+ * Check if multimodal features (image/audio) are supported on the current platform.
+ * Returns an error message if not supported, undefined if OK.
+ *
+ * @returns Error message if multimodal is not supported, undefined if OK
+ *
+ * @example
+ * ```typescript
+ * const error = checkMultimodalSupport();
+ * if (error) {
+ *   console.warn(error);
+ *   // Fall back to text-only
+ * } else {
+ *   llm.sendMessageWithImage('Describe this', imagePath);
+ * }
+ * ```
+ */
+export function checkMultimodalSupport(): string | undefined {
+  if (Platform.OS === "ios") {
+    return "Multimodal (image/audio) is not yet supported on iOS. LiteRT-LM iOS SDK is pending.";
+  }
+  return undefined;
+}
+
+/**
+ * Download URL for the Gemma 3n E2B IT INT4 model.
+ */
+export const GEMMA_3N_E2B_IT_INT4 =
+  "https://litert.dev/gemma-3n-E2B-it-int4.litertlm";

package/src/modelFactory.ts

@@ -0,0 +1,49 @@
+import { NitroModules } from "react-native-nitro-modules";
+import { LiteRTLM, LLMConfig } from "./specs/LiteRTLM.nitro";
+
+/**
+ * Creates a new LiteRT-LM inference engine instance.
+ */
+export function createLLM(): LiteRTLM {
+  const native = NitroModules.createHybridObject<LiteRTLM>("LiteRTLM");
+
+  return {
+    ...native,
+    loadModel: async (pathOrUrl: string, config?: LLMConfig) => {
+      let modelPath = pathOrUrl;
+
+      // Check if it's a URL
+      if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) {
+        // Extract filename from URL
+        const fileName = pathOrUrl.split("/").pop();
+        if (!fileName) {
+          throw new Error(`Invalid model URL: ${pathOrUrl}`);
+        }
+
+        console.log(`Checking model at ${pathOrUrl}...`);
+        modelPath = await native.downloadModel(
+          pathOrUrl,
+          fileName,
+          (progress) => {
+            console.log(`Download progress: ${progress}`);
+          },
+        );
+        console.log(`Model downloaded to: ${modelPath}`);
+      }
+
+      return native.loadModel(modelPath, config);
+    },
+    // Bind valid methods to native instance
+    sendMessage: native.sendMessage.bind(native),
+    sendMessageAsync: native.sendMessageAsync.bind(native),
+    sendMessageWithImage: native.sendMessageWithImage.bind(native),
+    sendMessageWithAudio: native.sendMessageWithAudio.bind(native),
+    getHistory: native.getHistory.bind(native),
+    resetConversation: native.resetConversation.bind(native),
+    isReady: native.isReady.bind(native),
+    getStats: native.getStats.bind(native),
+    close: native.close.bind(native),
+    downloadModel: native.downloadModel.bind(native),
+    deleteModel: native.deleteModel.bind(native),
+  };
+}

package/src/specs/LiteRTLM.nitro.ts

@@ -21,6 +21,13 @@ export type Role = "user" | "model" | "system";
  * Configuration options for loading an LLM.
  */
 export interface LLMConfig {
+  /**
+   * System prompt to set the model's behavior.
+   * This is prepended to the conversation to guide model responses.
+   * @example "You are a helpful coding assistant."
+   */
+  systemPrompt?: string;
+
   /**
    * Primary compute backend for text generation.
    * - 'cpu': CPU inference (slower but always available)
@@ -138,6 +145,25 @@ export interface LiteRTLM extends HybridObject<{
    */
   sendMessageWithImage(message: string, imagePath: string): Promise<string>;
 
+  /**
+   * Download a model file from a URL.
+   * @param url URL to download from.
+   * @param fileName Filename to save as (in app's files directory).
+   * @param onProgress Callback for download progress (0.0 - 1.0).
+   * @returns Absolute path to the downloaded file.
+   */
+  downloadModel(
+    url: string,
+    fileName: string,
+    onProgress?: (progress: number) => void,
+  ): Promise<string>;
+
+  /**
+   * Delete a downloaded model file.
+   * @param fileName Filename to delete (in app's files directory).
+   */
+  deleteModel(fileName: string): Promise<void>;
+
   /**
    * Send a text message with audio (multimodal).
    * @param message User message text.

package/src/templates.ts

@@ -0,0 +1,105 @@
+/**
+ * Prompt template utilities for different LLM families.
+ *
+ * LiteRT-LM's Conversation API may handle templates internally for some models,
+ * but these utilities give developers explicit control for custom workflows
+ * or when using models with different template formats.
+ *
+ * @example
+ * ```typescript
+ * import { applyGemmaTemplate, ChatMessage } from 'react-native-litert-lm';
+ *
+ * const history: ChatMessage[] = [
+ *   { role: 'user', content: 'What is React Native?' },
+ *   { role: 'model', content: 'React Native is a framework for building...' },
+ *   { role: 'user', content: 'How do I use hooks?' }
+ * ];
+ *
+ * const prompt = applyGemmaTemplate(history, 'You are a helpful coding assistant.');
+ * ```
+ */
+
+/**
+ * A message in a conversation.
+ */
+export type ChatMessage = {
+  role: "user" | "model" | "system";
+  content: string;
+};
+
+/**
+ * Apply Gemma chat template (Gemma 2, Gemma 3, Gemma 3n).
+ *
+ * @param history Array of previous messages
+ * @param systemPrompt Optional system prompt
+ * @returns Formatted prompt string
+ */
+export function applyGemmaTemplate(
+  history: ChatMessage[],
+  systemPrompt?: string,
+): string {
+  let result = "";
+
+  if (systemPrompt) {
+    result += `<start_of_turn>system\n${systemPrompt}<end_of_turn>\n`;
+  }
+
+  for (const m of history) {
+    result += `<start_of_turn>${m.role}\n${m.content}<end_of_turn>\n`;
+  }
+
+  result += "<start_of_turn>model\n";
+  return result;
+}
+
+/**
+ * Apply Phi chat template (Phi-3, Phi-4).
+ *
+ * @param history Array of previous messages
+ * @param systemPrompt Optional system prompt
+ * @returns Formatted prompt string
+ */
+export function applyPhiTemplate(
+  history: ChatMessage[],
+  systemPrompt?: string,
+): string {
+  let result = "";
+
+  if (systemPrompt) {
+    result += `<|system|>\n${systemPrompt}<|end|>\n`;
+  }
+
+  for (const m of history) {
+    const role = m.role === "model" ? "assistant" : m.role;
+    result += `<|${role}|>\n${m.content}<|end|>\n`;
+  }
+
+  result += "<|assistant|>\n";
+  return result;
+}
+
+/**
+ * Apply Llama 3 chat template.
+ *
+ * @param history Array of previous messages
+ * @param systemPrompt Optional system prompt
+ * @returns Formatted prompt string
+ */
+export function applyLlamaTemplate(
+  history: ChatMessage[],
+  systemPrompt?: string,
+): string {
+  let result = "<|begin_of_text|>";
+
+  if (systemPrompt) {
+    result += `<|start_header_id|>system<|end_header_id|>\n\n${systemPrompt}<|eot_id|>`;
+  }
+
+  for (const m of history) {
+    const role = m.role === "model" ? "assistant" : m.role;
+    result += `<|start_header_id|>${role}<|end_header_id|>\n\n${m.content}<|eot_id|>`;
+  }
+
+  result += "<|start_header_id|>assistant<|end_header_id|>\n\n";
+  return result;
+}