react-native-litert-lm 0.4.0 → 0.4.1

package/android/src/main/AndroidManifest.xml

@@ -12,5 +12,8 @@
         <uses-native-library
             android:name="libOpenCL.so"
             android:required="false" />
+        <uses-native-library
+            android:name="libvndksupport.so"
+            android:required="false" />
     </application>
 </manifest>

package/android/src/main/java/com/margelo/nitro/dev/litert/litertlm/HybridLiteRTLM.kt

@@ -501,6 +501,75 @@ class HybridLiteRTLM : HybridLiteRTLMSpec() {
         }
     }
 
+    override fun sendMessageWithImageAsync(message: String, imagePath: String, onToken: (String, Boolean) -> Unit): Promise<Unit> {
+        return Promise.parallel {
+            val latch = CountDownLatch(1)
+            val errorRef = AtomicReference<Throwable?>(null)
+
+            ensureLoaded()
+
+            Log.i(TAG, "sendMessageWithImageAsync: $message, path=$imagePath")
+
+            // Resize image to prevent OOM on high-resolution photos
+            val processedImagePath = resizeImageIfNeeded(imagePath)
+
+            val fullResponseBuilder = StringBuilder()
+            
+            val listener = StreamingCallbackListener(
+                onToken = { token, done ->
+                    onToken(token, done)
+                    if (done) {
+                        latch.countDown()
+                    }
+                },
+                responseBuilder = fullResponseBuilder,
+                history = history,
+                userMessage = message,
+                onStatsReady = { stats -> lastStats = stats },
+            )
+
+            try {
+                val textContent = Content.Text(message)
+                val userMsg = LiteRTMessage.user(Contents.of(textContent, Content.ImageFile(processedImagePath)))
+
+                history.add(Message(Role.USER, "$message [Image]"))
+
+                conversation!!.sendMessageAsync(message = userMsg, callback = listener)
+            } catch (e: Exception) {
+                // Clean up temp resized image to prevent cache dir bloat
+                if (processedImagePath != imagePath) {
+                    try {
+                        java.io.File(processedImagePath).delete()
+                    } catch (e: Exception) {
+                        Log.w(TAG, "Failed to clean up temp image: ${e.message}")
+                    }
+                }
+
+                Log.e(TAG, "Failed to initiate async multimodal generation", e)
+                errorRef.set(e)
+                onToken("Error: ${e.message}", true)
+                latch.countDown()
+            }
+
+            // Wait for completion or error
+            latch.await()
+
+            // Clean up temp resized image to prevent cache dir bloat
+            if (processedImagePath != imagePath) {
+                try {
+                    java.io.File(processedImagePath).delete()
+                } catch (e: Exception) {
+                    Log.w(TAG, "Failed to clean up temp image: ${e.message}")
+                }
+            }
+
+            val err = errorRef.get()
+            if (err != null) {
+                throw RuntimeException("Async multimodal inference failed: ${err.message}", err)
+            }
+        }
+    }
+
     override fun downloadModel(url: String, fileName: String, onProgress: ((Double) -> Unit)?): Promise<String> {
         return Promise.parallel {
             Log.i(TAG, "downloadModel: $url -> $fileName")
@@ -623,6 +692,54 @@ class HybridLiteRTLM : HybridLiteRTLMSpec() {
         }
     }
 
+    override fun sendMessageWithAudioAsync(message: String, audioPath: String, onToken: (String, Boolean) -> Unit): Promise<Unit> {
+        return Promise.parallel {
+            val latch = CountDownLatch(1)
+            val errorRef = AtomicReference<Throwable?>(null)
+
+            ensureLoaded()
+
+            Log.i(TAG, "sendMessageWithAudioAsync: $message, path=$audioPath")
+
+            val fullResponseBuilder = StringBuilder()
+
+            val listener = StreamingCallbackListener(
+                onToken = { token, done ->
+                    onToken(token, done)
+                    if (done) {
+                        latch.countDown()
+                    }
+                },
+                responseBuilder = fullResponseBuilder,
+                history = history,
+                userMessage = message,
+                onStatsReady = { stats -> lastStats = stats },
+            )
+
+            try {
+                val userMsg = LiteRTMessage.user(Contents.of(
+                    Content.Text(message),
+                    Content.AudioFile(audioPath)
+                ))
+
+                history.add(Message(Role.USER, "$message [Audio]"))
+
+                conversation!!.sendMessageAsync(message = userMsg, callback = listener)
+            } catch (e: Exception) {
+                Log.e(TAG, "Failed to initiate async audio generation", e)
+                errorRef.set(e)
+                onToken("Error: ${e.message}", true)
+                latch.countDown()
+            }
+
+            latch.await()
+            val err = errorRef.get()
+            if (err != null) {
+                throw RuntimeException("Async audio inference failed: ${err.message}", err)
+            }
+        }
+    }
+
     override fun sendMessageWithAudio(message: String, audioPath: String): Promise<String> {
         return Promise.parallel {
             ensureLoaded()

package/android/src/test/java/com/margelo/nitro/dev/litert/litertlm/HybridLiteRTLMTest.kt

@@ -69,6 +69,28 @@ class HybridLiteRTLMTest {
         assertTrue(mem.availableMemoryBytes >= 0.0)
     }
 
+    @Test
+    fun testSendMessageWithImageAsyncRejectsWithoutModel() {
+        val promise = bridge.sendMessageWithImageAsync("hello", "/tmp/image.jpg") { _, _ -> }
+        assertNotNull("Promise should not be null", promise)
+        assertTrue("Promise should be completed", promise.isCompleted)
+        assertNotNull("Promise should have rejected without model", promise.error)
+        val errMsg = promise.error!!.message ?: promise.error!!.cause?.message ?: ""
+        assertTrue("Expected no-model error, got: $errMsg",
+            errMsg.contains("No model loaded"))
+    }
+
+    @Test
+    fun testSendMessageWithAudioAsyncRejectsWithoutModel() {
+        val promise = bridge.sendMessageWithAudioAsync("hello", "/tmp/audio.wav") { _, _ -> }
+        assertNotNull("Promise should not be null", promise)
+        assertTrue("Promise should be completed", promise.isCompleted)
+        assertNotNull("Promise should have rejected without model", promise.error)
+        val errMsg = promise.error!!.message ?: promise.error!!.cause?.message ?: ""
+        assertTrue("Expected no-model error, got: $errMsg",
+            errMsg.contains("No model loaded"))
+    }
+
     @Test
     fun testAndroidInitialStats() {
         val stats = bridge.getStats()

package/ios/HybridLiteRTLM.swift

@@ -117,7 +117,7 @@ public class HybridLiteRTLM: HybridLiteRTLMSpec_base, HybridLiteRTLMSpec_protoco
     }
     
     public func countTokens(text: String) throws -> Double {
-        return try queue.sync {
+        return queue.sync {
             guard let engine = self.engine else {
                 return -1.0
             }
@@ -406,42 +406,50 @@ public class HybridLiteRTLM: HybridLiteRTLMSpec_base, HybridLiteRTLMSpec_protoco
                         ctx.onToken(remaining, false)
                     }
                     ctx.fullResponse = finalCleaned
-                    
-                    var completionTokens = Double(ctx.tokenCount)
-                    var tokensPerSecond = 0.0
-                    var ttft = 0.0
-                    
-                    if let benchInfo = litert_lm_conversation_get_benchmark_info(ctx.parent.conversation) {
-                        let numDecodeTurns = litert_lm_benchmark_info_get_num_decode_turns(benchInfo)
-                        if numDecodeTurns > 0 {
-                            let lastIdx = numDecodeTurns - 1
-                            tokensPerSecond = litert_lm_benchmark_info_get_decode_tokens_per_sec_at(benchInfo, lastIdx)
-                            completionTokens = Double(litert_lm_benchmark_info_get_decode_token_count_at(benchInfo, lastIdx))
+
+                    // This callback fires on an engine-internal thread (the C API
+                    // returns once the stream *starts*), so commit the shared
+                    // lastStats/history — and the conversation benchmark read — on
+                    // the serial engine queue to avoid racing getStats()/getHistory().
+                    // Resolving inside the same block guarantees JS observes the
+                    // final turn before the promise settles.
+                    ctx.parent.queue.async {
+                        var completionTokens = Double(ctx.tokenCount)
+                        var tokensPerSecond = 0.0
+                        var ttft = 0.0
+
+                        if let benchInfo = litert_lm_conversation_get_benchmark_info(ctx.parent.conversation) {
+                            let numDecodeTurns = litert_lm_benchmark_info_get_num_decode_turns(benchInfo)
+                            if numDecodeTurns > 0 {
+                                let lastIdx = numDecodeTurns - 1
+                                tokensPerSecond = litert_lm_benchmark_info_get_decode_tokens_per_sec_at(benchInfo, lastIdx)
+                                completionTokens = Double(litert_lm_benchmark_info_get_decode_token_count_at(benchInfo, lastIdx))
+                            }
+                            ttft = litert_lm_benchmark_info_get_time_to_first_token(benchInfo)
+                            litert_lm_benchmark_info_delete(benchInfo)
                         }
-                        ttft = litert_lm_benchmark_info_get_time_to_first_token(benchInfo)
-                        litert_lm_benchmark_info_delete(benchInfo)
-                    }
-                    
-                    let promptTokens = Double(ctx.userMessage.count) / 4.0
-                    if completionTokens == 0.0 {
-                        completionTokens = Double(ctx.fullResponse.count) / 4.0
+
+                        let promptTokens = Double(ctx.userMessage.count) / 4.0
+                        if completionTokens == 0.0 {
+                            completionTokens = Double(ctx.fullResponse.count) / 4.0
+                        }
+
+                        ctx.parent.lastStats = GenerationStats(
+                            promptTokens: promptTokens,
+                            completionTokens: completionTokens,
+                            totalTokens: promptTokens + completionTokens,
+                            timeToFirstToken: ttft,
+                            totalTime: totalTime,
+                            tokensPerSecond: tokensPerSecond > 0.0 ? tokensPerSecond : (completionTokens / totalTime)
+                        )
+
+                        ctx.parent.history.append(Message(role: .user, content: ctx.userMessage))
+                        ctx.parent.history.append(Message(role: .model, content: ctx.fullResponse))
+
+                        ctx.onToken("", true)
+                        ctx.promise.resolve()
+                        Unmanaged<StreamContext>.fromOpaque(callbackData).release()
                     }
-                    
-                    ctx.parent.lastStats = GenerationStats(
-                        promptTokens: promptTokens,
-                        completionTokens: completionTokens,
-                        totalTokens: promptTokens + completionTokens,
-                        timeToFirstToken: ttft,
-                        totalTime: totalTime,
-                        tokensPerSecond: tokensPerSecond > 0.0 ? tokensPerSecond : (completionTokens / totalTime)
-                    )
-                    
-                    ctx.parent.history.append(Message(role: .user, content: ctx.userMessage))
-                    ctx.parent.history.append(Message(role: .model, content: ctx.fullResponse))
-                    
-                    ctx.onToken("", true)
-                    ctx.promise.resolve()
-                    Unmanaged<StreamContext>.fromOpaque(callbackData).release()
                     return
                 }
                 
@@ -542,7 +550,285 @@ public class HybridLiteRTLM: HybridLiteRTLMSpec_base, HybridLiteRTLMSpec_protoco
         
         return promise
     }
+
+    public func sendMessageWithImageAsync(message: String, imagePath: String, onToken: @escaping (_ token: String, _ done: Bool) -> Void) throws -> Promise<Void> {
+        let promise = Promise<Void>()
+        
+        queue.async {
+            guard let conversation = self.conversation else {
+                promise.reject(withError: NSError(domain: "LiteRTLM", code: 400, userInfo: [NSLocalizedDescriptionKey: "LiteRTLM: No model loaded. Call loadModel() first."]))
+                return
+            }
+            
+            if !FileManager.default.fileExists(atPath: imagePath) {
+                promise.reject(withError: NSError(domain: "LiteRTLM", code: 404, userInfo: [NSLocalizedDescriptionKey: "Image file not found: \(imagePath)"]))
+                return
+            }
+            
+            let msgJson = self.buildImageMessageJson(text: message, imagePath: imagePath)
+            let startTime = Date()
+            
+            let historyUserContent = message + " [image: \(imagePath)]"
+            let context = StreamContext(
+                userMessage: message,
+                startTime: startTime,
+                onToken: onToken,
+                promise: promise,
+                parent: self
+            )
+            
+            let callbackData = Unmanaged.passRetained(context).toOpaque()
+            
+            let callback: LiteRtLmStreamCallback = { callbackData, chunk, isFinal, errorMsg in
+                guard let callbackData = callbackData else { return }
+                let ctx = Unmanaged<StreamContext>.fromOpaque(callbackData).takeUnretainedValue()
+                
+                if let errorMsg = errorMsg {
+                    let errorStr = String(cString: errorMsg)
+                    ctx.onToken("Error: \(errorStr)", true)
+                    ctx.promise.reject(withError: NSError(domain: "LiteRTLM", code: 500, userInfo: [NSLocalizedDescriptionKey: errorStr]))
+                    Unmanaged<StreamContext>.fromOpaque(callbackData).release()
+                    return
+                }
+                
+                if isFinal {
+                    let endTime = Date()
+                    let totalTime = endTime.timeIntervalSince(ctx.startTime)
+                    
+                    let cleaned = ctx.parent.stripControlTokens(ctx.rawResponse)
+                    var finalCleaned = cleaned.trimmingCharacters(in: .whitespacesAndNewlines)
+                    if !ctx.userMessage.isEmpty && finalCleaned.hasPrefix(ctx.userMessage) {
+                        finalCleaned = String(finalCleaned.dropFirst(ctx.userMessage.count))
+                            .trimmingCharacters(in: .whitespacesAndNewlines)
+                    }
+                    
+                    if finalCleaned.count > ctx.lastEmittedLength {
+                        let startIdx = finalCleaned.index(finalCleaned.startIndex, offsetBy: ctx.lastEmittedLength)
+                        let remaining = String(finalCleaned[startIdx...])
+                        ctx.onToken(remaining, false)
+                    }
+                    ctx.fullResponse = finalCleaned
+                    
+                    var completionTokens = Double(ctx.tokenCount)
+                    var tokensPerSecond = 0.0
+                    var ttft = 0.0
+                    if let benchInfo = litert_lm_conversation_get_benchmark_info(ctx.parent.conversation) {
+                        let numDecodeTurns = litert_lm_benchmark_info_get_num_decode_turns(benchInfo)
+                        if numDecodeTurns > 0 {
+                            let lastIdx = numDecodeTurns - 1
+                            tokensPerSecond = litert_lm_benchmark_info_get_decode_tokens_per_sec_at(benchInfo, lastIdx)
+                            completionTokens = Double(litert_lm_benchmark_info_get_decode_token_count_at(benchInfo, lastIdx))
+                        }
+                        ttft = litert_lm_benchmark_info_get_time_to_first_token(benchInfo)
+                        litert_lm_benchmark_info_delete(benchInfo)
+                    }
+
+                    let promptTokens = Double(ctx.userMessage.count) / 4.0
+                    if completionTokens == 0.0 {
+                        completionTokens = Double(ctx.fullResponse.count) / 4.0
+                    }
+                    ctx.parent.lastStats = GenerationStats(
+                        promptTokens: promptTokens,
+                        completionTokens: completionTokens,
+                        totalTokens: promptTokens + completionTokens,
+                        timeToFirstToken: ttft,
+                        totalTime: totalTime,
+                        tokensPerSecond: tokensPerSecond > 0.0 ? tokensPerSecond : (completionTokens / totalTime)
+                    )
+                    ctx.parent.history.append(Message(role: .user, content: historyUserContent))
+                    ctx.parent.history.append(Message(role: .model, content: ctx.fullResponse))
+                    ctx.onToken("", true)
+                    ctx.promise.resolve()
+                    Unmanaged<StreamContext>.fromOpaque(callbackData).release()
+                    return
+                }
+
+                if let chunk = chunk {
+                    let token = String(cString: chunk)
+                    let raw: String
+                    if token.hasPrefix("{") && token.contains("\"role\"") {
+                        raw = ctx.parent.extractTextFromResponse(token)
+                    } else {
+                        raw = token
+                    }
+
+                    ctx.rawResponse += raw
+                    let cleaned = ctx.parent.stripControlTokens(ctx.rawResponse)
+                        .trimmingLeadingCharacters(in: .whitespacesAndNewlines)
+
+                    var processed = cleaned
+                    if !ctx.userMessage.isEmpty && processed.hasPrefix(ctx.userMessage) {
+                        processed = String(processed.dropFirst(ctx.userMessage.count))
+                            .trimmingLeadingCharacters(in: .whitespacesAndNewlines)
+                    }
+
+                    let safeLen = ctx.parent.safeEmitLength(processed)
+                    if safeLen > ctx.lastEmittedLength {
+                        let chars = Array(processed)
+                        let newText = String(chars[ctx.lastEmittedLength..<safeLen])
+                        ctx.lastEmittedLength = safeLen
+                        ctx.tokenCount += 1
+                        ctx.onToken(newText, false)
+                    }
+                }
+            }
+
+            let status = litert_lm_conversation_send_message_stream(
+                conversation,
+                msgJson,
+                nil,
+                nil,
+                callback,
+                callbackData
+            )
+            if status != 0 {
+                Unmanaged<StreamContext>.fromOpaque(callbackData).release()
+                promise.reject(withError: NSError(domain: "LiteRTLM", code: Int(status), userInfo: [NSLocalizedDescriptionKey: "Failed to start streaming conversation."]))
+            }
+        }
+
+        return promise
+    }
     
+    public func sendMessageWithAudioAsync(message: String, audioPath: String, onToken: @escaping (_ token: String, _ done: Bool) -> Void) throws -> Promise<Void> {
+        let promise = Promise<Void>()
+
+        queue.async {
+            guard let conversation = self.conversation else {
+                promise.reject(withError: NSError(domain: "LiteRTLM", code: 400, userInfo: [NSLocalizedDescriptionKey: "LiteRTLM: No model loaded. Call loadModel() first."]))
+                return
+            }
+
+            if !FileManager.default.fileExists(atPath: audioPath) {
+                promise.reject(withError: NSError(domain: "LiteRTLM", code: 404, userInfo: [NSLocalizedDescriptionKey: "Audio file not found: \(audioPath)"]))
+                return
+            }
+
+            let msgJson = self.buildAudioMessageJson(text: message, audioPath: audioPath)
+            let startTime = Date()
+
+            let historyUserContent = message + " [audio: \(audioPath)]"
+            let context = StreamContext(
+                userMessage: message,
+                startTime: startTime,
+                onToken: onToken,
+                promise: promise,
+                parent: self
+            )
+
+            let callbackData = Unmanaged.passRetained(context).toOpaque()
+
+            let callback: LiteRtLmStreamCallback = { callbackData, chunk, isFinal, errorMsg in
+                guard let callbackData = callbackData else { return }
+                let ctx = Unmanaged<StreamContext>.fromOpaque(callbackData).takeUnretainedValue()
+
+                if let errorMsg = errorMsg {
+                    let errorStr = String(cString: errorMsg)
+                    ctx.onToken("Error: \(errorStr)", true)
+                    ctx.promise.reject(withError: NSError(domain: "LiteRTLM", code: 500, userInfo: [NSLocalizedDescriptionKey: errorStr]))
+                    Unmanaged<StreamContext>.fromOpaque(callbackData).release()
+                    return
+                }
+
+                if isFinal {
+                    let endTime = Date()
+                    let totalTime = endTime.timeIntervalSince(ctx.startTime)
+
+                    let cleaned = ctx.parent.stripControlTokens(ctx.rawResponse)
+                    var finalCleaned = cleaned.trimmingCharacters(in: .whitespacesAndNewlines)
+                    if !ctx.userMessage.isEmpty && finalCleaned.hasPrefix(ctx.userMessage) {
+                        finalCleaned = String(finalCleaned.dropFirst(ctx.userMessage.count))
+                            .trimmingCharacters(in: .whitespacesAndNewlines)
+                    }
+
+                    if finalCleaned.count > ctx.lastEmittedLength {
+                        let startIdx = finalCleaned.index(finalCleaned.startIndex, offsetBy: ctx.lastEmittedLength)
+                        let remaining = String(finalCleaned[startIdx...])
+                        ctx.onToken(remaining, false)
+                    }
+                    ctx.fullResponse = finalCleaned
+
+                    var completionTokens = Double(ctx.tokenCount)
+                    var tokensPerSecond = 0.0
+                    var ttft = 0.0
+                    if let benchInfo = litert_lm_conversation_get_benchmark_info(ctx.parent.conversation) {
+                        let numDecodeTurns = litert_lm_benchmark_info_get_num_decode_turns(benchInfo)
+                        if numDecodeTurns > 0 {
+                            let lastIdx = numDecodeTurns - 1
+                            tokensPerSecond = litert_lm_benchmark_info_get_decode_tokens_per_sec_at(benchInfo, lastIdx)
+                            completionTokens = Double(litert_lm_benchmark_info_get_decode_token_count_at(benchInfo, lastIdx))
+                        }
+                        ttft = litert_lm_benchmark_info_get_time_to_first_token(benchInfo)
+                        litert_lm_benchmark_info_delete(benchInfo)
+                    }
+
+                    let promptTokens = Double(ctx.userMessage.count) / 4.0
+                    if completionTokens == 0.0 {
+                        completionTokens = Double(ctx.fullResponse.count) / 4.0
+                    }
+                    ctx.parent.lastStats = GenerationStats(
+                        promptTokens: promptTokens,
+                        completionTokens: completionTokens,
+                        totalTokens: promptTokens + completionTokens,
+                        timeToFirstToken: ttft,
+                        totalTime: totalTime,
+                        tokensPerSecond: tokensPerSecond > 0.0 ? tokensPerSecond : (completionTokens / totalTime)
+                    )
+                    ctx.parent.history.append(Message(role: .user, content: historyUserContent))
+                    ctx.parent.history.append(Message(role: .model, content: ctx.fullResponse))
+                    ctx.onToken("", true)
+                    ctx.promise.resolve()
+                    Unmanaged<StreamContext>.fromOpaque(callbackData).release()
+                    return
+                }
+
+                if let chunk = chunk {
+                    let token = String(cString: chunk)
+                    let raw: String
+                    if token.hasPrefix("{") && token.contains("\"role\"") {
+                        raw = ctx.parent.extractTextFromResponse(token)
+                    } else {
+                        raw = token
+                    }
+
+                    ctx.rawResponse += raw
+                    let cleaned = ctx.parent.stripControlTokens(ctx.rawResponse)
+                        .trimmingLeadingCharacters(in: .whitespacesAndNewlines)
+
+                    var processed = cleaned
+                    if !ctx.userMessage.isEmpty && processed.hasPrefix(ctx.userMessage) {
+                        processed = String(processed.dropFirst(ctx.userMessage.count))
+                            .trimmingLeadingCharacters(in: .whitespacesAndNewlines)
+                    }
+
+                    let safeLen = ctx.parent.safeEmitLength(processed)
+                    if safeLen > ctx.lastEmittedLength {
+                        let chars = Array(processed)
+                        let newText = String(chars[ctx.lastEmittedLength..<safeLen])
+                        ctx.lastEmittedLength = safeLen
+                        ctx.tokenCount += 1
+                        ctx.onToken(newText, false)
+                    }
+                }
+            }
+
+            let status = litert_lm_conversation_send_message_stream(
+                conversation,
+                msgJson,
+                nil,
+                nil,
+                callback,
+                callbackData
+            )
+            if status != 0 {
+                Unmanaged<StreamContext>.fromOpaque(callbackData).release()
+                promise.reject(withError: NSError(domain: "LiteRTLM", code: Int(status), userInfo: [NSLocalizedDescriptionKey: "Failed to start streaming conversation."]))
+            }
+        }
+
+        return promise
+    }
+
     public func sendMessageWithAudio(message: String, audioPath: String) throws -> Promise<String> {
         let promise = Promise<String>()
         

package/ios/Tests/HybridLiteRTLMTests.swift

@@ -53,6 +53,52 @@ class HybridLiteRTLMTests: XCTestCase {
         }
     }
 
+    func testSendMessageWithImageAsyncRejectsWithoutModel() async throws {
+        do {
+            let promise = try bridge.sendMessageWithImageAsync(message: "hello", imagePath: "/tmp/image.jpg") { _, _ in }
+            _ = try await promise.await()
+            XCTFail("Should have failed without model")
+        } catch {
+            let nsError = error as NSError
+            XCTAssertEqual(nsError.domain, "LiteRTLM")
+            XCTAssertEqual(nsError.code, 400)
+        }
+    }
+
+    func testSendMessageWithAudioAsyncRejectsWithoutModel() async throws {
+        do {
+            let promise = try bridge.sendMessageWithAudioAsync(message: "hello", audioPath: "/tmp/audio.wav") { _, _ in }
+            _ = try await promise.await()
+            XCTFail("Should have failed without model")
+        } catch {
+            let nsError = error as NSError
+            XCTAssertEqual(nsError.domain, "LiteRTLM")
+            XCTAssertEqual(nsError.code, 400)
+        }
+    }
+
+    func testSendMessageWithImageAsyncRejectsFileNotFound() async throws {
+        do {
+            let promise = try bridge.sendMessageWithImageAsync(message: "hello", imagePath: "/nonexistent/image.jpg") { _, _ in }
+            _ = try await promise.await()
+            XCTFail("Should have failed without model")
+        } catch {
+            let nsError = error as NSError
+            XCTAssertEqual(nsError.domain, "LiteRTLM")
+        }
+    }
+
+    func testSendMessageWithAudioAsyncRejectsFileNotFound() async throws {
+        do {
+            let promise = try bridge.sendMessageWithAudioAsync(message: "hello", audioPath: "/nonexistent/audio.wav") { _, _ in }
+            _ = try await promise.await()
+            XCTFail("Should have failed without model")
+        } catch {
+            let nsError = error as NSError
+            XCTAssertEqual(nsError.domain, "LiteRTLM")
+        }
+    }
+
     func testInitialStats() {
         XCTAssertNoThrow(try bridge.getStats())
         if let stats = try? bridge.getStats() {

package/lib/__mocks__/react-native-nitro-modules.d.ts

@@ -8,6 +8,8 @@ export declare const mockLiteRTLM: {
     sendMessageWithAudio: jest.Mock<any, any, any>;
     sendMultimodalMessage: jest.Mock<any, any, any>;
     sendMessageAsync: jest.Mock<Promise<void>, [msg: any, onToken: any], any>;
+    sendMessageWithImageAsync: jest.Mock<Promise<void>, [msg: any, imagePath: any, onToken: any], any>;
+    sendMessageWithAudioAsync: jest.Mock<Promise<void>, [msg: any, audioPath: any, onToken: any], any>;
     getHistory: jest.Mock<never[], [], any>;
     resetConversation: jest.Mock<any, any, any>;
     getStats: jest.Mock<{
@@ -38,6 +40,8 @@ export declare const NitroModules: {
         sendMessageWithAudio: jest.Mock<any, any, any>;
         sendMultimodalMessage: jest.Mock<any, any, any>;
         sendMessageAsync: jest.Mock<Promise<void>, [msg: any, onToken: any], any>;
+        sendMessageWithImageAsync: jest.Mock<Promise<void>, [msg: any, imagePath: any, onToken: any], any>;
+        sendMessageWithAudioAsync: jest.Mock<Promise<void>, [msg: any, audioPath: any, onToken: any], any>;
         getHistory: jest.Mock<never[], [], any>;
         resetConversation: jest.Mock<any, any, any>;
         getStats: jest.Mock<{

package/lib/__mocks__/react-native-nitro-modules.js

@@ -18,6 +18,16 @@ exports.mockLiteRTLM = {
         onToken("token", true);
         return Promise.resolve();
     }),
+    sendMessageWithImageAsync: jest.fn((msg, imagePath, onToken) => {
+        onToken("Mock vision ", false);
+        onToken("token", true);
+        return Promise.resolve();
+    }),
+    sendMessageWithAudioAsync: jest.fn((msg, audioPath, onToken) => {
+        onToken("Mock audio ", false);
+        onToken("token", true);
+        return Promise.resolve();
+    }),
     getHistory: jest.fn(() => []),
     resetConversation: jest.fn(),
     getStats: jest.fn(() => ({

package/lib/__tests__/modelFactory.test.js

@@ -41,6 +41,22 @@ describe('modelFactory Security & Proxy Unit Tests', () => {
         expect(react_native_nitro_modules_1.mockLiteRTLM.sendMessageAsync).toHaveBeenCalled();
         expect(react_native_nitro_modules_1.mockLiteRTLM.getMemoryUsage).toHaveBeenCalled();
     });
+    it('should successfully proxy sendMessageWithImageAsync and record memory metrics when done', async () => {
+        const onToken = jest.fn();
+        await llm.sendMessageWithImageAsync("Vision prompt", "/path/to/image.jpg", onToken);
+        expect(onToken).toHaveBeenCalledWith("Mock vision ", false);
+        expect(onToken).toHaveBeenCalledWith("token", true);
+        expect(react_native_nitro_modules_1.mockLiteRTLM.sendMessageWithImageAsync).toHaveBeenCalledWith("Vision prompt", "/path/to/image.jpg", expect.any(Function));
+        expect(react_native_nitro_modules_1.mockLiteRTLM.getMemoryUsage).toHaveBeenCalled();
+    });
+    it('should successfully proxy sendMessageWithAudioAsync and record memory metrics when done', async () => {
+        const onToken = jest.fn();
+        await llm.sendMessageWithAudioAsync("Audio prompt", "/path/to/audio.wav", onToken);
+        expect(onToken).toHaveBeenCalledWith("Mock audio ", false);
+        expect(onToken).toHaveBeenCalledWith("token", true);
+        expect(react_native_nitro_modules_1.mockLiteRTLM.sendMessageWithAudioAsync).toHaveBeenCalledWith("Audio prompt", "/path/to/audio.wav", expect.any(Function));
+        expect(react_native_nitro_modules_1.mockLiteRTLM.getMemoryUsage).toHaveBeenCalled();
+    });
     it('should successfully access memoryTracker and getSnapshots when memory tracking is enabled', () => {
         expect(llm.memoryTracker).toBeDefined();
         expect(llm.memoryTracker?.getCapacity()).toBe(256);