react-native-litert-lm 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/ios/HybridLiteRTLM.swift
CHANGED
|
@@ -14,6 +14,7 @@ import os
|
|
|
14
14
|
/// A stream context passed to the low-level C FFI callback to forward chunks safely to the JS thread.
|
|
15
15
|
private class StreamContext {
|
|
16
16
|
let userMessage: String
|
|
17
|
+
let historyUserContent: String
|
|
17
18
|
let startTime: Date
|
|
18
19
|
let onToken: (_ token: String, _ done: Bool) -> Void
|
|
19
20
|
let promise: Promise<Void>
|
|
@@ -26,12 +27,14 @@ private class StreamContext {
|
|
|
26
27
|
|
|
27
28
|
init(
|
|
28
29
|
userMessage: String,
|
|
30
|
+
historyUserContent: String? = nil,
|
|
29
31
|
startTime: Date,
|
|
30
32
|
onToken: @escaping (_ token: String, _ done: Bool) -> Void,
|
|
31
33
|
promise: Promise<Void>,
|
|
32
34
|
parent: HybridLiteRTLM
|
|
33
35
|
) {
|
|
34
36
|
self.userMessage = userMessage
|
|
37
|
+
self.historyUserContent = historyUserContent ?? userMessage
|
|
35
38
|
self.startTime = startTime
|
|
36
39
|
self.onToken = onToken
|
|
37
40
|
self.promise = promise
|
|
@@ -571,6 +574,7 @@ public class HybridLiteRTLM: HybridLiteRTLMSpec_base, HybridLiteRTLMSpec_protoco
|
|
|
571
574
|
let historyUserContent = message + " [image: \(imagePath)]"
|
|
572
575
|
let context = StreamContext(
|
|
573
576
|
userMessage: message,
|
|
577
|
+
historyUserContent: historyUserContent,
|
|
574
578
|
startTime: startTime,
|
|
575
579
|
onToken: onToken,
|
|
576
580
|
promise: promise,
|
|
@@ -609,37 +613,39 @@ public class HybridLiteRTLM: HybridLiteRTLMSpec_base, HybridLiteRTLMSpec_protoco
|
|
|
609
613
|
}
|
|
610
614
|
ctx.fullResponse = finalCleaned
|
|
611
615
|
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
let
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
616
|
+
ctx.parent.queue.async {
|
|
617
|
+
var completionTokens = Double(ctx.tokenCount)
|
|
618
|
+
var tokensPerSecond = 0.0
|
|
619
|
+
var ttft = 0.0
|
|
620
|
+
if let benchInfo = litert_lm_conversation_get_benchmark_info(ctx.parent.conversation) {
|
|
621
|
+
let numDecodeTurns = litert_lm_benchmark_info_get_num_decode_turns(benchInfo)
|
|
622
|
+
if numDecodeTurns > 0 {
|
|
623
|
+
let lastIdx = numDecodeTurns - 1
|
|
624
|
+
tokensPerSecond = litert_lm_benchmark_info_get_decode_tokens_per_sec_at(benchInfo, lastIdx)
|
|
625
|
+
completionTokens = Double(litert_lm_benchmark_info_get_decode_token_count_at(benchInfo, lastIdx))
|
|
626
|
+
}
|
|
627
|
+
ttft = litert_lm_benchmark_info_get_time_to_first_token(benchInfo)
|
|
628
|
+
litert_lm_benchmark_info_delete(benchInfo)
|
|
621
629
|
}
|
|
622
|
-
ttft = litert_lm_benchmark_info_get_time_to_first_token(benchInfo)
|
|
623
|
-
litert_lm_benchmark_info_delete(benchInfo)
|
|
624
|
-
}
|
|
625
630
|
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
631
|
+
let promptTokens = Double(ctx.userMessage.count) / 4.0
|
|
632
|
+
if completionTokens == 0.0 {
|
|
633
|
+
completionTokens = Double(ctx.fullResponse.count) / 4.0
|
|
634
|
+
}
|
|
635
|
+
ctx.parent.lastStats = GenerationStats(
|
|
636
|
+
promptTokens: promptTokens,
|
|
637
|
+
completionTokens: completionTokens,
|
|
638
|
+
totalTokens: promptTokens + completionTokens,
|
|
639
|
+
timeToFirstToken: ttft,
|
|
640
|
+
totalTime: totalTime,
|
|
641
|
+
tokensPerSecond: tokensPerSecond > 0.0 ? tokensPerSecond : (completionTokens / totalTime)
|
|
642
|
+
)
|
|
643
|
+
ctx.parent.history.append(Message(role: .user, content: ctx.historyUserContent))
|
|
644
|
+
ctx.parent.history.append(Message(role: .model, content: ctx.fullResponse))
|
|
645
|
+
ctx.onToken("", true)
|
|
646
|
+
ctx.promise.resolve()
|
|
647
|
+
Unmanaged<StreamContext>.fromOpaque(callbackData).release()
|
|
629
648
|
}
|
|
630
|
-
ctx.parent.lastStats = GenerationStats(
|
|
631
|
-
promptTokens: promptTokens,
|
|
632
|
-
completionTokens: completionTokens,
|
|
633
|
-
totalTokens: promptTokens + completionTokens,
|
|
634
|
-
timeToFirstToken: ttft,
|
|
635
|
-
totalTime: totalTime,
|
|
636
|
-
tokensPerSecond: tokensPerSecond > 0.0 ? tokensPerSecond : (completionTokens / totalTime)
|
|
637
|
-
)
|
|
638
|
-
ctx.parent.history.append(Message(role: .user, content: historyUserContent))
|
|
639
|
-
ctx.parent.history.append(Message(role: .model, content: ctx.fullResponse))
|
|
640
|
-
ctx.onToken("", true)
|
|
641
|
-
ctx.promise.resolve()
|
|
642
|
-
Unmanaged<StreamContext>.fromOpaque(callbackData).release()
|
|
643
649
|
return
|
|
644
650
|
}
|
|
645
651
|
|
|
@@ -710,6 +716,7 @@ public class HybridLiteRTLM: HybridLiteRTLMSpec_base, HybridLiteRTLMSpec_protoco
|
|
|
710
716
|
let historyUserContent = message + " [audio: \(audioPath)]"
|
|
711
717
|
let context = StreamContext(
|
|
712
718
|
userMessage: message,
|
|
719
|
+
historyUserContent: historyUserContent,
|
|
713
720
|
startTime: startTime,
|
|
714
721
|
onToken: onToken,
|
|
715
722
|
promise: promise,
|
|
@@ -748,37 +755,39 @@ public class HybridLiteRTLM: HybridLiteRTLMSpec_base, HybridLiteRTLMSpec_protoco
|
|
|
748
755
|
}
|
|
749
756
|
ctx.fullResponse = finalCleaned
|
|
750
757
|
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
let
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
758
|
+
ctx.parent.queue.async {
|
|
759
|
+
var completionTokens = Double(ctx.tokenCount)
|
|
760
|
+
var tokensPerSecond = 0.0
|
|
761
|
+
var ttft = 0.0
|
|
762
|
+
if let benchInfo = litert_lm_conversation_get_benchmark_info(ctx.parent.conversation) {
|
|
763
|
+
let numDecodeTurns = litert_lm_benchmark_info_get_num_decode_turns(benchInfo)
|
|
764
|
+
if numDecodeTurns > 0 {
|
|
765
|
+
let lastIdx = numDecodeTurns - 1
|
|
766
|
+
tokensPerSecond = litert_lm_benchmark_info_get_decode_tokens_per_sec_at(benchInfo, lastIdx)
|
|
767
|
+
completionTokens = Double(litert_lm_benchmark_info_get_decode_token_count_at(benchInfo, lastIdx))
|
|
768
|
+
}
|
|
769
|
+
ttft = litert_lm_benchmark_info_get_time_to_first_token(benchInfo)
|
|
770
|
+
litert_lm_benchmark_info_delete(benchInfo)
|
|
760
771
|
}
|
|
761
|
-
ttft = litert_lm_benchmark_info_get_time_to_first_token(benchInfo)
|
|
762
|
-
litert_lm_benchmark_info_delete(benchInfo)
|
|
763
|
-
}
|
|
764
772
|
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
773
|
+
let promptTokens = Double(ctx.userMessage.count) / 4.0
|
|
774
|
+
if completionTokens == 0.0 {
|
|
775
|
+
completionTokens = Double(ctx.fullResponse.count) / 4.0
|
|
776
|
+
}
|
|
777
|
+
ctx.parent.lastStats = GenerationStats(
|
|
778
|
+
promptTokens: promptTokens,
|
|
779
|
+
completionTokens: completionTokens,
|
|
780
|
+
totalTokens: promptTokens + completionTokens,
|
|
781
|
+
timeToFirstToken: ttft,
|
|
782
|
+
totalTime: totalTime,
|
|
783
|
+
tokensPerSecond: tokensPerSecond > 0.0 ? tokensPerSecond : (completionTokens / totalTime)
|
|
784
|
+
)
|
|
785
|
+
ctx.parent.history.append(Message(role: .user, content: ctx.historyUserContent))
|
|
786
|
+
ctx.parent.history.append(Message(role: .model, content: ctx.fullResponse))
|
|
787
|
+
ctx.onToken("", true)
|
|
788
|
+
ctx.promise.resolve()
|
|
789
|
+
Unmanaged<StreamContext>.fromOpaque(callbackData).release()
|
|
768
790
|
}
|
|
769
|
-
ctx.parent.lastStats = GenerationStats(
|
|
770
|
-
promptTokens: promptTokens,
|
|
771
|
-
completionTokens: completionTokens,
|
|
772
|
-
totalTokens: promptTokens + completionTokens,
|
|
773
|
-
timeToFirstToken: ttft,
|
|
774
|
-
totalTime: totalTime,
|
|
775
|
-
tokensPerSecond: tokensPerSecond > 0.0 ? tokensPerSecond : (completionTokens / totalTime)
|
|
776
|
-
)
|
|
777
|
-
ctx.parent.history.append(Message(role: .user, content: historyUserContent))
|
|
778
|
-
ctx.parent.history.append(Message(role: .model, content: ctx.fullResponse))
|
|
779
|
-
ctx.onToken("", true)
|
|
780
|
-
ctx.promise.resolve()
|
|
781
|
-
Unmanaged<StreamContext>.fromOpaque(callbackData).release()
|
|
782
791
|
return
|
|
783
792
|
}
|
|
784
793
|
|
|
@@ -53,6 +53,18 @@ class HybridLiteRTLMTests: XCTestCase {
|
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
func testSendMessageAsyncRejectsWithoutModel() async throws {
|
|
57
|
+
do {
|
|
58
|
+
let promise = try bridge.sendMessageAsync(message: "hello") { _, _ in }
|
|
59
|
+
_ = try await promise.await()
|
|
60
|
+
XCTFail("Should have failed without model")
|
|
61
|
+
} catch {
|
|
62
|
+
let nsError = error as NSError
|
|
63
|
+
XCTAssertEqual(nsError.domain, "LiteRTLM")
|
|
64
|
+
XCTAssertEqual(nsError.code, 400)
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
56
68
|
func testSendMessageWithImageAsyncRejectsWithoutModel() async throws {
|
|
57
69
|
do {
|
|
58
70
|
let promise = try bridge.sendMessageWithImageAsync(message: "hello", imagePath: "/tmp/image.jpg") { _, _ in }
|