react-native-litert-lm 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +245 -29
- package/android/src/main/java/com/margelo/nitro/dev/litert/litertlm/HybridLiteRTLM.kt +301 -58
- package/cpp/HybridLiteRTLM.cpp +109 -9
- package/cpp/HybridLiteRTLM.hpp +16 -0
- package/cpp/cpp-adapter.cpp +10 -2
- package/lib/hooks.d.ts +41 -0
- package/lib/hooks.js +131 -0
- package/lib/index.d.ts +30 -3
- package/lib/index.js +53 -6
- package/lib/memoryTracker.d.ts +128 -0
- package/lib/memoryTracker.js +155 -0
- package/lib/modelFactory.d.ts +18 -0
- package/lib/modelFactory.js +104 -0
- package/lib/specs/LiteRTLM.nitro.d.ts +38 -0
- package/lib/templates.d.ts +51 -0
- package/lib/templates.js +81 -0
- package/nitrogen/generated/android/LiteRTLMOnLoad.cpp +22 -17
- package/nitrogen/generated/android/LiteRTLMOnLoad.hpp +13 -4
- package/nitrogen/generated/android/c++/JFunc_void_double.hpp +75 -0
- package/nitrogen/generated/android/c++/JHybridLiteRTLMSpec.cpp +42 -1
- package/nitrogen/generated/android/c++/JHybridLiteRTLMSpec.hpp +3 -0
- package/nitrogen/generated/android/c++/JLLMConfig.hpp +6 -1
- package/nitrogen/generated/android/c++/JMemoryUsage.hpp +69 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/dev/litert/litertlm/Func_void_double.kt +80 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/dev/litert/litertlm/HybridLiteRTLMSpec.kt +17 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/dev/litert/litertlm/LLMConfig.kt +5 -2
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/dev/litert/litertlm/MemoryUsage.kt +47 -0
- package/nitrogen/generated/shared/c++/HybridLiteRTLMSpec.cpp +3 -0
- package/nitrogen/generated/shared/c++/HybridLiteRTLMSpec.hpp +6 -0
- package/nitrogen/generated/shared/c++/LLMConfig.hpp +7 -2
- package/nitrogen/generated/shared/c++/MemoryUsage.hpp +95 -0
- package/package.json +3 -3
- package/src/hooks.ts +195 -0
- package/src/index.ts +51 -3
- package/src/memoryTracker.ts +268 -0
- package/src/modelFactory.ts +120 -0
- package/src/specs/LiteRTLM.nitro.ts +47 -0
- package/src/templates.ts +105 -0
package/nitrogen/generated/android/kotlin/com/margelo/nitro/dev/litert/litertlm/LLMConfig.kt
CHANGED
|
@@ -17,6 +17,9 @@ import com.facebook.proguard.annotations.DoNotStrip
|
|
|
17
17
|
@DoNotStrip
|
|
18
18
|
@Keep
|
|
19
19
|
data class LLMConfig(
|
|
20
|
+
@DoNotStrip
|
|
21
|
+
@Keep
|
|
22
|
+
val systemPrompt: String?,
|
|
20
23
|
@DoNotStrip
|
|
21
24
|
@Keep
|
|
22
25
|
val backend: Backend?,
|
|
@@ -43,8 +46,8 @@ data class LLMConfig(
|
|
|
43
46
|
@Keep
|
|
44
47
|
@Suppress("unused")
|
|
45
48
|
@JvmStatic
|
|
46
|
-
private fun fromCpp(backend: Backend?, maxTokens: Double?, temperature: Double?, topK: Double?, topP: Double?): LLMConfig {
|
|
47
|
-
return LLMConfig(backend, maxTokens, temperature, topK, topP)
|
|
49
|
+
private fun fromCpp(systemPrompt: String?, backend: Backend?, maxTokens: Double?, temperature: Double?, topK: Double?, topP: Double?): LLMConfig {
|
|
50
|
+
return LLMConfig(systemPrompt, backend, maxTokens, temperature, topK, topP)
|
|
48
51
|
}
|
|
49
52
|
}
|
|
50
53
|
}
|
package/nitrogen/generated/android/kotlin/com/margelo/nitro/dev/litert/litertlm/MemoryUsage.kt
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
///
|
|
2
|
+
/// MemoryUsage.kt
|
|
3
|
+
/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
|
|
4
|
+
/// https://github.com/mrousavy/nitro
|
|
5
|
+
/// Copyright © Marc Rousavy @ Margelo
|
|
6
|
+
///
|
|
7
|
+
|
|
8
|
+
package com.margelo.nitro.dev.litert.litertlm
|
|
9
|
+
|
|
10
|
+
import androidx.annotation.Keep
|
|
11
|
+
import com.facebook.proguard.annotations.DoNotStrip
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Represents the JavaScript object/struct "MemoryUsage".
|
|
16
|
+
*/
|
|
17
|
+
@DoNotStrip
|
|
18
|
+
@Keep
|
|
19
|
+
data class MemoryUsage(
|
|
20
|
+
@DoNotStrip
|
|
21
|
+
@Keep
|
|
22
|
+
val nativeHeapBytes: Double,
|
|
23
|
+
@DoNotStrip
|
|
24
|
+
@Keep
|
|
25
|
+
val residentBytes: Double,
|
|
26
|
+
@DoNotStrip
|
|
27
|
+
@Keep
|
|
28
|
+
val availableMemoryBytes: Double,
|
|
29
|
+
@DoNotStrip
|
|
30
|
+
@Keep
|
|
31
|
+
val isLowMemory: Boolean
|
|
32
|
+
) {
|
|
33
|
+
/* primary constructor */
|
|
34
|
+
|
|
35
|
+
companion object {
|
|
36
|
+
/**
|
|
37
|
+
* Constructor called from C++
|
|
38
|
+
*/
|
|
39
|
+
@DoNotStrip
|
|
40
|
+
@Keep
|
|
41
|
+
@Suppress("unused")
|
|
42
|
+
@JvmStatic
|
|
43
|
+
private fun fromCpp(nativeHeapBytes: Double, residentBytes: Double, availableMemoryBytes: Double, isLowMemory: Boolean): MemoryUsage {
|
|
44
|
+
return MemoryUsage(nativeHeapBytes, residentBytes, availableMemoryBytes, isLowMemory)
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -17,12 +17,15 @@ namespace margelo::nitro::litertlm {
|
|
|
17
17
|
prototype.registerHybridMethod("loadModel", &HybridLiteRTLMSpec::loadModel);
|
|
18
18
|
prototype.registerHybridMethod("sendMessage", &HybridLiteRTLMSpec::sendMessage);
|
|
19
19
|
prototype.registerHybridMethod("sendMessageWithImage", &HybridLiteRTLMSpec::sendMessageWithImage);
|
|
20
|
+
prototype.registerHybridMethod("downloadModel", &HybridLiteRTLMSpec::downloadModel);
|
|
21
|
+
prototype.registerHybridMethod("deleteModel", &HybridLiteRTLMSpec::deleteModel);
|
|
20
22
|
prototype.registerHybridMethod("sendMessageWithAudio", &HybridLiteRTLMSpec::sendMessageWithAudio);
|
|
21
23
|
prototype.registerHybridMethod("sendMessageAsync", &HybridLiteRTLMSpec::sendMessageAsync);
|
|
22
24
|
prototype.registerHybridMethod("getHistory", &HybridLiteRTLMSpec::getHistory);
|
|
23
25
|
prototype.registerHybridMethod("resetConversation", &HybridLiteRTLMSpec::resetConversation);
|
|
24
26
|
prototype.registerHybridMethod("isReady", &HybridLiteRTLMSpec::isReady);
|
|
25
27
|
prototype.registerHybridMethod("getStats", &HybridLiteRTLMSpec::getStats);
|
|
28
|
+
prototype.registerHybridMethod("getMemoryUsage", &HybridLiteRTLMSpec::getMemoryUsage);
|
|
26
29
|
prototype.registerHybridMethod("close", &HybridLiteRTLMSpec::close);
|
|
27
30
|
});
|
|
28
31
|
}
|
|
@@ -19,6 +19,8 @@ namespace margelo::nitro::litertlm { struct LLMConfig; }
|
|
|
19
19
|
namespace margelo::nitro::litertlm { struct Message; }
|
|
20
20
|
// Forward declaration of `GenerationStats` to properly resolve imports.
|
|
21
21
|
namespace margelo::nitro::litertlm { struct GenerationStats; }
|
|
22
|
+
// Forward declaration of `MemoryUsage` to properly resolve imports.
|
|
23
|
+
namespace margelo::nitro::litertlm { struct MemoryUsage; }
|
|
22
24
|
|
|
23
25
|
#include <NitroModules/Promise.hpp>
|
|
24
26
|
#include <string>
|
|
@@ -28,6 +30,7 @@ namespace margelo::nitro::litertlm { struct GenerationStats; }
|
|
|
28
30
|
#include "Message.hpp"
|
|
29
31
|
#include <vector>
|
|
30
32
|
#include "GenerationStats.hpp"
|
|
33
|
+
#include "MemoryUsage.hpp"
|
|
31
34
|
|
|
32
35
|
namespace margelo::nitro::litertlm {
|
|
33
36
|
|
|
@@ -63,12 +66,15 @@ namespace margelo::nitro::litertlm {
|
|
|
63
66
|
virtual std::shared_ptr<Promise<void>> loadModel(const std::string& modelPath, const std::optional<LLMConfig>& config) = 0;
|
|
64
67
|
virtual std::shared_ptr<Promise<std::string>> sendMessage(const std::string& message) = 0;
|
|
65
68
|
virtual std::shared_ptr<Promise<std::string>> sendMessageWithImage(const std::string& message, const std::string& imagePath) = 0;
|
|
69
|
+
virtual std::shared_ptr<Promise<std::string>> downloadModel(const std::string& url, const std::string& fileName, const std::optional<std::function<void(double /* progress */)>>& onProgress) = 0;
|
|
70
|
+
virtual std::shared_ptr<Promise<void>> deleteModel(const std::string& fileName) = 0;
|
|
66
71
|
virtual std::shared_ptr<Promise<std::string>> sendMessageWithAudio(const std::string& message, const std::string& audioPath) = 0;
|
|
67
72
|
virtual void sendMessageAsync(const std::string& message, const std::function<void(const std::string& /* token */, bool /* done */)>& onToken) = 0;
|
|
68
73
|
virtual std::vector<Message> getHistory() = 0;
|
|
69
74
|
virtual void resetConversation() = 0;
|
|
70
75
|
virtual bool isReady() = 0;
|
|
71
76
|
virtual GenerationStats getStats() = 0;
|
|
77
|
+
virtual MemoryUsage getMemoryUsage() = 0;
|
|
72
78
|
virtual void close() = 0;
|
|
73
79
|
|
|
74
80
|
protected:
|
|
@@ -31,8 +31,9 @@
|
|
|
31
31
|
// Forward declaration of `Backend` to properly resolve imports.
|
|
32
32
|
namespace margelo::nitro::litertlm { enum class Backend; }
|
|
33
33
|
|
|
34
|
-
#include
|
|
34
|
+
#include <string>
|
|
35
35
|
#include <optional>
|
|
36
|
+
#include "Backend.hpp"
|
|
36
37
|
|
|
37
38
|
namespace margelo::nitro::litertlm {
|
|
38
39
|
|
|
@@ -41,6 +42,7 @@ namespace margelo::nitro::litertlm {
|
|
|
41
42
|
*/
|
|
42
43
|
struct LLMConfig final {
|
|
43
44
|
public:
|
|
45
|
+
std::optional<std::string> systemPrompt SWIFT_PRIVATE;
|
|
44
46
|
std::optional<Backend> backend SWIFT_PRIVATE;
|
|
45
47
|
std::optional<double> maxTokens SWIFT_PRIVATE;
|
|
46
48
|
std::optional<double> temperature SWIFT_PRIVATE;
|
|
@@ -49,7 +51,7 @@ namespace margelo::nitro::litertlm {
|
|
|
49
51
|
|
|
50
52
|
public:
|
|
51
53
|
LLMConfig() = default;
|
|
52
|
-
explicit LLMConfig(std::optional<Backend> backend, std::optional<double> maxTokens, std::optional<double> temperature, std::optional<double> topK, std::optional<double> topP): backend(backend), maxTokens(maxTokens), temperature(temperature), topK(topK), topP(topP) {}
|
|
54
|
+
explicit LLMConfig(std::optional<std::string> systemPrompt, std::optional<Backend> backend, std::optional<double> maxTokens, std::optional<double> temperature, std::optional<double> topK, std::optional<double> topP): systemPrompt(systemPrompt), backend(backend), maxTokens(maxTokens), temperature(temperature), topK(topK), topP(topP) {}
|
|
53
55
|
|
|
54
56
|
public:
|
|
55
57
|
friend bool operator==(const LLMConfig& lhs, const LLMConfig& rhs) = default;
|
|
@@ -65,6 +67,7 @@ namespace margelo::nitro {
|
|
|
65
67
|
static inline margelo::nitro::litertlm::LLMConfig fromJSI(jsi::Runtime& runtime, const jsi::Value& arg) {
|
|
66
68
|
jsi::Object obj = arg.asObject(runtime);
|
|
67
69
|
return margelo::nitro::litertlm::LLMConfig(
|
|
70
|
+
JSIConverter<std::optional<std::string>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "systemPrompt"))),
|
|
68
71
|
JSIConverter<std::optional<margelo::nitro::litertlm::Backend>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "backend"))),
|
|
69
72
|
JSIConverter<std::optional<double>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "maxTokens"))),
|
|
70
73
|
JSIConverter<std::optional<double>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "temperature"))),
|
|
@@ -74,6 +77,7 @@ namespace margelo::nitro {
|
|
|
74
77
|
}
|
|
75
78
|
static inline jsi::Value toJSI(jsi::Runtime& runtime, const margelo::nitro::litertlm::LLMConfig& arg) {
|
|
76
79
|
jsi::Object obj(runtime);
|
|
80
|
+
obj.setProperty(runtime, PropNameIDCache::get(runtime, "systemPrompt"), JSIConverter<std::optional<std::string>>::toJSI(runtime, arg.systemPrompt));
|
|
77
81
|
obj.setProperty(runtime, PropNameIDCache::get(runtime, "backend"), JSIConverter<std::optional<margelo::nitro::litertlm::Backend>>::toJSI(runtime, arg.backend));
|
|
78
82
|
obj.setProperty(runtime, PropNameIDCache::get(runtime, "maxTokens"), JSIConverter<std::optional<double>>::toJSI(runtime, arg.maxTokens));
|
|
79
83
|
obj.setProperty(runtime, PropNameIDCache::get(runtime, "temperature"), JSIConverter<std::optional<double>>::toJSI(runtime, arg.temperature));
|
|
@@ -89,6 +93,7 @@ namespace margelo::nitro {
|
|
|
89
93
|
if (!nitro::isPlainObject(runtime, obj)) {
|
|
90
94
|
return false;
|
|
91
95
|
}
|
|
96
|
+
if (!JSIConverter<std::optional<std::string>>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "systemPrompt")))) return false;
|
|
92
97
|
if (!JSIConverter<std::optional<margelo::nitro::litertlm::Backend>>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "backend")))) return false;
|
|
93
98
|
if (!JSIConverter<std::optional<double>>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "maxTokens")))) return false;
|
|
94
99
|
if (!JSIConverter<std::optional<double>>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "temperature")))) return false;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
///
|
|
2
|
+
/// MemoryUsage.hpp
|
|
3
|
+
/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
|
|
4
|
+
/// https://github.com/mrousavy/nitro
|
|
5
|
+
/// Copyright © Marc Rousavy @ Margelo
|
|
6
|
+
///
|
|
7
|
+
|
|
8
|
+
#pragma once
|
|
9
|
+
|
|
10
|
+
#if __has_include(<NitroModules/JSIConverter.hpp>)
|
|
11
|
+
#include <NitroModules/JSIConverter.hpp>
|
|
12
|
+
#else
|
|
13
|
+
#error NitroModules cannot be found! Are you sure you installed NitroModules properly?
|
|
14
|
+
#endif
|
|
15
|
+
#if __has_include(<NitroModules/NitroDefines.hpp>)
|
|
16
|
+
#include <NitroModules/NitroDefines.hpp>
|
|
17
|
+
#else
|
|
18
|
+
#error NitroModules cannot be found! Are you sure you installed NitroModules properly?
|
|
19
|
+
#endif
|
|
20
|
+
#if __has_include(<NitroModules/JSIHelpers.hpp>)
|
|
21
|
+
#include <NitroModules/JSIHelpers.hpp>
|
|
22
|
+
#else
|
|
23
|
+
#error NitroModules cannot be found! Are you sure you installed NitroModules properly?
|
|
24
|
+
#endif
|
|
25
|
+
#if __has_include(<NitroModules/PropNameIDCache.hpp>)
|
|
26
|
+
#include <NitroModules/PropNameIDCache.hpp>
|
|
27
|
+
#else
|
|
28
|
+
#error NitroModules cannot be found! Are you sure you installed NitroModules properly?
|
|
29
|
+
#endif
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
namespace margelo::nitro::litertlm {
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* A struct which can be represented as a JavaScript object (MemoryUsage).
|
|
39
|
+
*/
|
|
40
|
+
struct MemoryUsage final {
|
|
41
|
+
public:
|
|
42
|
+
double nativeHeapBytes SWIFT_PRIVATE;
|
|
43
|
+
double residentBytes SWIFT_PRIVATE;
|
|
44
|
+
double availableMemoryBytes SWIFT_PRIVATE;
|
|
45
|
+
bool isLowMemory SWIFT_PRIVATE;
|
|
46
|
+
|
|
47
|
+
public:
|
|
48
|
+
MemoryUsage() = default;
|
|
49
|
+
explicit MemoryUsage(double nativeHeapBytes, double residentBytes, double availableMemoryBytes, bool isLowMemory): nativeHeapBytes(nativeHeapBytes), residentBytes(residentBytes), availableMemoryBytes(availableMemoryBytes), isLowMemory(isLowMemory) {}
|
|
50
|
+
|
|
51
|
+
public:
|
|
52
|
+
friend bool operator==(const MemoryUsage& lhs, const MemoryUsage& rhs) = default;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
} // namespace margelo::nitro::litertlm
|
|
56
|
+
|
|
57
|
+
namespace margelo::nitro {
|
|
58
|
+
|
|
59
|
+
// C++ MemoryUsage <> JS MemoryUsage (object)
|
|
60
|
+
template <>
|
|
61
|
+
struct JSIConverter<margelo::nitro::litertlm::MemoryUsage> final {
|
|
62
|
+
static inline margelo::nitro::litertlm::MemoryUsage fromJSI(jsi::Runtime& runtime, const jsi::Value& arg) {
|
|
63
|
+
jsi::Object obj = arg.asObject(runtime);
|
|
64
|
+
return margelo::nitro::litertlm::MemoryUsage(
|
|
65
|
+
JSIConverter<double>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "nativeHeapBytes"))),
|
|
66
|
+
JSIConverter<double>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "residentBytes"))),
|
|
67
|
+
JSIConverter<double>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "availableMemoryBytes"))),
|
|
68
|
+
JSIConverter<bool>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "isLowMemory")))
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
static inline jsi::Value toJSI(jsi::Runtime& runtime, const margelo::nitro::litertlm::MemoryUsage& arg) {
|
|
72
|
+
jsi::Object obj(runtime);
|
|
73
|
+
obj.setProperty(runtime, PropNameIDCache::get(runtime, "nativeHeapBytes"), JSIConverter<double>::toJSI(runtime, arg.nativeHeapBytes));
|
|
74
|
+
obj.setProperty(runtime, PropNameIDCache::get(runtime, "residentBytes"), JSIConverter<double>::toJSI(runtime, arg.residentBytes));
|
|
75
|
+
obj.setProperty(runtime, PropNameIDCache::get(runtime, "availableMemoryBytes"), JSIConverter<double>::toJSI(runtime, arg.availableMemoryBytes));
|
|
76
|
+
obj.setProperty(runtime, PropNameIDCache::get(runtime, "isLowMemory"), JSIConverter<bool>::toJSI(runtime, arg.isLowMemory));
|
|
77
|
+
return obj;
|
|
78
|
+
}
|
|
79
|
+
static inline bool canConvert(jsi::Runtime& runtime, const jsi::Value& value) {
|
|
80
|
+
if (!value.isObject()) {
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
jsi::Object obj = value.getObject(runtime);
|
|
84
|
+
if (!nitro::isPlainObject(runtime, obj)) {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
if (!JSIConverter<double>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "nativeHeapBytes")))) return false;
|
|
88
|
+
if (!JSIConverter<double>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "residentBytes")))) return false;
|
|
89
|
+
if (!JSIConverter<double>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "availableMemoryBytes")))) return false;
|
|
90
|
+
if (!JSIConverter<bool>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "isLowMemory")))) return false;
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
} // namespace margelo::nitro
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "react-native-litert-lm",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "High-performance LLM inference for React Native using LiteRT-LM. Optimized for Gemma 3n and other on-device language models.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Hugh Chen (https://github.com/hung-yueh)",
|
|
@@ -65,7 +65,7 @@
|
|
|
65
65
|
"@expo/config-plugins": "~54.0.4",
|
|
66
66
|
"@types/react": "~19.1.10",
|
|
67
67
|
"expo": "^54.0.31",
|
|
68
|
-
"nitrogen": "^0.
|
|
68
|
+
"nitrogen": "^0.34.1",
|
|
69
69
|
"react": "19.1.0",
|
|
70
70
|
"react-native": "0.81.5",
|
|
71
71
|
"release-it": "^19.2.4",
|
|
@@ -82,6 +82,6 @@
|
|
|
82
82
|
}
|
|
83
83
|
},
|
|
84
84
|
"dependencies": {
|
|
85
|
-
"react-native-nitro-modules": "^0.
|
|
85
|
+
"react-native-nitro-modules": "^0.34.1"
|
|
86
86
|
}
|
|
87
87
|
}
|
package/src/hooks.ts
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { useState, useEffect, useRef, useCallback } from "react";
|
|
2
|
+
import { LiteRTLM, LLMConfig } from "./index";
|
|
3
|
+
import { createLLM } from "./modelFactory";
|
|
4
|
+
import type { MemoryTracker, MemoryTrackerSummary } from "./memoryTracker";
|
|
5
|
+
|
|
6
|
+
export interface UseModelConfig extends LLMConfig {
|
|
7
|
+
autoLoad?: boolean;
|
|
8
|
+
/**
|
|
9
|
+
* Enable memory tracking using native ArrayBuffers (v0.34+).
|
|
10
|
+
* When enabled, memory usage is tracked after each inference call
|
|
11
|
+
* using `NitroModules.createNativeArrayBuffer()` for zero-copy storage.
|
|
12
|
+
* @default false
|
|
13
|
+
*/
|
|
14
|
+
enableMemoryTracking?: boolean;
|
|
15
|
+
/**
|
|
16
|
+
* Maximum number of memory snapshots to store.
|
|
17
|
+
* Each snapshot uses 32 bytes of native memory.
|
|
18
|
+
* @default 256
|
|
19
|
+
*/
|
|
20
|
+
maxMemorySnapshots?: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface UseModelResult {
|
|
24
|
+
model: LiteRTLM | null;
|
|
25
|
+
isReady: boolean;
|
|
26
|
+
isGenerating: boolean;
|
|
27
|
+
downloadProgress: number;
|
|
28
|
+
error: string | null;
|
|
29
|
+
generate: (prompt: string) => Promise<string>;
|
|
30
|
+
reset: () => void;
|
|
31
|
+
deleteModel: (fileName: string) => Promise<void>;
|
|
32
|
+
load: () => Promise<void>;
|
|
33
|
+
/**
|
|
34
|
+
* Memory tracker instance (available when enableMemoryTracking is true).
|
|
35
|
+
* Uses native ArrayBuffers allocated via `NitroModules.createNativeArrayBuffer()`
|
|
36
|
+
* for efficient, zero-copy memory usage tracking.
|
|
37
|
+
*/
|
|
38
|
+
memoryTracker: MemoryTracker | null;
|
|
39
|
+
/**
|
|
40
|
+
* Current memory tracking summary (null if tracking is disabled).
|
|
41
|
+
* Updates automatically after each inference call.
|
|
42
|
+
*/
|
|
43
|
+
memorySummary: MemoryTrackerSummary | null;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function useModel(
|
|
47
|
+
pathOrUrl: string,
|
|
48
|
+
config?: UseModelConfig,
|
|
49
|
+
): UseModelResult {
|
|
50
|
+
const modelRef = useRef<(LiteRTLM & { memoryTracker?: MemoryTracker }) | null>(null);
|
|
51
|
+
const [isReady, setIsReady] = useState(false);
|
|
52
|
+
const [isGenerating, setIsGenerating] = useState(false);
|
|
53
|
+
const [downloadProgress, setDownloadProgress] = useState(0);
|
|
54
|
+
const [error, setError] = useState<string | null>(null);
|
|
55
|
+
const [memorySummary, setMemorySummary] = useState<MemoryTrackerSummary | null>(null);
|
|
56
|
+
|
|
57
|
+
// Extract autoLoad (default true) and memory tracking options
|
|
58
|
+
const autoLoad = config?.autoLoad ?? true;
|
|
59
|
+
const enableMemoryTracking = config?.enableMemoryTracking ?? false;
|
|
60
|
+
const maxMemorySnapshots = config?.maxMemorySnapshots ?? 256;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Refresh memory summary from the tracker's native buffer.
|
|
64
|
+
*/
|
|
65
|
+
const refreshMemorySummary = useCallback(() => {
|
|
66
|
+
if (modelRef.current?.memoryTracker) {
|
|
67
|
+
setMemorySummary(modelRef.current.memoryTracker.getSummary());
|
|
68
|
+
}
|
|
69
|
+
}, []);
|
|
70
|
+
|
|
71
|
+
// Initialize the model instance
|
|
72
|
+
useEffect(() => {
|
|
73
|
+
modelRef.current = createLLM({
|
|
74
|
+
enableMemoryTracking,
|
|
75
|
+
maxMemorySnapshots,
|
|
76
|
+
});
|
|
77
|
+
let isMounted = true;
|
|
78
|
+
|
|
79
|
+
// Cleanup on unmount
|
|
80
|
+
return () => {
|
|
81
|
+
isMounted = false;
|
|
82
|
+
try {
|
|
83
|
+
modelRef.current?.close();
|
|
84
|
+
} catch (e) {
|
|
85
|
+
console.warn("Failed to close model", e);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
}, [enableMemoryTracking, maxMemorySnapshots]);
|
|
89
|
+
|
|
90
|
+
const load = useCallback(async () => {
|
|
91
|
+
setIsReady(false);
|
|
92
|
+
setError(null);
|
|
93
|
+
setDownloadProgress(0);
|
|
94
|
+
|
|
95
|
+
try {
|
|
96
|
+
let modelPath = pathOrUrl;
|
|
97
|
+
|
|
98
|
+
// Handle URL download manually to capture progress
|
|
99
|
+
if (pathOrUrl.startsWith("http://") || pathOrUrl.startsWith("https://")) {
|
|
100
|
+
const fileName = pathOrUrl.split("/").pop() || "model.bin";
|
|
101
|
+
|
|
102
|
+
if (modelRef.current) {
|
|
103
|
+
modelPath = await modelRef.current.downloadModel(
|
|
104
|
+
pathOrUrl,
|
|
105
|
+
fileName,
|
|
106
|
+
(progress) => {
|
|
107
|
+
setDownloadProgress(progress);
|
|
108
|
+
},
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (modelRef.current) {
|
|
114
|
+
// Create a clean config object for native loadModel (excluding autoLoad)
|
|
115
|
+
const nativeConfig: LLMConfig = { ...config };
|
|
116
|
+
delete (nativeConfig as any).autoLoad;
|
|
117
|
+
|
|
118
|
+
await modelRef.current.loadModel(modelPath, nativeConfig);
|
|
119
|
+
setIsReady(true);
|
|
120
|
+
}
|
|
121
|
+
} catch (e: any) {
|
|
122
|
+
setError(e.message || "Failed to load model");
|
|
123
|
+
console.error(e);
|
|
124
|
+
}
|
|
125
|
+
}, [pathOrUrl, config]);
|
|
126
|
+
|
|
127
|
+
useEffect(() => {
|
|
128
|
+
if (autoLoad) {
|
|
129
|
+
load();
|
|
130
|
+
}
|
|
131
|
+
}, [autoLoad, load]);
|
|
132
|
+
|
|
133
|
+
const generate = useCallback(
|
|
134
|
+
async (prompt: string): Promise<string> => {
|
|
135
|
+
if (!modelRef.current || !isReady) {
|
|
136
|
+
throw new Error("Model not ready");
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
setIsGenerating(true);
|
|
140
|
+
try {
|
|
141
|
+
return new Promise<string>((resolve, reject) => {
|
|
142
|
+
let fullResponse = "";
|
|
143
|
+
try {
|
|
144
|
+
modelRef.current?.sendMessageAsync(
|
|
145
|
+
prompt,
|
|
146
|
+
(token: string, done: boolean) => {
|
|
147
|
+
fullResponse += token;
|
|
148
|
+
if (done) {
|
|
149
|
+
refreshMemorySummary();
|
|
150
|
+
resolve(fullResponse);
|
|
151
|
+
}
|
|
152
|
+
},
|
|
153
|
+
);
|
|
154
|
+
} catch (e: any) {
|
|
155
|
+
reject(e);
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
} catch (e: any) {
|
|
159
|
+
setError(e.message || "Generation failed");
|
|
160
|
+
throw e;
|
|
161
|
+
} finally {
|
|
162
|
+
setIsGenerating(false);
|
|
163
|
+
}
|
|
164
|
+
},
|
|
165
|
+
[isReady, refreshMemorySummary],
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
const reset = useCallback(() => {
|
|
169
|
+
if (modelRef.current) {
|
|
170
|
+
modelRef.current.resetConversation();
|
|
171
|
+
}
|
|
172
|
+
}, []);
|
|
173
|
+
|
|
174
|
+
const deleteModel = useCallback(async (fileName: string): Promise<void> => {
|
|
175
|
+
if (modelRef.current) {
|
|
176
|
+
await modelRef.current.deleteModel(fileName);
|
|
177
|
+
setIsReady(false);
|
|
178
|
+
setDownloadProgress(0);
|
|
179
|
+
}
|
|
180
|
+
}, []);
|
|
181
|
+
|
|
182
|
+
return {
|
|
183
|
+
model: modelRef.current,
|
|
184
|
+
isReady,
|
|
185
|
+
isGenerating,
|
|
186
|
+
downloadProgress,
|
|
187
|
+
error,
|
|
188
|
+
generate,
|
|
189
|
+
reset,
|
|
190
|
+
deleteModel,
|
|
191
|
+
load,
|
|
192
|
+
memoryTracker: modelRef.current?.memoryTracker ?? null,
|
|
193
|
+
memorySummary,
|
|
194
|
+
};
|
|
195
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -7,6 +7,7 @@ import type {
|
|
|
7
7
|
Backend,
|
|
8
8
|
Role,
|
|
9
9
|
GenerationStats,
|
|
10
|
+
MemoryUsage,
|
|
10
11
|
} from "./specs/LiteRTLM.nitro";
|
|
11
12
|
|
|
12
13
|
export type {
|
|
@@ -16,8 +17,27 @@ export type {
|
|
|
16
17
|
Backend,
|
|
17
18
|
Role,
|
|
18
19
|
GenerationStats,
|
|
20
|
+
MemoryUsage,
|
|
19
21
|
} from "./specs/LiteRTLM.nitro";
|
|
20
22
|
|
|
23
|
+
// Re-export template utilities
|
|
24
|
+
export type { ChatMessage } from "./templates";
|
|
25
|
+
export {
|
|
26
|
+
applyGemmaTemplate,
|
|
27
|
+
applyPhiTemplate,
|
|
28
|
+
applyLlamaTemplate,
|
|
29
|
+
} from "./templates";
|
|
30
|
+
|
|
31
|
+
// Re-export memory tracking utilities (uses NitroModules.createNativeArrayBuffer v0.34+)
|
|
32
|
+
export type {
|
|
33
|
+
MemorySnapshot,
|
|
34
|
+
MemoryTracker,
|
|
35
|
+
MemoryTrackerSummary,
|
|
36
|
+
} from "./memoryTracker";
|
|
37
|
+
export { createMemoryTracker, createNativeBuffer } from "./memoryTracker";
|
|
38
|
+
|
|
39
|
+
export * from "./hooks";
|
|
40
|
+
|
|
21
41
|
/**
|
|
22
42
|
* Creates a new LiteRT-LM inference engine instance.
|
|
23
43
|
*
|
|
@@ -51,9 +71,7 @@ export type {
|
|
|
51
71
|
* llm.close();
|
|
52
72
|
* ```
|
|
53
73
|
*/
|
|
54
|
-
export
|
|
55
|
-
return NitroModules.createHybridObject<LiteRTLM>("LiteRTLM");
|
|
56
|
-
}
|
|
74
|
+
export { createLLM } from "./modelFactory";
|
|
57
75
|
|
|
58
76
|
/**
|
|
59
77
|
* Pre-defined model identifiers for common models.
|
|
@@ -123,3 +141,33 @@ export function checkBackendSupport(backend: Backend): string | undefined {
|
|
|
123
141
|
|
|
124
142
|
return undefined;
|
|
125
143
|
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Check if multimodal features (image/audio) are supported on the current platform.
|
|
147
|
+
* Returns an error message if not supported, undefined if OK.
|
|
148
|
+
*
|
|
149
|
+
* @returns Error message if multimodal is not supported, undefined if OK
|
|
150
|
+
*
|
|
151
|
+
* @example
|
|
152
|
+
* ```typescript
|
|
153
|
+
* const error = checkMultimodalSupport();
|
|
154
|
+
* if (error) {
|
|
155
|
+
* console.warn(error);
|
|
156
|
+
* // Fall back to text-only
|
|
157
|
+
* } else {
|
|
158
|
+
* llm.sendMessageWithImage('Describe this', imagePath);
|
|
159
|
+
* }
|
|
160
|
+
* ```
|
|
161
|
+
*/
|
|
162
|
+
export function checkMultimodalSupport(): string | undefined {
|
|
163
|
+
if (Platform.OS === "ios") {
|
|
164
|
+
return "Multimodal (image/audio) is not yet supported on iOS. LiteRT-LM iOS SDK is pending.";
|
|
165
|
+
}
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Download URL for the Gemma 3n E2B IT INT4 model.
|
|
171
|
+
*/
|
|
172
|
+
export const GEMMA_3N_E2B_IT_INT4 =
|
|
173
|
+
"https://litert.dev/gemma-3n-E2B-it-int4.litertlm";
|