react-native-nitro-mlx 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import NitroModules
|
|
3
|
+
internal import MLX
|
|
4
|
+
internal import MLXLLM
|
|
5
|
+
internal import MLXLMCommon
|
|
6
|
+
|
|
7
|
+
class HybridLLM: HybridLLMSpec {
|
|
8
|
+
private var session: ChatSession?
|
|
9
|
+
private var currentTask: Task<String, Error>?
|
|
10
|
+
private var lastStats: GenerationStats = GenerationStats(
|
|
11
|
+
tokenCount: 0,
|
|
12
|
+
tokensPerSecond: 0,
|
|
13
|
+
timeToFirstToken: 0,
|
|
14
|
+
totalTime: 0
|
|
15
|
+
)
|
|
16
|
+
private var modelFactory: ModelFactory = LLMModelFactory.shared
|
|
17
|
+
|
|
18
|
+
var isLoaded: Bool { session != nil }
|
|
19
|
+
var isGenerating: Bool { currentTask != nil }
|
|
20
|
+
var modelId: String = ""
|
|
21
|
+
var debug: Bool = false
|
|
22
|
+
var systemPrompt: String = "You are a helpful assistant."
|
|
23
|
+
|
|
24
|
+
private func log(_ message: String) {
|
|
25
|
+
if debug {
|
|
26
|
+
print("[MLXReactNative.HybridLLM] \(message)")
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
func load(modelId: String, onProgress: @escaping (Double) -> Void) throws -> Promise<Void> {
|
|
31
|
+
return Promise.async { [self] in
|
|
32
|
+
let modelDir = await ModelDownloader.shared.getModelDirectory(modelId: modelId)
|
|
33
|
+
log("Loading from directory: \(modelDir.path)")
|
|
34
|
+
|
|
35
|
+
let config = ModelConfiguration(directory: modelDir)
|
|
36
|
+
let container = try await modelFactory.loadContainer(
|
|
37
|
+
configuration: config
|
|
38
|
+
) { progress in
|
|
39
|
+
onProgress(progress.fractionCompleted)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
self.session = ChatSession(container, instructions: self.systemPrompt)
|
|
43
|
+
self.modelId = modelId
|
|
44
|
+
log("Model loaded with system prompt: \(self.systemPrompt.prefix(50))...")
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
func generate(prompt: String) throws -> Promise<String> {
|
|
49
|
+
guard let session = session else {
|
|
50
|
+
throw LLMError.notLoaded
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return Promise.async { [self] in
|
|
54
|
+
let task = Task<String, Error> {
|
|
55
|
+
log("Generating response for: \(prompt.prefix(50))...")
|
|
56
|
+
let result = try await session.respond(to: prompt)
|
|
57
|
+
log("Generation complete")
|
|
58
|
+
return result
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
self.currentTask = task
|
|
62
|
+
|
|
63
|
+
do {
|
|
64
|
+
let result = try await task.value
|
|
65
|
+
self.currentTask = nil
|
|
66
|
+
return result
|
|
67
|
+
} catch {
|
|
68
|
+
self.currentTask = nil
|
|
69
|
+
throw error
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
func stream(prompt: String, onToken: @escaping (String) -> Void) throws -> Promise<String> {
|
|
75
|
+
guard let session = session else {
|
|
76
|
+
throw LLMError.notLoaded
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return Promise.async { [self] in
|
|
80
|
+
let task = Task<String, Error> {
|
|
81
|
+
var result = ""
|
|
82
|
+
var tokenCount = 0
|
|
83
|
+
let startTime = Date()
|
|
84
|
+
var firstTokenTime: Date?
|
|
85
|
+
|
|
86
|
+
log("Streaming response for: \(prompt.prefix(50))...")
|
|
87
|
+
for try await chunk in session.streamResponse(to: prompt) {
|
|
88
|
+
if Task.isCancelled { break }
|
|
89
|
+
|
|
90
|
+
if firstTokenTime == nil {
|
|
91
|
+
firstTokenTime = Date()
|
|
92
|
+
}
|
|
93
|
+
tokenCount += 1
|
|
94
|
+
result += chunk
|
|
95
|
+
onToken(chunk)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
let endTime = Date()
|
|
99
|
+
let totalTime = endTime.timeIntervalSince(startTime) * 1000
|
|
100
|
+
let timeToFirstToken = (firstTokenTime ?? endTime).timeIntervalSince(startTime) * 1000
|
|
101
|
+
let tokensPerSecond = totalTime > 0 ? Double(tokenCount) / (totalTime / 1000) : 0
|
|
102
|
+
|
|
103
|
+
self.lastStats = GenerationStats(
|
|
104
|
+
tokenCount: Double(tokenCount),
|
|
105
|
+
tokensPerSecond: tokensPerSecond,
|
|
106
|
+
timeToFirstToken: timeToFirstToken,
|
|
107
|
+
totalTime: totalTime
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
log("Stream complete - \(tokenCount) tokens, \(String(format: "%.1f", tokensPerSecond)) tokens/s")
|
|
111
|
+
return result
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
self.currentTask = task
|
|
115
|
+
|
|
116
|
+
do {
|
|
117
|
+
let result = try await task.value
|
|
118
|
+
self.currentTask = nil
|
|
119
|
+
return result
|
|
120
|
+
} catch {
|
|
121
|
+
self.currentTask = nil
|
|
122
|
+
throw error
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
func stop() throws {
|
|
128
|
+
currentTask?.cancel()
|
|
129
|
+
currentTask = nil
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
func getLastGenerationStats() throws -> GenerationStats {
|
|
133
|
+
return lastStats
|
|
134
|
+
}
|
|
135
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import NitroModules
|
|
3
|
+
internal import MLXLMCommon
|
|
4
|
+
internal import MLXLLM
|
|
5
|
+
|
|
6
|
+
class HybridModelManager: HybridModelManagerSpec {
|
|
7
|
+
private let fileManager = FileManager.default
|
|
8
|
+
|
|
9
|
+
var debug: Bool {
|
|
10
|
+
get { ModelDownloader.debug }
|
|
11
|
+
set { ModelDownloader.debug = newValue }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
private func log(_ message: String) {
|
|
15
|
+
if debug {
|
|
16
|
+
print("[MLXReactNative.HybridModelManager] \(message)")
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
func download(
|
|
21
|
+
modelId: String,
|
|
22
|
+
progressCallback: @escaping (Double) -> Void
|
|
23
|
+
) throws -> Promise<String> {
|
|
24
|
+
return Promise.async { [self] in
|
|
25
|
+
log("Starting download for: \(modelId)")
|
|
26
|
+
|
|
27
|
+
let modelDir = try await ModelDownloader.shared.download(
|
|
28
|
+
modelId: modelId,
|
|
29
|
+
progressCallback: progressCallback
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
log("Download complete: \(modelDir.path)")
|
|
33
|
+
return modelDir.path
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
func isDownloaded(modelId: String) throws -> Promise<Bool> {
|
|
38
|
+
return Promise.async {
|
|
39
|
+
return await ModelDownloader.shared.isDownloaded(modelId: modelId)
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
func getDownloadedModels() throws -> Promise<[String]> {
|
|
44
|
+
return Promise.async { [self] in
|
|
45
|
+
let docsDir = fileManager.urls(for: .documentDirectory, in: .userDomainMask).first!
|
|
46
|
+
let modelsDir = docsDir.appendingPathComponent("huggingface/models")
|
|
47
|
+
|
|
48
|
+
guard fileManager.fileExists(atPath: modelsDir.path) else {
|
|
49
|
+
return []
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
let contents = try fileManager.contentsOfDirectory(
|
|
53
|
+
at: modelsDir,
|
|
54
|
+
includingPropertiesForKeys: [.isDirectoryKey]
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
return contents
|
|
58
|
+
.filter { url in
|
|
59
|
+
var isDir: ObjCBool = false
|
|
60
|
+
return fileManager.fileExists(atPath: url.path, isDirectory: &isDir) && isDir.boolValue
|
|
61
|
+
}
|
|
62
|
+
.map { $0.lastPathComponent.replacingOccurrences(of: "_", with: "/") }
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
func deleteModel(modelId: String) throws -> Promise<Void> {
|
|
67
|
+
return Promise.async {
|
|
68
|
+
try await ModelDownloader.shared.deleteModel(modelId: modelId)
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
func getModelPath(modelId: String) throws -> Promise<String> {
|
|
73
|
+
return Promise.async {
|
|
74
|
+
return await ModelDownloader.shared.getModelDirectory(modelId: modelId).path
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
actor ModelDownloader: NSObject {
|
|
4
|
+
static let shared = ModelDownloader()
|
|
5
|
+
static var debug: Bool = false
|
|
6
|
+
|
|
7
|
+
private let fileManager = FileManager.default
|
|
8
|
+
|
|
9
|
+
private func log(_ message: String) {
|
|
10
|
+
if Self.debug {
|
|
11
|
+
print("[Downloader] \(message)")
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
func download(
|
|
16
|
+
modelId: String,
|
|
17
|
+
progressCallback: @escaping (Double) -> Void
|
|
18
|
+
) async throws -> URL {
|
|
19
|
+
let requiredFiles = [
|
|
20
|
+
"config.json",
|
|
21
|
+
"tokenizer.json",
|
|
22
|
+
"tokenizer_config.json",
|
|
23
|
+
"model.safetensors"
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
let modelDir = getModelDirectory(modelId: modelId)
|
|
27
|
+
try fileManager.createDirectory(at: modelDir, withIntermediateDirectories: true)
|
|
28
|
+
|
|
29
|
+
log("Model directory: \(modelDir.path)")
|
|
30
|
+
log("Files to download: \(requiredFiles)")
|
|
31
|
+
|
|
32
|
+
var downloaded = 0
|
|
33
|
+
|
|
34
|
+
for file in requiredFiles {
|
|
35
|
+
let destURL = modelDir.appendingPathComponent(file)
|
|
36
|
+
|
|
37
|
+
if fileManager.fileExists(atPath: destURL.path) {
|
|
38
|
+
log("File exists, skipping: \(file)")
|
|
39
|
+
downloaded += 1
|
|
40
|
+
progressCallback(Double(downloaded) / Double(requiredFiles.count))
|
|
41
|
+
continue
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
let urlString = "https://huggingface.co/\(modelId)/resolve/main/\(file)"
|
|
45
|
+
guard let url = URL(string: urlString) else {
|
|
46
|
+
log("Invalid URL: \(urlString)")
|
|
47
|
+
continue
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
log("Downloading: \(file)")
|
|
51
|
+
|
|
52
|
+
let (tempURL, response) = try await URLSession.shared.download(from: url)
|
|
53
|
+
|
|
54
|
+
guard let httpResponse = response as? HTTPURLResponse else {
|
|
55
|
+
log("Invalid response for: \(file)")
|
|
56
|
+
continue
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
log("Response status: \(httpResponse.statusCode) for \(file)")
|
|
60
|
+
|
|
61
|
+
if httpResponse.statusCode == 200 {
|
|
62
|
+
if fileManager.fileExists(atPath: destURL.path) {
|
|
63
|
+
try fileManager.removeItem(at: destURL)
|
|
64
|
+
}
|
|
65
|
+
try fileManager.moveItem(at: tempURL, to: destURL)
|
|
66
|
+
log("Saved: \(file)")
|
|
67
|
+
} else {
|
|
68
|
+
log("Failed to download: \(file) - Status: \(httpResponse.statusCode)")
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
downloaded += 1
|
|
72
|
+
progressCallback(Double(downloaded) / Double(requiredFiles.count))
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return modelDir
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
func isDownloaded(modelId: String) -> Bool {
|
|
79
|
+
let modelDir = getModelDirectory(modelId: modelId)
|
|
80
|
+
let requiredFiles = ["config.json", "model.safetensors", "tokenizer.json"]
|
|
81
|
+
|
|
82
|
+
let allExist = requiredFiles.allSatisfy { file in
|
|
83
|
+
fileManager.fileExists(atPath: modelDir.appendingPathComponent(file).path)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
log("isDownloaded(\(modelId)): \(allExist)")
|
|
87
|
+
return allExist
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
func getModelDirectory(modelId: String) -> URL {
|
|
91
|
+
let docsDir = fileManager.urls(for: .documentDirectory, in: .userDomainMask).first!
|
|
92
|
+
return docsDir
|
|
93
|
+
.appendingPathComponent("huggingface/models")
|
|
94
|
+
.appendingPathComponent(modelId.replacingOccurrences(of: "/", with: "_"))
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
func deleteModel(modelId: String) throws {
|
|
98
|
+
let modelDir = getModelDirectory(modelId: modelId)
|
|
99
|
+
if fileManager.fileExists(atPath: modelDir.path) {
|
|
100
|
+
try fileManager.removeItem(at: modelDir)
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "react-native-nitro-mlx",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Nitro module package",
|
|
5
5
|
"main": "./lib/module/index.js",
|
|
6
6
|
"module": "./lib/module/index.js",
|
|
@@ -41,26 +41,29 @@
|
|
|
41
41
|
"ios/**/*.m",
|
|
42
42
|
"ios/**/*.mm",
|
|
43
43
|
"ios/**/*.cpp",
|
|
44
|
-
"ios
|
|
44
|
+
"ios/**/*.swift",
|
|
45
45
|
"app.plugin.js",
|
|
46
46
|
"*.podspec",
|
|
47
47
|
"README.md"
|
|
48
48
|
],
|
|
49
49
|
"repository": {
|
|
50
50
|
"type": "git",
|
|
51
|
-
"url": "git+https://github.com/
|
|
51
|
+
"url": "git+https://github.com/corasan/react-native-nitro-mlx.git"
|
|
52
52
|
},
|
|
53
53
|
"author": "Henry Paulino",
|
|
54
54
|
"license": "MIT",
|
|
55
|
-
"bugs": "https://github.com/
|
|
56
|
-
"homepage": "https://github.com/
|
|
55
|
+
"bugs": "https://github.com/corasan/react-native-nitro-mlx/issues",
|
|
56
|
+
"homepage": "https://github.com/corasan/react-native-nitro-mlx#readme",
|
|
57
57
|
"publishConfig": {
|
|
58
58
|
"registry": "https://registry.npmjs.org/"
|
|
59
59
|
},
|
|
60
60
|
"devDependencies": {
|
|
61
61
|
"@expo/config-plugins": "^9.0.10",
|
|
62
|
+
"@release-it/bumper": "^7.0.5",
|
|
63
|
+
"@release-it/conventional-changelog": "^10.0.1",
|
|
62
64
|
"nitrogen": "^0.31.10",
|
|
63
|
-
"react-native-builder-bob": "^0.40.13"
|
|
65
|
+
"react-native-builder-bob": "^0.40.13",
|
|
66
|
+
"release-it": "^19.0.4"
|
|
64
67
|
},
|
|
65
68
|
"peerDependencies": {
|
|
66
69
|
"react": "*",
|
|
@@ -69,15 +72,58 @@
|
|
|
69
72
|
},
|
|
70
73
|
"release-it": {
|
|
71
74
|
"npm": {
|
|
72
|
-
"publish": true
|
|
75
|
+
"publish": true,
|
|
76
|
+
"skipVersion": true
|
|
73
77
|
},
|
|
74
|
-
"git": false,
|
|
75
78
|
"github": {
|
|
76
|
-
"release":
|
|
79
|
+
"release": true,
|
|
80
|
+
"releaseName": "v${version}"
|
|
77
81
|
},
|
|
78
82
|
"hooks": {
|
|
79
|
-
"
|
|
80
|
-
"
|
|
83
|
+
"after:bump": "bun specs",
|
|
84
|
+
"before:release": "bun run build"
|
|
85
|
+
},
|
|
86
|
+
"git": {
|
|
87
|
+
"commitMessage": "chore: release ${version}",
|
|
88
|
+
"tagName": "v${version}",
|
|
89
|
+
"requireCleanWorkingDir": false
|
|
90
|
+
},
|
|
91
|
+
"plugins": {
|
|
92
|
+
"@release-it/bumper": {
|
|
93
|
+
"out": [
|
|
94
|
+
{
|
|
95
|
+
"file": "package.json",
|
|
96
|
+
"path": "version"
|
|
97
|
+
}
|
|
98
|
+
]
|
|
99
|
+
},
|
|
100
|
+
"@release-it/conventional-changelog": {
|
|
101
|
+
"preset": {
|
|
102
|
+
"name": "conventionalcommits",
|
|
103
|
+
"types": [
|
|
104
|
+
{
|
|
105
|
+
"type": "feat",
|
|
106
|
+
"section": "✨ Features"
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
"type": "fix",
|
|
110
|
+
"section": "🐞 Fixes"
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
"type": "chore(deps)",
|
|
114
|
+
"section": "🛠️ Dependency Upgrades"
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
"type": "perf",
|
|
118
|
+
"section": "🏎️ Performance Improvements"
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
"type": "docs",
|
|
122
|
+
"section": "📚 Documentation"
|
|
123
|
+
}
|
|
124
|
+
]
|
|
125
|
+
}
|
|
126
|
+
}
|
|
81
127
|
}
|
|
82
128
|
},
|
|
83
129
|
"react-native-builder-bob": {
|