react-native-yolo 0.0.5 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/src/main/java/com/yolo/HybridYolo.kt +12 -235
- package/android/src/main/java/com/yolo/HybridYoloModel.kt +212 -0
- package/android/src/main/java/com/yolo/utils/BitmapOrientationFixer.kt +8 -1
- package/lib/commonjs/index.js +4 -3
- package/lib/commonjs/index.js.map +1 -1
- package/lib/module/index.js +4 -3
- package/lib/module/index.js.map +1 -1
- package/lib/typescript/src/index.d.ts +12 -3
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/specs/yolo.nitro.d.ts +10 -6
- package/lib/typescript/src/specs/yolo.nitro.d.ts.map +1 -1
- package/nitrogen/generated/android/Yolo+autolinking.cmake +2 -0
- package/nitrogen/generated/android/YoloOnLoad.cpp +2 -0
- package/nitrogen/generated/android/c++/JHybridYoloModelSpec.cpp +78 -0
- package/nitrogen/generated/android/c++/JHybridYoloModelSpec.hpp +64 -0
- package/nitrogen/generated/android/c++/JHybridYoloSpec.cpp +9 -32
- package/nitrogen/generated/android/c++/JHybridYoloSpec.hpp +1 -3
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/yolo/HybridYoloModelSpec.kt +59 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/yolo/HybridYoloSpec.kt +1 -9
- package/nitrogen/generated/ios/Yolo-Swift-Cxx-Bridge.cpp +17 -0
- package/nitrogen/generated/ios/Yolo-Swift-Cxx-Bridge.hpp +48 -31
- package/nitrogen/generated/ios/Yolo-Swift-Cxx-Umbrella.hpp +5 -0
- package/nitrogen/generated/ios/c++/HybridYoloModelSpecSwift.cpp +11 -0
- package/nitrogen/generated/ios/c++/HybridYoloModelSpecSwift.hpp +97 -0
- package/nitrogen/generated/ios/c++/HybridYoloSpecSwift.hpp +6 -24
- package/nitrogen/generated/ios/swift/HybridYoloModelSpec.swift +57 -0
- package/nitrogen/generated/ios/swift/HybridYoloModelSpec_cxx.swift +160 -0
- package/nitrogen/generated/ios/swift/HybridYoloSpec.swift +1 -3
- package/nitrogen/generated/ios/swift/HybridYoloSpec_cxx.swift +8 -38
- package/nitrogen/generated/shared/c++/HybridYoloModelSpec.cpp +22 -0
- package/nitrogen/generated/shared/c++/HybridYoloModelSpec.hpp +69 -0
- package/nitrogen/generated/shared/c++/HybridYoloSpec.cpp +0 -2
- package/nitrogen/generated/shared/c++/HybridYoloSpec.hpp +5 -8
- package/package.json +1 -1
- package/src/index.ts +14 -4
- package/src/specs/yolo.nitro.ts +12 -3
|
@@ -1,265 +1,42 @@
|
|
|
1
1
|
package com.yolo
|
|
2
2
|
|
|
3
|
-
import android.
|
|
3
|
+
import android.util.Base64
|
|
4
4
|
import android.util.Log
|
|
5
|
-
|
|
6
|
-
import com.margelo.nitro.yolo.HybridYoloSpec
|
|
7
|
-
import java.io.File
|
|
8
|
-
import java.io.RandomAccessFile
|
|
9
|
-
import java.net.URL
|
|
10
|
-
import java.nio.MappedByteBuffer
|
|
11
|
-
import java.nio.channels.FileChannel
|
|
12
|
-
import org.tensorflow.lite.Interpreter
|
|
13
|
-
import org.tensorflow.lite.support.common.FileUtil
|
|
14
|
-
import yolo.com.loader.YoloModelLoader
|
|
5
|
+
|
|
15
6
|
import com.margelo.nitro.camera.HybridFrameSpec
|
|
7
|
+
import com.margelo.nitro.yolo.HybridYoloModelSpec
|
|
8
|
+
import com.margelo.nitro.yolo.HybridYoloSpec
|
|
16
9
|
|
|
17
|
-
import android.util.Base64
|
|
18
10
|
import com.yolo.utils.FrameJpegConverter
|
|
19
11
|
import com.yolo.utils.FrameValidator
|
|
20
12
|
|
|
21
|
-
import com.margelo.nitro.yolo.Detection
|
|
22
|
-
import com.margelo.nitro.yolo.BoundingBox
|
|
23
|
-
|
|
24
|
-
import java.nio.ByteBuffer
|
|
25
|
-
import java.nio.ByteOrder
|
|
26
|
-
import org.tensorflow.lite.DataType
|
|
27
|
-
import kotlin.math.roundToInt
|
|
28
|
-
|
|
29
13
|
|
|
30
14
|
class HybridYolo : HybridYoloSpec() {
|
|
31
15
|
companion object {
|
|
32
16
|
private const val TAG = "YOLO_TAG"
|
|
33
17
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
private var inputWidth = 0
|
|
38
|
-
private var inputHeight = 0
|
|
39
|
-
private var inputDataType: DataType? = null
|
|
40
|
-
|
|
41
|
-
private val modelLoader = YoloModelLoader()
|
|
42
|
-
|
|
43
|
-
override fun sum(num1: Double, num2: Double): Double {
|
|
44
|
-
return num1 + num2
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
override fun loadModel(modelPath: String) {
|
|
48
|
-
val context =
|
|
49
|
-
NitroModules.applicationContext ?: throw IllegalStateException("Context is null")
|
|
50
|
-
|
|
51
|
-
Log.d(TAG, "Trying to load: $modelPath")
|
|
52
|
-
|
|
53
|
-
try {
|
|
54
|
-
val modelBuffer = modelLoader.load(modelPath)
|
|
55
|
-
|
|
56
|
-
interpreter?.close()
|
|
57
|
-
interpreter = Interpreter(modelBuffer)
|
|
58
|
-
|
|
59
|
-
val inputTensor = interpreter!!.getInputTensor(0)
|
|
60
|
-
val shape = inputTensor.shape()
|
|
61
|
-
|
|
62
|
-
inputHeight = shape[1]
|
|
63
|
-
inputWidth = shape[2]
|
|
64
|
-
inputDataType = inputTensor.dataType()
|
|
65
|
-
inputBuffer = modelLoader.makeInputBuffer(interpreter!!)
|
|
66
|
-
|
|
67
|
-
val outputTensor = interpreter!!.getOutputTensor(0)
|
|
68
|
-
Log.d(TAG, "✅ YOLO model loaded")
|
|
69
|
-
Log.d(TAG, "📥 Input shape: ${inputTensor.shape().contentToString()}")
|
|
70
|
-
Log.d(TAG, "📤 Output shape: ${outputTensor.shape().contentToString()}")
|
|
71
|
-
Log.d(TAG, "📥 Input type: ${inputTensor.dataType()}")
|
|
72
|
-
Log.d(TAG, "📤 Output type: ${outputTensor.dataType()}")
|
|
73
|
-
|
|
74
|
-
Log.d(TAG, "Input shape=${shape.contentToString()} type=$inputDataType")
|
|
75
|
-
Log.d(TAG, "✅ Model loaded successfully!")
|
|
76
|
-
} catch (e: Exception) {
|
|
77
|
-
Log.e(TAG, "❌ Failed to load model: ${e.message}", e)
|
|
78
|
-
}
|
|
18
|
+
override fun loadModel(modelPath: String): HybridYoloModelSpec {
|
|
19
|
+
Log.d(TAG, "Trying to load model object: $modelPath")
|
|
20
|
+
return HybridYoloModel(modelPath)
|
|
79
21
|
}
|
|
80
22
|
override fun frameToBase64(frame: HybridFrameSpec): String {
|
|
23
|
+
Log.d(TAG, "Trying to convert frame to base64")
|
|
81
24
|
return try {
|
|
25
|
+
Log.d(TAG, "frameToBase64: frame: $frame")
|
|
82
26
|
if (!FrameValidator.isValidYuv(frame)) return ""
|
|
27
|
+
Log.d(TAG, "frameToBase64: frame is valid YUV")
|
|
83
28
|
|
|
84
29
|
val jpegBytes = FrameJpegConverter.toJpegBytes(
|
|
85
30
|
frame = frame,
|
|
86
31
|
quality = 80
|
|
87
32
|
)
|
|
33
|
+
Log.d(TAG, "frameToBase64: jpegBytes size: ${jpegBytes.size}")
|
|
88
34
|
|
|
89
35
|
Base64.encodeToString(jpegBytes, Base64.NO_WRAP)
|
|
36
|
+
|
|
90
37
|
} catch (e: Exception) {
|
|
91
38
|
Log.e(TAG, "❌ frameToBase64 failed", e)
|
|
92
39
|
""
|
|
93
40
|
}
|
|
94
41
|
}
|
|
95
|
-
|
|
96
|
-
override fun detect(frame: HybridFrameSpec): Array<Detection> {
|
|
97
|
-
val localInterpreter = interpreter ?: run {
|
|
98
|
-
Log.e(TAG, "❌ Model is not loaded. Please call loadModel() first.")
|
|
99
|
-
return emptyArray()
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
if (!FrameValidator.isValidYuv(frame)) {
|
|
103
|
-
Log.e(TAG, "❌ Invalid frame provided for detection.")
|
|
104
|
-
return emptyArray()
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
val input = inputBuffer ?: return emptyArray()
|
|
108
|
-
|
|
109
|
-
fillInputFromYuvFrame(
|
|
110
|
-
frame = frame,
|
|
111
|
-
input = input,
|
|
112
|
-
dstWidth = inputWidth,
|
|
113
|
-
dstHeight = inputHeight,
|
|
114
|
-
dataType = inputDataType ?: DataType.FLOAT32
|
|
115
|
-
)
|
|
116
|
-
|
|
117
|
-
val output = Array(1) { Array(300) { FloatArray(6) } }
|
|
118
|
-
|
|
119
|
-
localInterpreter.run(input, output)
|
|
120
|
-
|
|
121
|
-
val detections = parseNmsOutput(output, confidenceThreshold = 0.5f)
|
|
122
|
-
if (detections.isEmpty()) {
|
|
123
|
-
val best = output[0].maxByOrNull { it[4] }
|
|
124
|
-
Log.d(TAG, "No detections. Best row=${best?.contentToString()}")
|
|
125
|
-
} else {
|
|
126
|
-
Log.d(TAG, "✅ Detections: ${detections.size}")
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
return detections.toTypedArray()
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
private fun parseNmsOutput(
|
|
134
|
-
output: Array<Array<FloatArray>>,
|
|
135
|
-
confidenceThreshold: Float = 0.5f
|
|
136
|
-
): List<Detection> {
|
|
137
|
-
val detections = mutableListOf<Detection>()
|
|
138
|
-
|
|
139
|
-
val batchMatrix = output[0]
|
|
140
|
-
|
|
141
|
-
for (i in batchMatrix.indices) {
|
|
142
|
-
val row = batchMatrix[i]
|
|
143
|
-
|
|
144
|
-
val score = row[4]
|
|
145
|
-
if (score < confidenceThreshold) continue
|
|
146
|
-
|
|
147
|
-
val x1 = row[0].toDouble()
|
|
148
|
-
val y1 = row[1].toDouble()
|
|
149
|
-
val x2 = row[2].toDouble()
|
|
150
|
-
val y2 = row[3].toDouble()
|
|
151
|
-
val classId = row[5].toDouble()
|
|
152
|
-
|
|
153
|
-
detections.add(
|
|
154
|
-
Detection(
|
|
155
|
-
boundingBox = BoundingBox(
|
|
156
|
-
x1 = x1,
|
|
157
|
-
y1 = y1,
|
|
158
|
-
x2 = x2,
|
|
159
|
-
y2 = y2
|
|
160
|
-
),
|
|
161
|
-
score = score.toDouble(),
|
|
162
|
-
classId = classId
|
|
163
|
-
)
|
|
164
|
-
)
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
return detections
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
private fun fillInputFromYuvFrame(
|
|
173
|
-
frame: HybridFrameSpec,
|
|
174
|
-
input: ByteBuffer,
|
|
175
|
-
dstWidth: Int,
|
|
176
|
-
dstHeight: Int,
|
|
177
|
-
dataType: DataType
|
|
178
|
-
) {
|
|
179
|
-
val srcWidth = frame.width.toInt()
|
|
180
|
-
val srcHeight = frame.height.toInt()
|
|
181
|
-
|
|
182
|
-
val planes = frame.getPlanes()
|
|
183
|
-
|
|
184
|
-
val yPlane = planes[0]
|
|
185
|
-
val uPlane = planes[1]
|
|
186
|
-
val vPlane = planes[2]
|
|
187
|
-
|
|
188
|
-
val yBytes = yPlane.getPixelBuffer().toByteArray()
|
|
189
|
-
val uBytes = uPlane.getPixelBuffer().toByteArray()
|
|
190
|
-
val vBytes = vPlane.getPixelBuffer().toByteArray()
|
|
191
|
-
|
|
192
|
-
val yRowStride = yPlane.bytesPerRow.roundToInt()
|
|
193
|
-
val uRowStride = uPlane.bytesPerRow.roundToInt()
|
|
194
|
-
val vRowStride = vPlane.bytesPerRow.roundToInt()
|
|
195
|
-
|
|
196
|
-
input.rewind()
|
|
197
|
-
|
|
198
|
-
for (dy in 0 until dstHeight) {
|
|
199
|
-
// val sy = dy * srcHeight / dstHeight
|
|
200
|
-
|
|
201
|
-
for (dx in 0 until dstWidth) {
|
|
202
|
-
val srcX = dy * srcWidth / dstHeight
|
|
203
|
-
val srcY = srcHeight - 1 - (dx * srcHeight / dstWidth)
|
|
204
|
-
|
|
205
|
-
val yIndex = srcY * yRowStride + srcX
|
|
206
|
-
|
|
207
|
-
val uvX = srcX / 2
|
|
208
|
-
val uvY = srcY / 2
|
|
209
|
-
|
|
210
|
-
val uIndex = uvY * uRowStride + uvX * 2
|
|
211
|
-
val vIndex = uvY * vRowStride + uvX * 2
|
|
212
|
-
|
|
213
|
-
if (
|
|
214
|
-
yIndex >= yBytes.size ||
|
|
215
|
-
uIndex >= uBytes.size ||
|
|
216
|
-
vIndex >= vBytes.size
|
|
217
|
-
) {
|
|
218
|
-
continue
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
val y = yBytes[yIndex].toInt() and 0xFF
|
|
222
|
-
val u = uBytes[uIndex].toInt() and 0xFF
|
|
223
|
-
val v = vBytes[vIndex].toInt() and 0xFF
|
|
224
|
-
|
|
225
|
-
val rFloat = y + 1.402f * (v - 128)
|
|
226
|
-
val gFloat = y - 0.344136f * (u - 128) - 0.714136f * (v - 128)
|
|
227
|
-
val bFloat = y + 1.772f * (u - 128)
|
|
228
|
-
|
|
229
|
-
val r = rFloat.roundToInt().coerceIn(0, 255)
|
|
230
|
-
val g = gFloat.roundToInt().coerceIn(0, 255)
|
|
231
|
-
val b = bFloat.roundToInt().coerceIn(0, 255)
|
|
232
|
-
|
|
233
|
-
when (dataType) {
|
|
234
|
-
DataType.FLOAT32 -> {
|
|
235
|
-
input.putFloat(r / 255f)
|
|
236
|
-
input.putFloat(g / 255f)
|
|
237
|
-
input.putFloat(b / 255f)
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
DataType.UINT8 -> {
|
|
241
|
-
input.put(r.toByte())
|
|
242
|
-
input.put(g.toByte())
|
|
243
|
-
input.put(b.toByte())
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
else -> error("Unsupported input type: $dataType")
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
input.rewind()
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
private fun yuvToRgb(y: Int, u: Int, v: Int): IntArray {
|
|
254
|
-
val yf = y.toFloat()
|
|
255
|
-
val uf = u.toFloat() - 128f
|
|
256
|
-
val vf = v.toFloat() - 128f
|
|
257
|
-
|
|
258
|
-
val r = (yf + 1.402f * vf).roundToInt().coerceIn(0, 255)
|
|
259
|
-
val g = (yf - 0.344136f * uf - 0.714136f * vf).roundToInt().coerceIn(0, 255)
|
|
260
|
-
val b = (yf + 1.772f * uf).roundToInt().coerceIn(0, 255)
|
|
261
|
-
|
|
262
|
-
return intArrayOf(r, g, b)
|
|
263
|
-
}
|
|
264
|
-
|
|
265
42
|
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
package com.yolo
|
|
2
|
+
|
|
3
|
+
import android.util.Log
|
|
4
|
+
import com.margelo.nitro.yolo.HybridYoloModelSpec
|
|
5
|
+
import com.margelo.nitro.yolo.Detection
|
|
6
|
+
import com.margelo.nitro.yolo.BoundingBox
|
|
7
|
+
import com.margelo.nitro.camera.HybridFrameSpec
|
|
8
|
+
import com.yolo.utils.FrameValidator
|
|
9
|
+
import java.nio.ByteBuffer
|
|
10
|
+
import org.tensorflow.lite.DataType
|
|
11
|
+
import org.tensorflow.lite.Interpreter
|
|
12
|
+
import yolo.com.loader.YoloModelLoader
|
|
13
|
+
import kotlin.math.roundToInt
|
|
14
|
+
|
|
15
|
+
class HybridYoloModel(
|
|
16
|
+
modelPath: String
|
|
17
|
+
) : HybridYoloModelSpec() {
|
|
18
|
+
|
|
19
|
+
companion object {
|
|
20
|
+
private const val TAG = "YOLO_MODEL_TAG"
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
private val modelLoader = YoloModelLoader()
|
|
24
|
+
|
|
25
|
+
private var interpreter: Interpreter? = null
|
|
26
|
+
private var inputBuffer: ByteBuffer? = null
|
|
27
|
+
private var inputWidth = 0
|
|
28
|
+
private var inputHeight = 0
|
|
29
|
+
private var inputDataType: DataType? = null
|
|
30
|
+
|
|
31
|
+
private val lock = Any()
|
|
32
|
+
|
|
33
|
+
init {
|
|
34
|
+
load(modelPath)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
private fun load(modelPath: String) {
|
|
38
|
+
try {
|
|
39
|
+
val modelBuffer = modelLoader.load(modelPath)
|
|
40
|
+
|
|
41
|
+
interpreter = Interpreter(modelBuffer)
|
|
42
|
+
|
|
43
|
+
val localInterpreter = interpreter!!
|
|
44
|
+
|
|
45
|
+
val inputTensor = localInterpreter.getInputTensor(0)
|
|
46
|
+
val outputTensor = localInterpreter.getOutputTensor(0)
|
|
47
|
+
|
|
48
|
+
val shape = inputTensor.shape()
|
|
49
|
+
|
|
50
|
+
inputHeight = shape[1]
|
|
51
|
+
inputWidth = shape[2]
|
|
52
|
+
inputDataType = inputTensor.dataType()
|
|
53
|
+
inputBuffer = modelLoader.makeInputBuffer(localInterpreter)
|
|
54
|
+
|
|
55
|
+
Log.d(TAG, "✅ YOLO model instance loaded")
|
|
56
|
+
Log.d(TAG, "📥 Input shape: ${inputTensor.shape().contentToString()}")
|
|
57
|
+
Log.d(TAG, "📤 Output shape: ${outputTensor.shape().contentToString()}")
|
|
58
|
+
Log.d(TAG, "📥 Input type: ${inputTensor.dataType()}")
|
|
59
|
+
Log.d(TAG, "📤 Output type: ${outputTensor.dataType()}")
|
|
60
|
+
} catch (e: Exception) {
|
|
61
|
+
Log.e(TAG, "❌ Failed to load model instance", e)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
override fun detect(frame: HybridFrameSpec): Array<Detection> {
|
|
66
|
+
val localInterpreter = interpreter ?: run {
|
|
67
|
+
Log.e(TAG, "❌ This model instance is not loaded.")
|
|
68
|
+
return emptyArray()
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (!FrameValidator.isValidYuv(frame)) {
|
|
72
|
+
Log.e(TAG, "❌ Invalid frame provided for detection.")
|
|
73
|
+
return emptyArray()
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
val input = inputBuffer ?: return emptyArray()
|
|
77
|
+
|
|
78
|
+
synchronized(lock) {
|
|
79
|
+
input.rewind()
|
|
80
|
+
|
|
81
|
+
fillInputFromYuvFrame(
|
|
82
|
+
frame = frame,
|
|
83
|
+
input = input,
|
|
84
|
+
dstWidth = inputWidth,
|
|
85
|
+
dstHeight = inputHeight,
|
|
86
|
+
dataType = inputDataType ?: DataType.FLOAT32
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
val output = Array(1) { Array(300) { FloatArray(6) } }
|
|
90
|
+
|
|
91
|
+
localInterpreter.run(input, output)
|
|
92
|
+
|
|
93
|
+
val detections = parseNmsOutput(output, confidenceThreshold = 0.5f)
|
|
94
|
+
|
|
95
|
+
return detections.toTypedArray()
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
override fun close() {
|
|
100
|
+
interpreter?.close()
|
|
101
|
+
interpreter = null
|
|
102
|
+
inputBuffer = null
|
|
103
|
+
Log.d(TAG, "🧹 YOLO model disposed")
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
private fun parseNmsOutput(
|
|
107
|
+
output: Array<Array<FloatArray>>,
|
|
108
|
+
confidenceThreshold: Float = 0.5f
|
|
109
|
+
): List<Detection> {
|
|
110
|
+
val detections = mutableListOf<Detection>()
|
|
111
|
+
|
|
112
|
+
for (row in output[0]) {
|
|
113
|
+
val score = row[4]
|
|
114
|
+
if (score < confidenceThreshold) continue
|
|
115
|
+
|
|
116
|
+
detections.add(
|
|
117
|
+
Detection(
|
|
118
|
+
boundingBox = BoundingBox(
|
|
119
|
+
x1 = row[0].toDouble(),
|
|
120
|
+
y1 = row[1].toDouble(),
|
|
121
|
+
x2 = row[2].toDouble(),
|
|
122
|
+
y2 = row[3].toDouble()
|
|
123
|
+
),
|
|
124
|
+
score = score.toDouble(),
|
|
125
|
+
classId = row[5].toDouble()
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return detections
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
private fun fillInputFromYuvFrame(
|
|
134
|
+
frame: HybridFrameSpec,
|
|
135
|
+
input: ByteBuffer,
|
|
136
|
+
dstWidth: Int,
|
|
137
|
+
dstHeight: Int,
|
|
138
|
+
dataType: DataType
|
|
139
|
+
) {
|
|
140
|
+
val srcWidth = frame.width.toInt()
|
|
141
|
+
val srcHeight = frame.height.toInt()
|
|
142
|
+
|
|
143
|
+
val planes = frame.getPlanes()
|
|
144
|
+
|
|
145
|
+
val yPlane = planes[0]
|
|
146
|
+
val uPlane = planes[1]
|
|
147
|
+
val vPlane = planes[2]
|
|
148
|
+
|
|
149
|
+
val yBytes = yPlane.getPixelBuffer().toByteArray()
|
|
150
|
+
val uBytes = uPlane.getPixelBuffer().toByteArray()
|
|
151
|
+
val vBytes = vPlane.getPixelBuffer().toByteArray()
|
|
152
|
+
|
|
153
|
+
val yRowStride = yPlane.bytesPerRow.roundToInt()
|
|
154
|
+
val uRowStride = uPlane.bytesPerRow.roundToInt()
|
|
155
|
+
val vRowStride = vPlane.bytesPerRow.roundToInt()
|
|
156
|
+
|
|
157
|
+
input.rewind()
|
|
158
|
+
|
|
159
|
+
for (dy in 0 until dstHeight) {
|
|
160
|
+
for (dx in 0 until dstWidth) {
|
|
161
|
+
val srcX = dy * srcWidth / dstHeight
|
|
162
|
+
val srcY = srcHeight - 1 - (dx * srcHeight / dstWidth)
|
|
163
|
+
|
|
164
|
+
val yIndex = srcY * yRowStride + srcX
|
|
165
|
+
|
|
166
|
+
val uvX = srcX / 2
|
|
167
|
+
val uvY = srcY / 2
|
|
168
|
+
|
|
169
|
+
val uIndex = uvY * uRowStride + uvX * 2
|
|
170
|
+
val vIndex = uvY * vRowStride + uvX * 2
|
|
171
|
+
|
|
172
|
+
if (
|
|
173
|
+
yIndex >= yBytes.size ||
|
|
174
|
+
uIndex >= uBytes.size ||
|
|
175
|
+
vIndex >= vBytes.size
|
|
176
|
+
) {
|
|
177
|
+
continue
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
val y = yBytes[yIndex].toInt() and 0xFF
|
|
181
|
+
val u = uBytes[uIndex].toInt() and 0xFF
|
|
182
|
+
val v = vBytes[vIndex].toInt() and 0xFF
|
|
183
|
+
|
|
184
|
+
val rFloat = y + 1.402f * (v - 128)
|
|
185
|
+
val gFloat = y - 0.344136f * (u - 128) - 0.714136f * (v - 128)
|
|
186
|
+
val bFloat = y + 1.772f * (u - 128)
|
|
187
|
+
|
|
188
|
+
val r = rFloat.roundToInt().coerceIn(0, 255)
|
|
189
|
+
val g = gFloat.roundToInt().coerceIn(0, 255)
|
|
190
|
+
val b = bFloat.roundToInt().coerceIn(0, 255)
|
|
191
|
+
|
|
192
|
+
when (dataType) {
|
|
193
|
+
DataType.FLOAT32 -> {
|
|
194
|
+
input.putFloat(r / 255f)
|
|
195
|
+
input.putFloat(g / 255f)
|
|
196
|
+
input.putFloat(b / 255f)
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
DataType.UINT8 -> {
|
|
200
|
+
input.put(r.toByte())
|
|
201
|
+
input.put(g.toByte())
|
|
202
|
+
input.put(b.toByte())
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
else -> error("Unsupported input type: $dataType")
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
input.rewind()
|
|
211
|
+
}
|
|
212
|
+
}
|
|
@@ -4,6 +4,7 @@ import android.graphics.Bitmap
|
|
|
4
4
|
import android.graphics.BitmapFactory
|
|
5
5
|
import android.graphics.Matrix
|
|
6
6
|
import com.margelo.nitro.camera.HybridFrameSpec
|
|
7
|
+
import com.margelo.nitro.camera.CameraOrientation
|
|
7
8
|
import java.io.ByteArrayOutputStream
|
|
8
9
|
|
|
9
10
|
|
|
@@ -20,7 +21,13 @@ object BitmapOrientationFixer {
|
|
|
20
21
|
jpegBytes.size
|
|
21
22
|
) ?: return jpegBytes
|
|
22
23
|
|
|
23
|
-
val rotationDegrees =
|
|
24
|
+
val rotationDegrees = when (frame.orientation) {
|
|
25
|
+
CameraOrientation.LEFT -> 90f
|
|
26
|
+
CameraOrientation.RIGHT -> 270f
|
|
27
|
+
CameraOrientation.UP -> 0f
|
|
28
|
+
CameraOrientation.DOWN -> 180f
|
|
29
|
+
else -> 0f
|
|
30
|
+
}
|
|
24
31
|
|
|
25
32
|
val matrix = Matrix().apply {
|
|
26
33
|
postRotate(rotationDegrees)
|
package/lib/commonjs/index.js
CHANGED
|
@@ -7,12 +7,13 @@ exports.Yolo = void 0;
|
|
|
7
7
|
var _reactNativeNitroModules = require("react-native-nitro-modules");
|
|
8
8
|
var _reactNative = require("react-native");
|
|
9
9
|
const NativeYolo = _reactNativeNitroModules.NitroModules.createHybridObject('Yolo');
|
|
10
|
-
const Yolo = exports.Yolo =
|
|
11
|
-
|
|
10
|
+
const Yolo = exports.Yolo = {
|
|
11
|
+
...NativeYolo,
|
|
12
|
+
loadModel(modelAssetId) {
|
|
12
13
|
const {
|
|
13
14
|
uri
|
|
14
15
|
} = _reactNative.Image.resolveAssetSource(modelAssetId);
|
|
15
16
|
return NativeYolo.loadModel(uri);
|
|
16
17
|
}
|
|
17
|
-
}
|
|
18
|
+
};
|
|
18
19
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_reactNativeNitroModules","require","_reactNative","NativeYolo","NitroModules","createHybridObject","Yolo","exports","
|
|
1
|
+
{"version":3,"names":["_reactNativeNitroModules","require","_reactNative","NativeYolo","NitroModules","createHybridObject","Yolo","exports","loadModel","modelAssetId","uri","Image","resolveAssetSource"],"sourceRoot":"..\\..\\src","sources":["index.ts"],"mappings":";;;;;;AAAA,IAAAA,wBAAA,GAAAC,OAAA;AAEA,IAAAC,YAAA,GAAAD,OAAA;AAEA,MAAME,UAAU,GAAGC,qCAAY,CAACC,kBAAkB,CAAW,MAAM,CAAC;AAG7D,MAAMC,IAAI,GAAAC,OAAA,CAAAD,IAAA,GAAG;EAClB,GAAGH,UAAU;EAEbK,SAASA,CAACC,YAAoB,EAAE;IAC9B,MAAM;MAAEC;IAAI,CAAC,GAAGC,kBAAK,CAACC,kBAAkB,CAACH,YAAY,CAAC;IACtD,OAAON,UAAU,CAACK,SAAS,CAACE,GAAG,CAAC;EAClC;AACF,CAAC","ignoreList":[]}
|
package/lib/module/index.js
CHANGED
|
@@ -3,12 +3,13 @@
|
|
|
3
3
|
import { NitroModules } from 'react-native-nitro-modules';
|
|
4
4
|
import { Image } from 'react-native';
|
|
5
5
|
const NativeYolo = NitroModules.createHybridObject('Yolo');
|
|
6
|
-
export const Yolo =
|
|
7
|
-
|
|
6
|
+
export const Yolo = {
|
|
7
|
+
...NativeYolo,
|
|
8
|
+
loadModel(modelAssetId) {
|
|
8
9
|
const {
|
|
9
10
|
uri
|
|
10
11
|
} = Image.resolveAssetSource(modelAssetId);
|
|
11
12
|
return NativeYolo.loadModel(uri);
|
|
12
13
|
}
|
|
13
|
-
}
|
|
14
|
+
};
|
|
14
15
|
//# sourceMappingURL=index.js.map
|
package/lib/module/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["NitroModules","Image","NativeYolo","createHybridObject","Yolo","
|
|
1
|
+
{"version":3,"names":["NitroModules","Image","NativeYolo","createHybridObject","Yolo","loadModel","modelAssetId","uri","resolveAssetSource"],"sourceRoot":"..\\..\\src","sources":["index.ts"],"mappings":";;AAAA,SAASA,YAAY,QAAQ,4BAA4B;AAEzD,SAASC,KAAK,QAAQ,cAAc;AAEpC,MAAMC,UAAU,GAAGF,YAAY,CAACG,kBAAkB,CAAW,MAAM,CAAC;AAGpE,OAAO,MAAMC,IAAI,GAAG;EAClB,GAAGF,UAAU;EAEbG,SAASA,CAACC,YAAoB,EAAE;IAC9B,MAAM;MAAEC;IAAI,CAAC,GAAGN,KAAK,CAACO,kBAAkB,CAACF,YAAY,CAAC;IACtD,OAAOJ,UAAU,CAACG,SAAS,CAACE,GAAG,CAAC;EAClC;AACF,CAAC","ignoreList":[]}
|
|
@@ -1,5 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
export declare const Yolo: {
|
|
2
|
+
loadModel(modelAssetId: number): import("./specs/yolo.nitro").YoloModel;
|
|
3
|
+
frameToBase64(frame: import("react-native-vision-camera").Frame): string;
|
|
4
|
+
__type?: string;
|
|
5
|
+
name: string;
|
|
6
|
+
toString(): string;
|
|
7
|
+
equals(other: import("react-native-nitro-modules").HybridObject<{
|
|
8
|
+
ios: "swift";
|
|
9
|
+
android: "kotlin";
|
|
10
|
+
}>): boolean;
|
|
11
|
+
dispose(): void;
|
|
4
12
|
};
|
|
13
|
+
export type { Detection, BoundingBox, YoloModel, } from './specs/yolo.nitro';
|
|
5
14
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAOA,eAAO,MAAM,IAAI;4BAGS,MAAM;;;;;;;;;;CAI/B,CAAA;AACD,YAAY,EACV,SAAS,EACT,WAAW,EACX,SAAS,GACV,MAAM,oBAAoB,CAAA"}
|
|
@@ -1,24 +1,28 @@
|
|
|
1
1
|
import type { HybridObject } from 'react-native-nitro-modules';
|
|
2
2
|
import type { Frame } from 'react-native-vision-camera';
|
|
3
|
-
type Detection = {
|
|
3
|
+
export type Detection = {
|
|
4
4
|
classId: number;
|
|
5
5
|
score: number;
|
|
6
6
|
boundingBox: BoundingBox;
|
|
7
7
|
};
|
|
8
|
-
type BoundingBox = {
|
|
8
|
+
export type BoundingBox = {
|
|
9
9
|
x1: number;
|
|
10
10
|
y1: number;
|
|
11
11
|
x2: number;
|
|
12
12
|
y2: number;
|
|
13
13
|
};
|
|
14
|
+
export interface YoloModel extends HybridObject<{
|
|
15
|
+
ios: 'swift';
|
|
16
|
+
android: 'kotlin';
|
|
17
|
+
}> {
|
|
18
|
+
detect(frame: Frame): Detection[];
|
|
19
|
+
close(): void;
|
|
20
|
+
}
|
|
14
21
|
export interface Yolo extends HybridObject<{
|
|
15
22
|
ios: 'swift';
|
|
16
23
|
android: 'kotlin';
|
|
17
24
|
}> {
|
|
18
|
-
|
|
19
|
-
loadModel(modelPath: string): void;
|
|
25
|
+
loadModel(modelPath: string): YoloModel;
|
|
20
26
|
frameToBase64(frame: Frame): string;
|
|
21
|
-
detect(frame: Frame): Detection[];
|
|
22
27
|
}
|
|
23
|
-
export {};
|
|
24
28
|
//# sourceMappingURL=yolo.nitro.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"yolo.nitro.d.ts","sourceRoot":"","sources":["../../../../src/specs/yolo.nitro.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAA;AAEvD,
|
|
1
|
+
{"version":3,"file":"yolo.nitro.d.ts","sourceRoot":"","sources":["../../../../src/specs/yolo.nitro.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,4BAA4B,CAAA;AAEvD,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,WAAW,CAAA;CACzB,CAAA;AACD,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,EAAE,MAAM,CAAA;IACV,EAAE,EAAE,MAAM,CAAA;IACV,EAAE,EAAE,MAAM,CAAA;IACV,EAAE,EAAE,MAAM,CAAA;CACX,CAAA;AAED,MAAM,WAAW,SAAU,SAAQ,YAAY,CAAC;IAC9C,GAAG,EAAE,OAAO,CAAA;IACZ,OAAO,EAAE,QAAQ,CAAA;CAClB,CAAC;IACA,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,SAAS,EAAE,CAAA;IACjC,KAAK,IAAI,IAAI,CAAA;CAEd;AAED,MAAM,WAAW,IAAK,SAAQ,YAAY,CAAC;IACzC,GAAG,EAAE,OAAO,CAAA;IACZ,OAAO,EAAE,QAAQ,CAAA;CAClB,CAAC;IACA,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAA;IACvC,aAAa,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAA;CACpC"}
|
|
@@ -33,8 +33,10 @@ target_sources(
|
|
|
33
33
|
# Autolinking Setup
|
|
34
34
|
../nitrogen/generated/android/YoloOnLoad.cpp
|
|
35
35
|
# Shared Nitrogen C++ sources
|
|
36
|
+
../nitrogen/generated/shared/c++/HybridYoloModelSpec.cpp
|
|
36
37
|
../nitrogen/generated/shared/c++/HybridYoloSpec.cpp
|
|
37
38
|
# Android-specific Nitrogen C++ sources
|
|
39
|
+
../nitrogen/generated/android/c++/JHybridYoloModelSpec.cpp
|
|
38
40
|
../nitrogen/generated/android/c++/JHybridYoloSpec.cpp
|
|
39
41
|
)
|
|
40
42
|
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
#include <fbjni/fbjni.h>
|
|
16
16
|
#include <NitroModules/HybridObjectRegistry.hpp>
|
|
17
17
|
|
|
18
|
+
#include "JHybridYoloModelSpec.hpp"
|
|
18
19
|
#include "JHybridYoloSpec.hpp"
|
|
19
20
|
#include <NitroModules/DefaultConstructableObject.hpp>
|
|
20
21
|
|
|
@@ -40,6 +41,7 @@ void registerAllNatives() {
|
|
|
40
41
|
using namespace margelo::nitro::yolo;
|
|
41
42
|
|
|
42
43
|
// Register native JNI methods
|
|
44
|
+
margelo::nitro::yolo::JHybridYoloModelSpec::CxxPart::registerNatives();
|
|
43
45
|
margelo::nitro::yolo::JHybridYoloSpec::CxxPart::registerNatives();
|
|
44
46
|
|
|
45
47
|
// Register Nitro Hybrid Objects
|