vision-camera-face-detection 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -0
- package/README.md +33 -0
- package/VisionCameraFaceDetection.podspec +45 -0
- package/android/build.gradle +106 -0
- package/android/gradle.properties +6 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/AndroidManifestNew.xml +2 -0
- package/android/src/main/java/com/visioncamerafacedetection/FaceHelper.kt +112 -0
- package/android/src/main/java/com/visioncamerafacedetection/VisionCameraFaceDetectionModule.kt +118 -0
- package/android/src/main/java/com/visioncamerafacedetection/VisionCameraFaceDetectionPackage.kt +25 -0
- package/android/src/main/java/com/visioncamerafacedetection/VisionCameraFaceDetectionPlugin.kt +359 -0
- package/ios/FaceHelper.swift +238 -0
- package/ios/VisionCameraFaceDetection-Bridging-Header.h +6 -0
- package/ios/VisionCameraFaceDetectionModule.mm +19 -0
- package/ios/VisionCameraFaceDetectionModule.swift +105 -0
- package/ios/VisionCameraFaceDetectionPlugin.mm +22 -0
- package/ios/VisionCameraFaceDetectionPlugin.swift +341 -0
- package/lib/commonjs/Camera.cjs +161 -0
- package/lib/commonjs/Camera.cjs.map +1 -0
- package/lib/commonjs/FaceDetector.cjs +42 -0
- package/lib/commonjs/FaceDetector.cjs.map +1 -0
- package/lib/commonjs/Tensor.cjs +24 -0
- package/lib/commonjs/Tensor.cjs.map +1 -0
- package/lib/commonjs/index.cjs +39 -0
- package/lib/commonjs/index.cjs.map +1 -0
- package/lib/module/Camera.mjs +158 -0
- package/lib/module/Camera.mjs.map +1 -0
- package/lib/module/FaceDetector.mjs +36 -0
- package/lib/module/FaceDetector.mjs.map +1 -0
- package/lib/module/Tensor.mjs +17 -0
- package/lib/module/Tensor.mjs.map +1 -0
- package/lib/module/index.mjs +4 -0
- package/lib/module/index.mjs.map +1 -0
- package/lib/typescript/src/Camera.d.ts +17 -0
- package/lib/typescript/src/Camera.d.ts.map +1 -0
- package/lib/typescript/src/FaceDetector.d.ts +118 -0
- package/lib/typescript/src/FaceDetector.d.ts.map +1 -0
- package/lib/typescript/src/Tensor.d.ts +3 -0
- package/lib/typescript/src/Tensor.d.ts.map +1 -0
- package/lib/typescript/src/index.d.ts +4 -0
- package/lib/typescript/src/index.d.ts.map +1 -0
- package/package.json +186 -0
- package/src/Camera.tsx +192 -0
- package/src/FaceDetector.ts +161 -0
- package/src/Tensor.ts +27 -0
- package/src/index.tsx +3 -0
package/android/src/main/java/com/visioncamerafacedetection/VisionCameraFaceDetectionPlugin.kt
ADDED
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
package com.visioncamerafacedetection
|
|
2
|
+
|
|
3
|
+
import android.graphics.Bitmap
|
|
4
|
+
import android.graphics.Canvas
|
|
5
|
+
import android.graphics.Matrix
|
|
6
|
+
import android.graphics.Rect
|
|
7
|
+
import android.graphics.RectF
|
|
8
|
+
import android.util.Log
|
|
9
|
+
import com.google.android.gms.tasks.Tasks
|
|
10
|
+
import com.google.mlkit.vision.common.InputImage
|
|
11
|
+
import com.google.mlkit.vision.common.internal.ImageConvertUtils
|
|
12
|
+
import com.google.mlkit.vision.face.Face
|
|
13
|
+
import com.google.mlkit.vision.face.FaceContour
|
|
14
|
+
import com.google.mlkit.vision.face.FaceDetection
|
|
15
|
+
import com.google.mlkit.vision.face.FaceDetector
|
|
16
|
+
import com.google.mlkit.vision.face.FaceDetectorOptions
|
|
17
|
+
import com.google.mlkit.vision.face.FaceLandmark
|
|
18
|
+
import com.mrousavy.camera.core.FrameInvalidError
|
|
19
|
+
import com.mrousavy.camera.core.types.Orientation
|
|
20
|
+
import com.mrousavy.camera.frameprocessors.Frame
|
|
21
|
+
import com.mrousavy.camera.frameprocessors.FrameProcessorPlugin
|
|
22
|
+
import com.mrousavy.camera.frameprocessors.VisionCameraProxy
|
|
23
|
+
import java.nio.ByteBuffer
|
|
24
|
+
import java.nio.FloatBuffer
|
|
25
|
+
|
|
26
|
+
private const val TAG = "FaceDetector"
|
|
27
|
+
|
|
28
|
+
class VisionCameraFaceDetectionPlugin(
|
|
29
|
+
proxy: VisionCameraProxy,
|
|
30
|
+
options: Map<String, Any>?
|
|
31
|
+
) : FrameProcessorPlugin() {
|
|
32
|
+
// device display data
|
|
33
|
+
private val displayMetrics = proxy.context.resources.displayMetrics
|
|
34
|
+
private val density = displayMetrics.density
|
|
35
|
+
private val windowWidth = (displayMetrics.widthPixels).toDouble() / density
|
|
36
|
+
private val windowHeight = (displayMetrics.heightPixels).toDouble() / density
|
|
37
|
+
|
|
38
|
+
// detection props
|
|
39
|
+
private var autoScale = false
|
|
40
|
+
private var faceDetector: FaceDetector? = null
|
|
41
|
+
private var runLandmarks = false
|
|
42
|
+
private var runClassifications = false
|
|
43
|
+
private var runContours = false
|
|
44
|
+
private var trackingEnabled = false
|
|
45
|
+
|
|
46
|
+
init {
|
|
47
|
+
// handle auto scaling
|
|
48
|
+
autoScale = options?.get("autoScale").toString() == "true"
|
|
49
|
+
|
|
50
|
+
// initializes faceDetector on creation
|
|
51
|
+
var performanceModeValue = FaceDetectorOptions.PERFORMANCE_MODE_FAST
|
|
52
|
+
var landmarkModeValue = FaceDetectorOptions.LANDMARK_MODE_NONE
|
|
53
|
+
var classificationModeValue = FaceDetectorOptions.CLASSIFICATION_MODE_NONE
|
|
54
|
+
var contourModeValue = FaceDetectorOptions.CONTOUR_MODE_NONE
|
|
55
|
+
var minFaceSize = 0.15f
|
|
56
|
+
|
|
57
|
+
if (options?.get("performanceMode").toString() == "accurate") {
|
|
58
|
+
performanceModeValue = FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if (options?.get("landmarkMode").toString() == "all") {
|
|
62
|
+
runLandmarks = true
|
|
63
|
+
landmarkModeValue = FaceDetectorOptions.LANDMARK_MODE_ALL
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (options?.get("classificationMode").toString() == "all") {
|
|
67
|
+
runClassifications = true
|
|
68
|
+
classificationModeValue = FaceDetectorOptions.CLASSIFICATION_MODE_ALL
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (options?.get("contourMode").toString() == "all") {
|
|
72
|
+
runContours = true
|
|
73
|
+
contourModeValue = FaceDetectorOptions.CONTOUR_MODE_ALL
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
val minFaceSizeParam = options?.get("minFaceSize").toString()
|
|
77
|
+
if (
|
|
78
|
+
minFaceSizeParam != "null" &&
|
|
79
|
+
minFaceSizeParam != minFaceSize.toString()
|
|
80
|
+
) {
|
|
81
|
+
minFaceSize = minFaceSizeParam.toFloat()
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
val optionsBuilder = FaceDetectorOptions.Builder()
|
|
85
|
+
.setPerformanceMode(performanceModeValue)
|
|
86
|
+
.setLandmarkMode(landmarkModeValue)
|
|
87
|
+
.setContourMode(contourModeValue)
|
|
88
|
+
.setClassificationMode(classificationModeValue)
|
|
89
|
+
.setMinFaceSize(minFaceSize)
|
|
90
|
+
|
|
91
|
+
if (options?.get("trackingEnabled").toString() == "true") {
|
|
92
|
+
trackingEnabled = true
|
|
93
|
+
optionsBuilder.enableTracking()
|
|
94
|
+
}
|
|
95
|
+
faceDetector = FaceDetection.getClient(
|
|
96
|
+
optionsBuilder.build()
|
|
97
|
+
)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
private fun processBoundingBox(
|
|
101
|
+
boundingBox: Rect,
|
|
102
|
+
sourceWidth: Double,
|
|
103
|
+
sourceHeight: Double,
|
|
104
|
+
orientation: Orientation,
|
|
105
|
+
scaleX: Double,
|
|
106
|
+
scaleY: Double
|
|
107
|
+
): Map<String, Any> {
|
|
108
|
+
val bounds: MutableMap<String, Any> = HashMap()
|
|
109
|
+
val width = boundingBox.width().toDouble() * scaleX
|
|
110
|
+
val height = boundingBox.height().toDouble() * scaleY
|
|
111
|
+
val x = boundingBox.left.toDouble() * scaleX
|
|
112
|
+
val y = boundingBox.top.toDouble() * scaleY
|
|
113
|
+
|
|
114
|
+
when (orientation) {
|
|
115
|
+
Orientation.PORTRAIT -> {
|
|
116
|
+
// device is landscape left
|
|
117
|
+
bounds["x"] = (-y + sourceWidth * scaleX) - width
|
|
118
|
+
bounds["y"] = (-x + sourceHeight * scaleY) - height
|
|
119
|
+
}
|
|
120
|
+
Orientation.LANDSCAPE_LEFT -> {
|
|
121
|
+
// device is portrait
|
|
122
|
+
bounds["x"] = (-x + sourceWidth * scaleX) - width
|
|
123
|
+
bounds["y"] = y
|
|
124
|
+
}
|
|
125
|
+
Orientation.PORTRAIT_UPSIDE_DOWN -> {
|
|
126
|
+
// device is landscape right
|
|
127
|
+
bounds["x"] = y
|
|
128
|
+
bounds["y"] = x
|
|
129
|
+
}
|
|
130
|
+
Orientation.LANDSCAPE_RIGHT -> {
|
|
131
|
+
// device is upside down
|
|
132
|
+
bounds["x"] = x
|
|
133
|
+
bounds["y"] = (-y + sourceHeight * scaleY) - height
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
bounds["width"] = width
|
|
137
|
+
bounds["height"] = height
|
|
138
|
+
return bounds
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private fun processLandmarks(
|
|
142
|
+
face: Face,
|
|
143
|
+
scaleX: Double,
|
|
144
|
+
scaleY: Double
|
|
145
|
+
): Map<String, Any> {
|
|
146
|
+
val faceLandmarksTypes = intArrayOf(
|
|
147
|
+
FaceLandmark.LEFT_CHEEK,
|
|
148
|
+
FaceLandmark.LEFT_EAR,
|
|
149
|
+
FaceLandmark.LEFT_EYE,
|
|
150
|
+
FaceLandmark.MOUTH_BOTTOM,
|
|
151
|
+
FaceLandmark.MOUTH_LEFT,
|
|
152
|
+
FaceLandmark.MOUTH_RIGHT,
|
|
153
|
+
FaceLandmark.NOSE_BASE,
|
|
154
|
+
FaceLandmark.RIGHT_CHEEK,
|
|
155
|
+
FaceLandmark.RIGHT_EAR,
|
|
156
|
+
FaceLandmark.RIGHT_EYE
|
|
157
|
+
)
|
|
158
|
+
val faceLandmarksTypesStrings = arrayOf(
|
|
159
|
+
"LEFT_CHEEK",
|
|
160
|
+
"LEFT_EAR",
|
|
161
|
+
"LEFT_EYE",
|
|
162
|
+
"MOUTH_BOTTOM",
|
|
163
|
+
"MOUTH_LEFT",
|
|
164
|
+
"MOUTH_RIGHT",
|
|
165
|
+
"NOSE_BASE",
|
|
166
|
+
"RIGHT_CHEEK",
|
|
167
|
+
"RIGHT_EAR",
|
|
168
|
+
"RIGHT_EYE"
|
|
169
|
+
)
|
|
170
|
+
val faceLandmarksTypesMap: MutableMap<String, Any> = HashMap()
|
|
171
|
+
for (i in faceLandmarksTypesStrings.indices) {
|
|
172
|
+
val landmark = face.getLandmark(faceLandmarksTypes[i])
|
|
173
|
+
val landmarkName = faceLandmarksTypesStrings[i]
|
|
174
|
+
Log.d(
|
|
175
|
+
TAG,
|
|
176
|
+
"Getting '$landmarkName' landmark"
|
|
177
|
+
)
|
|
178
|
+
if (landmark == null) {
|
|
179
|
+
Log.d(
|
|
180
|
+
TAG,
|
|
181
|
+
"Landmark '$landmarkName' is null - going next"
|
|
182
|
+
)
|
|
183
|
+
continue
|
|
184
|
+
}
|
|
185
|
+
val point = landmark.position
|
|
186
|
+
val currentPointsMap: MutableMap<String, Double> = HashMap()
|
|
187
|
+
currentPointsMap["x"] = point.x.toDouble() * scaleX
|
|
188
|
+
currentPointsMap["y"] = point.y.toDouble() * scaleY
|
|
189
|
+
faceLandmarksTypesMap[landmarkName] = currentPointsMap
|
|
190
|
+
}
|
|
191
|
+
return faceLandmarksTypesMap
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
private fun processFaceContours(
|
|
195
|
+
face: Face,
|
|
196
|
+
scaleX: Double,
|
|
197
|
+
scaleY: Double
|
|
198
|
+
): Map<String, Any> {
|
|
199
|
+
val faceContoursTypes = intArrayOf(
|
|
200
|
+
FaceContour.FACE,
|
|
201
|
+
FaceContour.LEFT_CHEEK,
|
|
202
|
+
FaceContour.LEFT_EYE,
|
|
203
|
+
FaceContour.LEFT_EYEBROW_BOTTOM,
|
|
204
|
+
FaceContour.LEFT_EYEBROW_TOP,
|
|
205
|
+
FaceContour.LOWER_LIP_BOTTOM,
|
|
206
|
+
FaceContour.LOWER_LIP_TOP,
|
|
207
|
+
FaceContour.NOSE_BOTTOM,
|
|
208
|
+
FaceContour.NOSE_BRIDGE,
|
|
209
|
+
FaceContour.RIGHT_CHEEK,
|
|
210
|
+
FaceContour.RIGHT_EYE,
|
|
211
|
+
FaceContour.RIGHT_EYEBROW_BOTTOM,
|
|
212
|
+
FaceContour.RIGHT_EYEBROW_TOP,
|
|
213
|
+
FaceContour.UPPER_LIP_BOTTOM,
|
|
214
|
+
FaceContour.UPPER_LIP_TOP
|
|
215
|
+
)
|
|
216
|
+
val faceContoursTypesStrings = arrayOf(
|
|
217
|
+
"FACE",
|
|
218
|
+
"LEFT_CHEEK",
|
|
219
|
+
"LEFT_EYE",
|
|
220
|
+
"LEFT_EYEBROW_BOTTOM",
|
|
221
|
+
"LEFT_EYEBROW_TOP",
|
|
222
|
+
"LOWER_LIP_BOTTOM",
|
|
223
|
+
"LOWER_LIP_TOP",
|
|
224
|
+
"NOSE_BOTTOM",
|
|
225
|
+
"NOSE_BRIDGE",
|
|
226
|
+
"RIGHT_CHEEK",
|
|
227
|
+
"RIGHT_EYE",
|
|
228
|
+
"RIGHT_EYEBROW_BOTTOM",
|
|
229
|
+
"RIGHT_EYEBROW_TOP",
|
|
230
|
+
"UPPER_LIP_BOTTOM",
|
|
231
|
+
"UPPER_LIP_TOP"
|
|
232
|
+
)
|
|
233
|
+
val faceContoursTypesMap: MutableMap<String, Any> = HashMap()
|
|
234
|
+
for (i in faceContoursTypesStrings.indices) {
|
|
235
|
+
val contour = face.getContour(faceContoursTypes[i])
|
|
236
|
+
val contourName = faceContoursTypesStrings[i]
|
|
237
|
+
Log.d(
|
|
238
|
+
TAG,
|
|
239
|
+
"Getting '$contourName' contour"
|
|
240
|
+
)
|
|
241
|
+
if (contour == null) {
|
|
242
|
+
Log.d(
|
|
243
|
+
TAG,
|
|
244
|
+
"Face contour '$contourName' is null - going next"
|
|
245
|
+
)
|
|
246
|
+
continue
|
|
247
|
+
}
|
|
248
|
+
val points = contour.points
|
|
249
|
+
val pointsMap: MutableMap<String, Map<String, Double>> = HashMap()
|
|
250
|
+
for (j in points.indices) {
|
|
251
|
+
val currentPointsMap: MutableMap<String, Double> = HashMap()
|
|
252
|
+
currentPointsMap["x"] = points[j].x.toDouble() * scaleX
|
|
253
|
+
currentPointsMap["y"] = points[j].y.toDouble() * scaleY
|
|
254
|
+
pointsMap[j.toString()] = currentPointsMap
|
|
255
|
+
}
|
|
256
|
+
faceContoursTypesMap[contourName] = pointsMap
|
|
257
|
+
}
|
|
258
|
+
return faceContoursTypesMap
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
private fun getOrientation(
|
|
262
|
+
orientation: Orientation
|
|
263
|
+
): Int {
|
|
264
|
+
return when (orientation) {
|
|
265
|
+
// device is landscape left
|
|
266
|
+
Orientation.PORTRAIT -> 0
|
|
267
|
+
// device is portrait
|
|
268
|
+
Orientation.LANDSCAPE_LEFT -> 270
|
|
269
|
+
// device is landscape right
|
|
270
|
+
Orientation.PORTRAIT_UPSIDE_DOWN -> 180
|
|
271
|
+
// device is upside-down
|
|
272
|
+
Orientation.LANDSCAPE_RIGHT -> 90
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
override fun callback(
|
|
277
|
+
frame: Frame,
|
|
278
|
+
params: Map<String, Any>?
|
|
279
|
+
): Any {
|
|
280
|
+
val result = ArrayList<Map<String, Any>>()
|
|
281
|
+
try {
|
|
282
|
+
val orientation = getOrientation(frame.orientation)
|
|
283
|
+
val image = InputImage.fromMediaImage(frame.image, orientation)
|
|
284
|
+
// we need to invert sizes as frame is always -90deg rotated
|
|
285
|
+
val width = image.height.toDouble()
|
|
286
|
+
val height = image.width.toDouble()
|
|
287
|
+
val scaleX = if (autoScale) windowWidth / width else 1.0
|
|
288
|
+
val scaleY = if (autoScale) windowHeight / height else 1.0
|
|
289
|
+
val task = faceDetector!!.process(image)
|
|
290
|
+
val faces = Tasks.await(task)
|
|
291
|
+
faces.forEach { face ->
|
|
292
|
+
val bmpFrameResult = ImageConvertUtils.getInstance().getUpRightBitmap(image)
|
|
293
|
+
val bmpFaceResult =
|
|
294
|
+
Bitmap.createBitmap(
|
|
295
|
+
TF_OD_API_INPUT_SIZE,
|
|
296
|
+
TF_OD_API_INPUT_SIZE,
|
|
297
|
+
Bitmap.Config.ARGB_8888
|
|
298
|
+
)
|
|
299
|
+
val faceBB = RectF(face.boundingBox)
|
|
300
|
+
val cvFace = Canvas(bmpFaceResult)
|
|
301
|
+
val sx = TF_OD_API_INPUT_SIZE.toFloat() / faceBB.width()
|
|
302
|
+
val sy = TF_OD_API_INPUT_SIZE.toFloat() / faceBB.height()
|
|
303
|
+
val matrix = Matrix()
|
|
304
|
+
matrix.postTranslate(-faceBB.left, -faceBB.top)
|
|
305
|
+
matrix.postScale(sx, sy)
|
|
306
|
+
cvFace.drawBitmap(bmpFrameResult, matrix, null)
|
|
307
|
+
val input: ByteBuffer = FaceHelper().bitmap2ByteBuffer(bmpFaceResult)
|
|
308
|
+
val output: FloatBuffer = FloatBuffer.allocate(192)
|
|
309
|
+
interpreter?.run(input, output)
|
|
310
|
+
|
|
311
|
+
val arrayData: MutableList<Any> = ArrayList()
|
|
312
|
+
for (i: Float in output.array()) {
|
|
313
|
+
arrayData.add(i.toDouble())
|
|
314
|
+
}
|
|
315
|
+
val map: MutableMap<String, Any> = HashMap()
|
|
316
|
+
if (runLandmarks) {
|
|
317
|
+
map["landmarks"] = processLandmarks(
|
|
318
|
+
face,
|
|
319
|
+
scaleX,
|
|
320
|
+
scaleY
|
|
321
|
+
)
|
|
322
|
+
}
|
|
323
|
+
if (runClassifications) {
|
|
324
|
+
map["leftEyeOpenProbability"] = face.leftEyeOpenProbability?.toDouble() ?: -1
|
|
325
|
+
map["rightEyeOpenProbability"] = face.rightEyeOpenProbability?.toDouble() ?: -1
|
|
326
|
+
map["smilingProbability"] = face.smilingProbability?.toDouble() ?: -1
|
|
327
|
+
}
|
|
328
|
+
if (runContours) {
|
|
329
|
+
map["contours"] = processFaceContours(
|
|
330
|
+
face,
|
|
331
|
+
scaleX,
|
|
332
|
+
scaleY
|
|
333
|
+
)
|
|
334
|
+
}
|
|
335
|
+
if (trackingEnabled) {
|
|
336
|
+
map["trackingId"] = face.trackingId ?: -1
|
|
337
|
+
}
|
|
338
|
+
map["rollAngle"] = face.headEulerAngleZ.toDouble()
|
|
339
|
+
map["pitchAngle"] = face.headEulerAngleX.toDouble()
|
|
340
|
+
map["yawAngle"] = face.headEulerAngleY.toDouble()
|
|
341
|
+
map["bounds"] = processBoundingBox(
|
|
342
|
+
face.boundingBox,
|
|
343
|
+
width,
|
|
344
|
+
height,
|
|
345
|
+
frame.orientation,
|
|
346
|
+
scaleX,
|
|
347
|
+
scaleY
|
|
348
|
+
)
|
|
349
|
+
map["data"] = arrayData
|
|
350
|
+
result.add(map)
|
|
351
|
+
}
|
|
352
|
+
} catch (e: Exception) {
|
|
353
|
+
Log.e(TAG, "Error processing face detection: ", e)
|
|
354
|
+
} catch (e: FrameInvalidError) {
|
|
355
|
+
Log.e(TAG, "Frame invalid error: ", e)
|
|
356
|
+
}
|
|
357
|
+
return result
|
|
358
|
+
}
|
|
359
|
+
}
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import VisionCamera
|
|
2
|
+
import MLKitFaceDetection
|
|
3
|
+
import MLKitVision
|
|
4
|
+
import CoreML
|
|
5
|
+
import UIKit
|
|
6
|
+
import AVFoundation
|
|
7
|
+
import Accelerate
|
|
8
|
+
import TensorFlowLite
|
|
9
|
+
|
|
10
|
+
let batchSize = 1
|
|
11
|
+
let inputChannels = 1
|
|
12
|
+
let inputWidth = 112
|
|
13
|
+
let inputHeight = 112
|
|
14
|
+
|
|
15
|
+
// TensorFlow Lite `Interpreter` object for performing inference on a given model.
|
|
16
|
+
var interpreter: Interpreter? = nil
|
|
17
|
+
|
|
18
|
+
class FaceHelper {
|
|
19
|
+
static func processContours(from face: Face) -> [String:[[String:CGFloat]]] {
|
|
20
|
+
let faceContoursTypes = [
|
|
21
|
+
FaceContourType.face,
|
|
22
|
+
FaceContourType.leftEyebrowTop,
|
|
23
|
+
FaceContourType.leftEyebrowBottom,
|
|
24
|
+
FaceContourType.rightEyebrowTop,
|
|
25
|
+
FaceContourType.rightEyebrowBottom,
|
|
26
|
+
FaceContourType.leftEye,
|
|
27
|
+
FaceContourType.rightEye,
|
|
28
|
+
FaceContourType.upperLipTop,
|
|
29
|
+
FaceContourType.upperLipBottom,
|
|
30
|
+
FaceContourType.lowerLipTop,
|
|
31
|
+
FaceContourType.lowerLipBottom,
|
|
32
|
+
FaceContourType.noseBridge,
|
|
33
|
+
FaceContourType.noseBottom,
|
|
34
|
+
FaceContourType.leftCheek,
|
|
35
|
+
FaceContourType.rightCheek,
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
let faceContoursTypesStrings = [
|
|
39
|
+
"FACE",
|
|
40
|
+
"LEFT_EYEBROW_TOP",
|
|
41
|
+
"LEFT_EYEBROW_BOTTOM",
|
|
42
|
+
"RIGHT_EYEBROW_TOP",
|
|
43
|
+
"RIGHT_EYEBROW_BOTTOM",
|
|
44
|
+
"LEFT_EYE",
|
|
45
|
+
"RIGHT_EYE",
|
|
46
|
+
"UPPER_LIP_TOP",
|
|
47
|
+
"UPPER_LIP_BOTTOM",
|
|
48
|
+
"LOWER_LIP_TOP",
|
|
49
|
+
"LOWER_LIP_BOTTOM",
|
|
50
|
+
"NOSE_BRIDGE",
|
|
51
|
+
"NOSE_BOTTOM",
|
|
52
|
+
"LEFT_CHEEK",
|
|
53
|
+
"RIGHT_CHEEK",
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
var faceContoursTypesMap: [String:[[String:CGFloat]]] = [:]
|
|
57
|
+
|
|
58
|
+
for i in 0..<faceContoursTypes.count {
|
|
59
|
+
let contour = face.contour(ofType: faceContoursTypes[i]);
|
|
60
|
+
|
|
61
|
+
var pointsArray: [[String:CGFloat]] = []
|
|
62
|
+
|
|
63
|
+
if let points = contour?.points {
|
|
64
|
+
for point in points {
|
|
65
|
+
let currentPointsMap = [
|
|
66
|
+
"x": point.x,
|
|
67
|
+
"y": point.y,
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
pointsArray.append(currentPointsMap)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
faceContoursTypesMap[faceContoursTypesStrings[i]] = pointsArray
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return faceContoursTypesMap
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
static func processBoundingBox(from face: Face) -> [String:Any] {
|
|
81
|
+
let frameRect = face.frame
|
|
82
|
+
|
|
83
|
+
let offsetX = (frameRect.midX - ceil(frameRect.width)) / 2.0
|
|
84
|
+
let offsetY = (frameRect.midY - ceil(frameRect.height)) / 2.0
|
|
85
|
+
|
|
86
|
+
let x = frameRect.maxX + offsetX
|
|
87
|
+
let y = frameRect.minY + offsetY
|
|
88
|
+
|
|
89
|
+
return [
|
|
90
|
+
"x": frameRect.midX + (frameRect.midX - x),
|
|
91
|
+
"y": frameRect.midY + (y - frameRect.midY),
|
|
92
|
+
"width": frameRect.width,
|
|
93
|
+
"height": frameRect.height,
|
|
94
|
+
"boundingCenterX": frameRect.midX,
|
|
95
|
+
"boundingCenterY": frameRect.midY
|
|
96
|
+
]
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
static func getImageFaceFromUIImage(from image: UIImage, rectImage: CGRect) -> UIImage? {
|
|
100
|
+
let imageRef: CGImage = (image.cgImage?.cropping(to: rectImage)!)!
|
|
101
|
+
let imageCrop: UIImage = UIImage(cgImage: imageRef, scale: 0.5, orientation: image.imageOrientation)
|
|
102
|
+
return imageCrop
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
static func convertImageToBase64(image: UIImage) -> String {
|
|
106
|
+
let imageData = image.pngData()!
|
|
107
|
+
return imageData.base64EncodedString()
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
static func rgbDataFromBuffer(
|
|
111
|
+
_ buffer: CVPixelBuffer,
|
|
112
|
+
byteCount: Int,
|
|
113
|
+
isModelQuantized: Bool
|
|
114
|
+
) -> Data? {
|
|
115
|
+
CVPixelBufferLockBaseAddress(buffer, .readOnly)
|
|
116
|
+
defer {
|
|
117
|
+
CVPixelBufferUnlockBaseAddress(buffer, .readOnly)
|
|
118
|
+
}
|
|
119
|
+
guard let sourceData = CVPixelBufferGetBaseAddress(buffer) else {
|
|
120
|
+
return nil
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
let width = CVPixelBufferGetWidth(buffer)
|
|
124
|
+
let height = CVPixelBufferGetHeight(buffer)
|
|
125
|
+
let sourceBytesPerRow = CVPixelBufferGetBytesPerRow(buffer)
|
|
126
|
+
let destinationChannelCount = 3
|
|
127
|
+
let destinationBytesPerRow = destinationChannelCount * width
|
|
128
|
+
|
|
129
|
+
var sourceBuffer = vImage_Buffer(data: sourceData,
|
|
130
|
+
height: vImagePixelCount(height),
|
|
131
|
+
width: vImagePixelCount(width),
|
|
132
|
+
rowBytes: sourceBytesPerRow)
|
|
133
|
+
|
|
134
|
+
guard let destinationData = malloc(height * destinationBytesPerRow) else {
|
|
135
|
+
print("Error: out of memory")
|
|
136
|
+
return nil
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
defer {
|
|
140
|
+
free(destinationData)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
var destinationBuffer = vImage_Buffer(data: destinationData,
|
|
144
|
+
height: vImagePixelCount(height),
|
|
145
|
+
width: vImagePixelCount(width),
|
|
146
|
+
rowBytes: destinationBytesPerRow)
|
|
147
|
+
|
|
148
|
+
let pixelBufferFormat = CVPixelBufferGetPixelFormatType(buffer)
|
|
149
|
+
|
|
150
|
+
switch (pixelBufferFormat) {
|
|
151
|
+
case kCVPixelFormatType_32BGRA:
|
|
152
|
+
vImageConvert_BGRA8888toRGB888(&sourceBuffer, &destinationBuffer, UInt32(kvImageNoFlags))
|
|
153
|
+
case kCVPixelFormatType_32ARGB:
|
|
154
|
+
vImageConvert_ARGB8888toRGB888(&sourceBuffer, &destinationBuffer, UInt32(kvImageNoFlags))
|
|
155
|
+
case kCVPixelFormatType_32RGBA:
|
|
156
|
+
vImageConvert_RGBA8888toRGB888(&sourceBuffer, &destinationBuffer, UInt32(kvImageNoFlags))
|
|
157
|
+
default:
|
|
158
|
+
// Unknown pixel format.
|
|
159
|
+
return nil
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
let byteData = Data(bytes: destinationBuffer.data, count: destinationBuffer.rowBytes * height)
|
|
163
|
+
if isModelQuantized {
|
|
164
|
+
return byteData
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Not quantized, convert to floats
|
|
168
|
+
let bytes = Array<UInt8>(unsafeData: byteData)!
|
|
169
|
+
var floats = [Float]()
|
|
170
|
+
for i in 0..<bytes.count {
|
|
171
|
+
floats.append(Float(bytes[i]) / 255.0)
|
|
172
|
+
}
|
|
173
|
+
return Data(copyingBufferOf: floats)
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
static func uiImageToPixelBuffer(image: UIImage, size: Int) -> CVPixelBuffer? {
|
|
177
|
+
let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue, kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
|
|
178
|
+
var pixelBuffer : CVPixelBuffer?
|
|
179
|
+
let status = CVPixelBufferCreate(kCFAllocatorDefault, size, size, kCVPixelFormatType_32ARGB, attrs, &pixelBuffer)
|
|
180
|
+
guard (status == kCVReturnSuccess) else {
|
|
181
|
+
return nil
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
CVPixelBufferLockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
|
|
185
|
+
let pixelData = CVPixelBufferGetBaseAddress(pixelBuffer!)
|
|
186
|
+
|
|
187
|
+
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
|
|
188
|
+
let context = CGContext(data: pixelData, width: size, height: size, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer!), space: rgbColorSpace, bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue)
|
|
189
|
+
|
|
190
|
+
context?.translateBy(x: 0, y: CGFloat(size))
|
|
191
|
+
context?.scaleBy(x: 1.0, y: -1.0)
|
|
192
|
+
|
|
193
|
+
UIGraphicsPushContext(context!)
|
|
194
|
+
image.draw(in: CGRect(x: 0, y: 0, width: size, height: size))
|
|
195
|
+
UIGraphicsPopContext()
|
|
196
|
+
CVPixelBufferUnlockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
|
|
197
|
+
return pixelBuffer
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
static func getImageFaceFromBuffer(from sampleBuffer: CMSampleBuffer?, rectImage: CGRect) -> UIImage? {
|
|
201
|
+
guard let sampleBuffer = sampleBuffer else {
|
|
202
|
+
print("Sample buffer is NULL.")
|
|
203
|
+
return nil
|
|
204
|
+
}
|
|
205
|
+
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
|
|
206
|
+
print("Invalid sample buffer.")
|
|
207
|
+
return nil
|
|
208
|
+
}
|
|
209
|
+
let ciimage = CIImage(cvPixelBuffer: imageBuffer)
|
|
210
|
+
let context = CIContext(options: nil)
|
|
211
|
+
let cgImage = context.createCGImage(ciimage, from: ciimage.extent)!
|
|
212
|
+
|
|
213
|
+
if (!rectImage.isNull) {
|
|
214
|
+
let imageRef: CGImage = cgImage.cropping(to: rectImage)!
|
|
215
|
+
let imageCrop: UIImage = UIImage(cgImage: imageRef, scale: 0.5, orientation: .up)
|
|
216
|
+
return imageCrop
|
|
217
|
+
} else {
|
|
218
|
+
return nil
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// MARK: - Extensions
|
|
224
|
+
extension Data {
|
|
225
|
+
// Creates a new buffer by copying the buffer pointer of the given array.
|
|
226
|
+
init<T>(copyingBufferOf array: [T]) {
|
|
227
|
+
self = array.withUnsafeBufferPointer(Data.init)
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
extension Array {
|
|
232
|
+
// Creates a new array from the bytes of the given unsafe data.
|
|
233
|
+
init?(unsafeData: Data) {
|
|
234
|
+
guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
|
|
235
|
+
self = unsafeData.withUnsafeBytes { .init($0.bindMemory(to: Element.self)) }
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#import <React/RCTBridgeModule.h>
|
|
2
|
+
|
|
3
|
+
@interface RCT_EXTERN_MODULE(VisionCameraFaceDetectionModule, NSObject)
|
|
4
|
+
|
|
5
|
+
RCT_EXTERN_METHOD(detectFromBase64:(NSString)imageString
|
|
6
|
+
withResolver:(RCTPromiseResolveBlock)resolve
|
|
7
|
+
withRejecter:(RCTPromiseRejectBlock)reject)
|
|
8
|
+
|
|
9
|
+
RCT_EXTERN_METHOD(initTensor:(NSString)modelName
|
|
10
|
+
withCount:(NSNumber)count
|
|
11
|
+
withResolver:(RCTPromiseResolveBlock)resolve
|
|
12
|
+
withRejecter:(RCTPromiseRejectBlock)reject)
|
|
13
|
+
|
|
14
|
+
+ (BOOL)requiresMainQueueSetup
|
|
15
|
+
{
|
|
16
|
+
return NO;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
@end
|