vision-camera-face-detection 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/LICENSE +20 -0
  2. package/README.md +33 -0
  3. package/VisionCameraFaceDetection.podspec +45 -0
  4. package/android/build.gradle +106 -0
  5. package/android/gradle.properties +6 -0
  6. package/android/src/main/AndroidManifest.xml +3 -0
  7. package/android/src/main/AndroidManifestNew.xml +2 -0
  8. package/android/src/main/java/com/visioncamerafacedetection/FaceHelper.kt +112 -0
  9. package/android/src/main/java/com/visioncamerafacedetection/VisionCameraFaceDetectionModule.kt +118 -0
  10. package/android/src/main/java/com/visioncamerafacedetection/VisionCameraFaceDetectionPackage.kt +25 -0
  11. package/android/src/main/java/com/visioncamerafacedetection/VisionCameraFaceDetectionPlugin.kt +359 -0
  12. package/ios/FaceHelper.swift +238 -0
  13. package/ios/VisionCameraFaceDetection-Bridging-Header.h +6 -0
  14. package/ios/VisionCameraFaceDetectionModule.mm +19 -0
  15. package/ios/VisionCameraFaceDetectionModule.swift +105 -0
  16. package/ios/VisionCameraFaceDetectionPlugin.mm +22 -0
  17. package/ios/VisionCameraFaceDetectionPlugin.swift +341 -0
  18. package/lib/commonjs/Camera.cjs +161 -0
  19. package/lib/commonjs/Camera.cjs.map +1 -0
  20. package/lib/commonjs/FaceDetector.cjs +42 -0
  21. package/lib/commonjs/FaceDetector.cjs.map +1 -0
  22. package/lib/commonjs/Tensor.cjs +24 -0
  23. package/lib/commonjs/Tensor.cjs.map +1 -0
  24. package/lib/commonjs/index.cjs +39 -0
  25. package/lib/commonjs/index.cjs.map +1 -0
  26. package/lib/module/Camera.mjs +158 -0
  27. package/lib/module/Camera.mjs.map +1 -0
  28. package/lib/module/FaceDetector.mjs +36 -0
  29. package/lib/module/FaceDetector.mjs.map +1 -0
  30. package/lib/module/Tensor.mjs +17 -0
  31. package/lib/module/Tensor.mjs.map +1 -0
  32. package/lib/module/index.mjs +4 -0
  33. package/lib/module/index.mjs.map +1 -0
  34. package/lib/typescript/src/Camera.d.ts +17 -0
  35. package/lib/typescript/src/Camera.d.ts.map +1 -0
  36. package/lib/typescript/src/FaceDetector.d.ts +118 -0
  37. package/lib/typescript/src/FaceDetector.d.ts.map +1 -0
  38. package/lib/typescript/src/Tensor.d.ts +3 -0
  39. package/lib/typescript/src/Tensor.d.ts.map +1 -0
  40. package/lib/typescript/src/index.d.ts +4 -0
  41. package/lib/typescript/src/index.d.ts.map +1 -0
  42. package/package.json +186 -0
  43. package/src/Camera.tsx +192 -0
  44. package/src/FaceDetector.ts +161 -0
  45. package/src/Tensor.ts +27 -0
  46. package/src/index.tsx +3 -0
@@ -0,0 +1,359 @@
1
+ package com.visioncamerafacedetection
2
+
3
+ import android.graphics.Bitmap
4
+ import android.graphics.Canvas
5
+ import android.graphics.Matrix
6
+ import android.graphics.Rect
7
+ import android.graphics.RectF
8
+ import android.util.Log
9
+ import com.google.android.gms.tasks.Tasks
10
+ import com.google.mlkit.vision.common.InputImage
11
+ import com.google.mlkit.vision.common.internal.ImageConvertUtils
12
+ import com.google.mlkit.vision.face.Face
13
+ import com.google.mlkit.vision.face.FaceContour
14
+ import com.google.mlkit.vision.face.FaceDetection
15
+ import com.google.mlkit.vision.face.FaceDetector
16
+ import com.google.mlkit.vision.face.FaceDetectorOptions
17
+ import com.google.mlkit.vision.face.FaceLandmark
18
+ import com.mrousavy.camera.core.FrameInvalidError
19
+ import com.mrousavy.camera.core.types.Orientation
20
+ import com.mrousavy.camera.frameprocessors.Frame
21
+ import com.mrousavy.camera.frameprocessors.FrameProcessorPlugin
22
+ import com.mrousavy.camera.frameprocessors.VisionCameraProxy
23
+ import java.nio.ByteBuffer
24
+ import java.nio.FloatBuffer
25
+
26
+ private const val TAG = "FaceDetector"
27
+
28
+ class VisionCameraFaceDetectionPlugin(
29
+ proxy: VisionCameraProxy,
30
+ options: Map<String, Any>?
31
+ ) : FrameProcessorPlugin() {
32
+ // device display data
33
+ private val displayMetrics = proxy.context.resources.displayMetrics
34
+ private val density = displayMetrics.density
35
+ private val windowWidth = (displayMetrics.widthPixels).toDouble() / density
36
+ private val windowHeight = (displayMetrics.heightPixels).toDouble() / density
37
+
38
+ // detection props
39
+ private var autoScale = false
40
+ private var faceDetector: FaceDetector? = null
41
+ private var runLandmarks = false
42
+ private var runClassifications = false
43
+ private var runContours = false
44
+ private var trackingEnabled = false
45
+
46
+ init {
47
+ // handle auto scaling
48
+ autoScale = options?.get("autoScale").toString() == "true"
49
+
50
+ // initializes faceDetector on creation
51
+ var performanceModeValue = FaceDetectorOptions.PERFORMANCE_MODE_FAST
52
+ var landmarkModeValue = FaceDetectorOptions.LANDMARK_MODE_NONE
53
+ var classificationModeValue = FaceDetectorOptions.CLASSIFICATION_MODE_NONE
54
+ var contourModeValue = FaceDetectorOptions.CONTOUR_MODE_NONE
55
+ var minFaceSize = 0.15f
56
+
57
+ if (options?.get("performanceMode").toString() == "accurate") {
58
+ performanceModeValue = FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE
59
+ }
60
+
61
+ if (options?.get("landmarkMode").toString() == "all") {
62
+ runLandmarks = true
63
+ landmarkModeValue = FaceDetectorOptions.LANDMARK_MODE_ALL
64
+ }
65
+
66
+ if (options?.get("classificationMode").toString() == "all") {
67
+ runClassifications = true
68
+ classificationModeValue = FaceDetectorOptions.CLASSIFICATION_MODE_ALL
69
+ }
70
+
71
+ if (options?.get("contourMode").toString() == "all") {
72
+ runContours = true
73
+ contourModeValue = FaceDetectorOptions.CONTOUR_MODE_ALL
74
+ }
75
+
76
+ val minFaceSizeParam = options?.get("minFaceSize").toString()
77
+ if (
78
+ minFaceSizeParam != "null" &&
79
+ minFaceSizeParam != minFaceSize.toString()
80
+ ) {
81
+ minFaceSize = minFaceSizeParam.toFloat()
82
+ }
83
+
84
+ val optionsBuilder = FaceDetectorOptions.Builder()
85
+ .setPerformanceMode(performanceModeValue)
86
+ .setLandmarkMode(landmarkModeValue)
87
+ .setContourMode(contourModeValue)
88
+ .setClassificationMode(classificationModeValue)
89
+ .setMinFaceSize(minFaceSize)
90
+
91
+ if (options?.get("trackingEnabled").toString() == "true") {
92
+ trackingEnabled = true
93
+ optionsBuilder.enableTracking()
94
+ }
95
+ faceDetector = FaceDetection.getClient(
96
+ optionsBuilder.build()
97
+ )
98
+ }
99
+
100
+ private fun processBoundingBox(
101
+ boundingBox: Rect,
102
+ sourceWidth: Double,
103
+ sourceHeight: Double,
104
+ orientation: Orientation,
105
+ scaleX: Double,
106
+ scaleY: Double
107
+ ): Map<String, Any> {
108
+ val bounds: MutableMap<String, Any> = HashMap()
109
+ val width = boundingBox.width().toDouble() * scaleX
110
+ val height = boundingBox.height().toDouble() * scaleY
111
+ val x = boundingBox.left.toDouble() * scaleX
112
+ val y = boundingBox.top.toDouble() * scaleY
113
+
114
+ when (orientation) {
115
+ Orientation.PORTRAIT -> {
116
+ // device is landscape left
117
+ bounds["x"] = (-y + sourceWidth * scaleX) - width
118
+ bounds["y"] = (-x + sourceHeight * scaleY) - height
119
+ }
120
+ Orientation.LANDSCAPE_LEFT -> {
121
+ // device is portrait
122
+ bounds["x"] = (-x + sourceWidth * scaleX) - width
123
+ bounds["y"] = y
124
+ }
125
+ Orientation.PORTRAIT_UPSIDE_DOWN -> {
126
+ // device is landscape right
127
+ bounds["x"] = y
128
+ bounds["y"] = x
129
+ }
130
+ Orientation.LANDSCAPE_RIGHT -> {
131
+ // device is upside down
132
+ bounds["x"] = x
133
+ bounds["y"] = (-y + sourceHeight * scaleY) - height
134
+ }
135
+ }
136
+ bounds["width"] = width
137
+ bounds["height"] = height
138
+ return bounds
139
+ }
140
+
141
+ private fun processLandmarks(
142
+ face: Face,
143
+ scaleX: Double,
144
+ scaleY: Double
145
+ ): Map<String, Any> {
146
+ val faceLandmarksTypes = intArrayOf(
147
+ FaceLandmark.LEFT_CHEEK,
148
+ FaceLandmark.LEFT_EAR,
149
+ FaceLandmark.LEFT_EYE,
150
+ FaceLandmark.MOUTH_BOTTOM,
151
+ FaceLandmark.MOUTH_LEFT,
152
+ FaceLandmark.MOUTH_RIGHT,
153
+ FaceLandmark.NOSE_BASE,
154
+ FaceLandmark.RIGHT_CHEEK,
155
+ FaceLandmark.RIGHT_EAR,
156
+ FaceLandmark.RIGHT_EYE
157
+ )
158
+ val faceLandmarksTypesStrings = arrayOf(
159
+ "LEFT_CHEEK",
160
+ "LEFT_EAR",
161
+ "LEFT_EYE",
162
+ "MOUTH_BOTTOM",
163
+ "MOUTH_LEFT",
164
+ "MOUTH_RIGHT",
165
+ "NOSE_BASE",
166
+ "RIGHT_CHEEK",
167
+ "RIGHT_EAR",
168
+ "RIGHT_EYE"
169
+ )
170
+ val faceLandmarksTypesMap: MutableMap<String, Any> = HashMap()
171
+ for (i in faceLandmarksTypesStrings.indices) {
172
+ val landmark = face.getLandmark(faceLandmarksTypes[i])
173
+ val landmarkName = faceLandmarksTypesStrings[i]
174
+ Log.d(
175
+ TAG,
176
+ "Getting '$landmarkName' landmark"
177
+ )
178
+ if (landmark == null) {
179
+ Log.d(
180
+ TAG,
181
+ "Landmark '$landmarkName' is null - going next"
182
+ )
183
+ continue
184
+ }
185
+ val point = landmark.position
186
+ val currentPointsMap: MutableMap<String, Double> = HashMap()
187
+ currentPointsMap["x"] = point.x.toDouble() * scaleX
188
+ currentPointsMap["y"] = point.y.toDouble() * scaleY
189
+ faceLandmarksTypesMap[landmarkName] = currentPointsMap
190
+ }
191
+ return faceLandmarksTypesMap
192
+ }
193
+
194
+ private fun processFaceContours(
195
+ face: Face,
196
+ scaleX: Double,
197
+ scaleY: Double
198
+ ): Map<String, Any> {
199
+ val faceContoursTypes = intArrayOf(
200
+ FaceContour.FACE,
201
+ FaceContour.LEFT_CHEEK,
202
+ FaceContour.LEFT_EYE,
203
+ FaceContour.LEFT_EYEBROW_BOTTOM,
204
+ FaceContour.LEFT_EYEBROW_TOP,
205
+ FaceContour.LOWER_LIP_BOTTOM,
206
+ FaceContour.LOWER_LIP_TOP,
207
+ FaceContour.NOSE_BOTTOM,
208
+ FaceContour.NOSE_BRIDGE,
209
+ FaceContour.RIGHT_CHEEK,
210
+ FaceContour.RIGHT_EYE,
211
+ FaceContour.RIGHT_EYEBROW_BOTTOM,
212
+ FaceContour.RIGHT_EYEBROW_TOP,
213
+ FaceContour.UPPER_LIP_BOTTOM,
214
+ FaceContour.UPPER_LIP_TOP
215
+ )
216
+ val faceContoursTypesStrings = arrayOf(
217
+ "FACE",
218
+ "LEFT_CHEEK",
219
+ "LEFT_EYE",
220
+ "LEFT_EYEBROW_BOTTOM",
221
+ "LEFT_EYEBROW_TOP",
222
+ "LOWER_LIP_BOTTOM",
223
+ "LOWER_LIP_TOP",
224
+ "NOSE_BOTTOM",
225
+ "NOSE_BRIDGE",
226
+ "RIGHT_CHEEK",
227
+ "RIGHT_EYE",
228
+ "RIGHT_EYEBROW_BOTTOM",
229
+ "RIGHT_EYEBROW_TOP",
230
+ "UPPER_LIP_BOTTOM",
231
+ "UPPER_LIP_TOP"
232
+ )
233
+ val faceContoursTypesMap: MutableMap<String, Any> = HashMap()
234
+ for (i in faceContoursTypesStrings.indices) {
235
+ val contour = face.getContour(faceContoursTypes[i])
236
+ val contourName = faceContoursTypesStrings[i]
237
+ Log.d(
238
+ TAG,
239
+ "Getting '$contourName' contour"
240
+ )
241
+ if (contour == null) {
242
+ Log.d(
243
+ TAG,
244
+ "Face contour '$contourName' is null - going next"
245
+ )
246
+ continue
247
+ }
248
+ val points = contour.points
249
+ val pointsMap: MutableMap<String, Map<String, Double>> = HashMap()
250
+ for (j in points.indices) {
251
+ val currentPointsMap: MutableMap<String, Double> = HashMap()
252
+ currentPointsMap["x"] = points[j].x.toDouble() * scaleX
253
+ currentPointsMap["y"] = points[j].y.toDouble() * scaleY
254
+ pointsMap[j.toString()] = currentPointsMap
255
+ }
256
+ faceContoursTypesMap[contourName] = pointsMap
257
+ }
258
+ return faceContoursTypesMap
259
+ }
260
+
261
+ private fun getOrientation(
262
+ orientation: Orientation
263
+ ): Int {
264
+ return when (orientation) {
265
+ // device is landscape left
266
+ Orientation.PORTRAIT -> 0
267
+ // device is portrait
268
+ Orientation.LANDSCAPE_LEFT -> 270
269
+ // device is landscape right
270
+ Orientation.PORTRAIT_UPSIDE_DOWN -> 180
271
+ // device is upside-down
272
+ Orientation.LANDSCAPE_RIGHT -> 90
273
+ }
274
+ }
275
+
276
+ override fun callback(
277
+ frame: Frame,
278
+ params: Map<String, Any>?
279
+ ): Any {
280
+ val result = ArrayList<Map<String, Any>>()
281
+ try {
282
+ val orientation = getOrientation(frame.orientation)
283
+ val image = InputImage.fromMediaImage(frame.image, orientation)
284
+ // we need to invert sizes as frame is always -90deg rotated
285
+ val width = image.height.toDouble()
286
+ val height = image.width.toDouble()
287
+ val scaleX = if (autoScale) windowWidth / width else 1.0
288
+ val scaleY = if (autoScale) windowHeight / height else 1.0
289
+ val task = faceDetector!!.process(image)
290
+ val faces = Tasks.await(task)
291
+ faces.forEach { face ->
292
+ val bmpFrameResult = ImageConvertUtils.getInstance().getUpRightBitmap(image)
293
+ val bmpFaceResult =
294
+ Bitmap.createBitmap(
295
+ TF_OD_API_INPUT_SIZE,
296
+ TF_OD_API_INPUT_SIZE,
297
+ Bitmap.Config.ARGB_8888
298
+ )
299
+ val faceBB = RectF(face.boundingBox)
300
+ val cvFace = Canvas(bmpFaceResult)
301
+ val sx = TF_OD_API_INPUT_SIZE.toFloat() / faceBB.width()
302
+ val sy = TF_OD_API_INPUT_SIZE.toFloat() / faceBB.height()
303
+ val matrix = Matrix()
304
+ matrix.postTranslate(-faceBB.left, -faceBB.top)
305
+ matrix.postScale(sx, sy)
306
+ cvFace.drawBitmap(bmpFrameResult, matrix, null)
307
+ val input: ByteBuffer = FaceHelper().bitmap2ByteBuffer(bmpFaceResult)
308
+ val output: FloatBuffer = FloatBuffer.allocate(192)
309
+ interpreter?.run(input, output)
310
+
311
+ val arrayData: MutableList<Any> = ArrayList()
312
+ for (i: Float in output.array()) {
313
+ arrayData.add(i.toDouble())
314
+ }
315
+ val map: MutableMap<String, Any> = HashMap()
316
+ if (runLandmarks) {
317
+ map["landmarks"] = processLandmarks(
318
+ face,
319
+ scaleX,
320
+ scaleY
321
+ )
322
+ }
323
+ if (runClassifications) {
324
+ map["leftEyeOpenProbability"] = face.leftEyeOpenProbability?.toDouble() ?: -1
325
+ map["rightEyeOpenProbability"] = face.rightEyeOpenProbability?.toDouble() ?: -1
326
+ map["smilingProbability"] = face.smilingProbability?.toDouble() ?: -1
327
+ }
328
+ if (runContours) {
329
+ map["contours"] = processFaceContours(
330
+ face,
331
+ scaleX,
332
+ scaleY
333
+ )
334
+ }
335
+ if (trackingEnabled) {
336
+ map["trackingId"] = face.trackingId ?: -1
337
+ }
338
+ map["rollAngle"] = face.headEulerAngleZ.toDouble()
339
+ map["pitchAngle"] = face.headEulerAngleX.toDouble()
340
+ map["yawAngle"] = face.headEulerAngleY.toDouble()
341
+ map["bounds"] = processBoundingBox(
342
+ face.boundingBox,
343
+ width,
344
+ height,
345
+ frame.orientation,
346
+ scaleX,
347
+ scaleY
348
+ )
349
+ map["data"] = arrayData
350
+ result.add(map)
351
+ }
352
+ } catch (e: Exception) {
353
+ Log.e(TAG, "Error processing face detection: ", e)
354
+ } catch (e: FrameInvalidError) {
355
+ Log.e(TAG, "Frame invalid error: ", e)
356
+ }
357
+ return result
358
+ }
359
+ }
@@ -0,0 +1,238 @@
1
+ import VisionCamera
2
+ import MLKitFaceDetection
3
+ import MLKitVision
4
+ import CoreML
5
+ import UIKit
6
+ import AVFoundation
7
+ import Accelerate
8
+ import TensorFlowLite
9
+
10
+ let batchSize = 1
11
+ let inputChannels = 1
12
+ let inputWidth = 112
13
+ let inputHeight = 112
14
+
15
+ // TensorFlow Lite `Interpreter` object for performing inference on a given model.
16
+ var interpreter: Interpreter? = nil
17
+
18
+ class FaceHelper {
19
+ static func processContours(from face: Face) -> [String:[[String:CGFloat]]] {
20
+ let faceContoursTypes = [
21
+ FaceContourType.face,
22
+ FaceContourType.leftEyebrowTop,
23
+ FaceContourType.leftEyebrowBottom,
24
+ FaceContourType.rightEyebrowTop,
25
+ FaceContourType.rightEyebrowBottom,
26
+ FaceContourType.leftEye,
27
+ FaceContourType.rightEye,
28
+ FaceContourType.upperLipTop,
29
+ FaceContourType.upperLipBottom,
30
+ FaceContourType.lowerLipTop,
31
+ FaceContourType.lowerLipBottom,
32
+ FaceContourType.noseBridge,
33
+ FaceContourType.noseBottom,
34
+ FaceContourType.leftCheek,
35
+ FaceContourType.rightCheek,
36
+ ]
37
+
38
+ let faceContoursTypesStrings = [
39
+ "FACE",
40
+ "LEFT_EYEBROW_TOP",
41
+ "LEFT_EYEBROW_BOTTOM",
42
+ "RIGHT_EYEBROW_TOP",
43
+ "RIGHT_EYEBROW_BOTTOM",
44
+ "LEFT_EYE",
45
+ "RIGHT_EYE",
46
+ "UPPER_LIP_TOP",
47
+ "UPPER_LIP_BOTTOM",
48
+ "LOWER_LIP_TOP",
49
+ "LOWER_LIP_BOTTOM",
50
+ "NOSE_BRIDGE",
51
+ "NOSE_BOTTOM",
52
+ "LEFT_CHEEK",
53
+ "RIGHT_CHEEK",
54
+ ];
55
+
56
+ var faceContoursTypesMap: [String:[[String:CGFloat]]] = [:]
57
+
58
+ for i in 0..<faceContoursTypes.count {
59
+ let contour = face.contour(ofType: faceContoursTypes[i]);
60
+
61
+ var pointsArray: [[String:CGFloat]] = []
62
+
63
+ if let points = contour?.points {
64
+ for point in points {
65
+ let currentPointsMap = [
66
+ "x": point.x,
67
+ "y": point.y,
68
+ ]
69
+
70
+ pointsArray.append(currentPointsMap)
71
+ }
72
+
73
+ faceContoursTypesMap[faceContoursTypesStrings[i]] = pointsArray
74
+ }
75
+ }
76
+
77
+ return faceContoursTypesMap
78
+ }
79
+
80
+ static func processBoundingBox(from face: Face) -> [String:Any] {
81
+ let frameRect = face.frame
82
+
83
+ let offsetX = (frameRect.midX - ceil(frameRect.width)) / 2.0
84
+ let offsetY = (frameRect.midY - ceil(frameRect.height)) / 2.0
85
+
86
+ let x = frameRect.maxX + offsetX
87
+ let y = frameRect.minY + offsetY
88
+
89
+ return [
90
+ "x": frameRect.midX + (frameRect.midX - x),
91
+ "y": frameRect.midY + (y - frameRect.midY),
92
+ "width": frameRect.width,
93
+ "height": frameRect.height,
94
+ "boundingCenterX": frameRect.midX,
95
+ "boundingCenterY": frameRect.midY
96
+ ]
97
+ }
98
+
99
+ static func getImageFaceFromUIImage(from image: UIImage, rectImage: CGRect) -> UIImage? {
100
+ let imageRef: CGImage = (image.cgImage?.cropping(to: rectImage)!)!
101
+ let imageCrop: UIImage = UIImage(cgImage: imageRef, scale: 0.5, orientation: image.imageOrientation)
102
+ return imageCrop
103
+ }
104
+
105
+ static func convertImageToBase64(image: UIImage) -> String {
106
+ let imageData = image.pngData()!
107
+ return imageData.base64EncodedString()
108
+ }
109
+
110
+ static func rgbDataFromBuffer(
111
+ _ buffer: CVPixelBuffer,
112
+ byteCount: Int,
113
+ isModelQuantized: Bool
114
+ ) -> Data? {
115
+ CVPixelBufferLockBaseAddress(buffer, .readOnly)
116
+ defer {
117
+ CVPixelBufferUnlockBaseAddress(buffer, .readOnly)
118
+ }
119
+ guard let sourceData = CVPixelBufferGetBaseAddress(buffer) else {
120
+ return nil
121
+ }
122
+
123
+ let width = CVPixelBufferGetWidth(buffer)
124
+ let height = CVPixelBufferGetHeight(buffer)
125
+ let sourceBytesPerRow = CVPixelBufferGetBytesPerRow(buffer)
126
+ let destinationChannelCount = 3
127
+ let destinationBytesPerRow = destinationChannelCount * width
128
+
129
+ var sourceBuffer = vImage_Buffer(data: sourceData,
130
+ height: vImagePixelCount(height),
131
+ width: vImagePixelCount(width),
132
+ rowBytes: sourceBytesPerRow)
133
+
134
+ guard let destinationData = malloc(height * destinationBytesPerRow) else {
135
+ print("Error: out of memory")
136
+ return nil
137
+ }
138
+
139
+ defer {
140
+ free(destinationData)
141
+ }
142
+
143
+ var destinationBuffer = vImage_Buffer(data: destinationData,
144
+ height: vImagePixelCount(height),
145
+ width: vImagePixelCount(width),
146
+ rowBytes: destinationBytesPerRow)
147
+
148
+ let pixelBufferFormat = CVPixelBufferGetPixelFormatType(buffer)
149
+
150
+ switch (pixelBufferFormat) {
151
+ case kCVPixelFormatType_32BGRA:
152
+ vImageConvert_BGRA8888toRGB888(&sourceBuffer, &destinationBuffer, UInt32(kvImageNoFlags))
153
+ case kCVPixelFormatType_32ARGB:
154
+ vImageConvert_ARGB8888toRGB888(&sourceBuffer, &destinationBuffer, UInt32(kvImageNoFlags))
155
+ case kCVPixelFormatType_32RGBA:
156
+ vImageConvert_RGBA8888toRGB888(&sourceBuffer, &destinationBuffer, UInt32(kvImageNoFlags))
157
+ default:
158
+ // Unknown pixel format.
159
+ return nil
160
+ }
161
+
162
+ let byteData = Data(bytes: destinationBuffer.data, count: destinationBuffer.rowBytes * height)
163
+ if isModelQuantized {
164
+ return byteData
165
+ }
166
+
167
+ // Not quantized, convert to floats
168
+ let bytes = Array<UInt8>(unsafeData: byteData)!
169
+ var floats = [Float]()
170
+ for i in 0..<bytes.count {
171
+ floats.append(Float(bytes[i]) / 255.0)
172
+ }
173
+ return Data(copyingBufferOf: floats)
174
+ }
175
+
176
+ static func uiImageToPixelBuffer(image: UIImage, size: Int) -> CVPixelBuffer? {
177
+ let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue, kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
178
+ var pixelBuffer : CVPixelBuffer?
179
+ let status = CVPixelBufferCreate(kCFAllocatorDefault, size, size, kCVPixelFormatType_32ARGB, attrs, &pixelBuffer)
180
+ guard (status == kCVReturnSuccess) else {
181
+ return nil
182
+ }
183
+
184
+ CVPixelBufferLockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
185
+ let pixelData = CVPixelBufferGetBaseAddress(pixelBuffer!)
186
+
187
+ let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
188
+ let context = CGContext(data: pixelData, width: size, height: size, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer!), space: rgbColorSpace, bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue)
189
+
190
+ context?.translateBy(x: 0, y: CGFloat(size))
191
+ context?.scaleBy(x: 1.0, y: -1.0)
192
+
193
+ UIGraphicsPushContext(context!)
194
+ image.draw(in: CGRect(x: 0, y: 0, width: size, height: size))
195
+ UIGraphicsPopContext()
196
+ CVPixelBufferUnlockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
197
+ return pixelBuffer
198
+ }
199
+
200
+ static func getImageFaceFromBuffer(from sampleBuffer: CMSampleBuffer?, rectImage: CGRect) -> UIImage? {
201
+ guard let sampleBuffer = sampleBuffer else {
202
+ print("Sample buffer is NULL.")
203
+ return nil
204
+ }
205
+ guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
206
+ print("Invalid sample buffer.")
207
+ return nil
208
+ }
209
+ let ciimage = CIImage(cvPixelBuffer: imageBuffer)
210
+ let context = CIContext(options: nil)
211
+ let cgImage = context.createCGImage(ciimage, from: ciimage.extent)!
212
+
213
+ if (!rectImage.isNull) {
214
+ let imageRef: CGImage = cgImage.cropping(to: rectImage)!
215
+ let imageCrop: UIImage = UIImage(cgImage: imageRef, scale: 0.5, orientation: .up)
216
+ return imageCrop
217
+ } else {
218
+ return nil
219
+ }
220
+ }
221
+ }
222
+
223
+ // MARK: - Extensions
224
+ extension Data {
225
+ // Creates a new buffer by copying the buffer pointer of the given array.
226
+ init<T>(copyingBufferOf array: [T]) {
227
+ self = array.withUnsafeBufferPointer(Data.init)
228
+ }
229
+ }
230
+
231
+ extension Array {
232
+ // Creates a new array from the bytes of the given unsafe data.
233
+ init?(unsafeData: Data) {
234
+ guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
235
+ self = unsafeData.withUnsafeBytes { .init($0.bindMemory(to: Element.self)) }
236
+ }
237
+ }
238
+
@@ -0,0 +1,6 @@
1
+ #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
2
+ #import <VisionCamera/FrameProcessorPlugin.h>
3
+ #import <VisionCamera/Frame.h>
4
+ #endif
5
+ #import <React/RCTBridgeModule.h>
6
+ #import <React/RCTViewManager.h>
@@ -0,0 +1,19 @@
1
+ #import <React/RCTBridgeModule.h>
2
+
3
+ @interface RCT_EXTERN_MODULE(VisionCameraFaceDetectionModule, NSObject)
4
+
5
+ RCT_EXTERN_METHOD(detectFromBase64:(NSString)imageString
6
+ withResolver:(RCTPromiseResolveBlock)resolve
7
+ withRejecter:(RCTPromiseRejectBlock)reject)
8
+
9
+ RCT_EXTERN_METHOD(initTensor:(NSString)modelName
10
+ withCount:(NSNumber)count
11
+ withResolver:(RCTPromiseResolveBlock)resolve
12
+ withRejecter:(RCTPromiseRejectBlock)reject)
13
+
14
+ + (BOOL)requiresMainQueueSetup
15
+ {
16
+ return NO;
17
+ }
18
+
19
+ @end