vision-camera-face-detection 2.2.0 → 2.2.2

package/VisionCameraFaceDetection.podspec

@@ -11,7 +11,7 @@ Pod::Spec.new do |s|
   s.license      = package["license"]
   s.authors      = package["author"]
 
-  s.platforms    = { :ios => min_ios_version_supported }
+  s.platforms    = { :ios => "15.5" } # 15.5 is the minimum version for GoogleMLKit/FaceDetection 7.0.0
   s.source       = { :git => "https://github.com/edritech93/vision-camera-face-detection.git", :tag => "#{s.version}" }
 
   s.source_files = "ios/**/*.{h,m,mm,swift}"

package/android/build.gradle

@@ -80,7 +80,6 @@ repositories {
 }
 
 def kotlin_version = getExtOrDefault("kotlinVersion")
-def tensor_version = getExtOrDefault("tensorVersion")
 
 dependencies {
   implementation "com.facebook.react:react-android"
@@ -89,7 +88,9 @@ dependencies {
   implementation "androidx.annotation:annotation:1.8.2"
   implementation "androidx.camera:camera-core:1.3.4"
   implementation "com.google.mlkit:face-detection:16.1.7"
-  implementation "org.tensorflow:tensorflow-lite:$tensor_version"
-  implementation 'org.tensorflow:tensorflow-lite-support:0.1.0'
+  implementation 'com.google.ai.edge.litert:litert:1.4.0'
+  implementation 'com.google.ai.edge.litert:litert-api:1.4.0'
+  implementation 'com.google.ai.edge.litert:litert-support:1.4.0'
+  implementation 'com.google.ai.edge.litert:litert-metadata:1.4.0'
 }
 

package/android/gradle.properties

@@ -3,4 +3,3 @@ VisionCameraFaceDetection_minSdkVersion=24
 VisionCameraFaceDetection_targetSdkVersion=34
 VisionCameraFaceDetection_compileSdkVersion=35
 VisionCameraFaceDetection_ndkVersion=27.1.12297006
-VisionCameraFaceDetection_tensorVersion=2.11.0

package/android/src/main/java/com/visioncamerafacedetection/VisionCameraFaceDetectionPlugin.kt

@@ -6,6 +6,7 @@ import android.graphics.Matrix
 import android.graphics.Rect
 import android.graphics.RectF
 import android.util.Log
+import android.view.Surface
 import com.google.android.gms.tasks.Tasks
 import com.google.mlkit.vision.common.InputImage
 import com.google.mlkit.vision.common.internal.ImageConvertUtils
@@ -17,6 +18,7 @@ import com.google.mlkit.vision.face.FaceDetectorOptions
 import com.google.mlkit.vision.face.FaceLandmark
 import com.mrousavy.camera.core.FrameInvalidError
 import com.mrousavy.camera.core.types.Orientation
+import com.mrousavy.camera.core.types.Position
 import com.mrousavy.camera.frameprocessors.Frame
 import com.mrousavy.camera.frameprocessors.FrameProcessorPlugin
 import com.mrousavy.camera.frameprocessors.VisionCameraProxy
@@ -24,29 +26,26 @@ import java.nio.ByteBuffer
 import java.nio.FloatBuffer
 
 private const val TAG = "FaceDetector"
-
 class VisionCameraFaceDetectionPlugin(
   proxy: VisionCameraProxy,
   options: Map<String, Any>?
 ) : FrameProcessorPlugin() {
-  // device display data
-  private val displayMetrics = proxy.context.resources.displayMetrics
-  private val density = displayMetrics.density
-  private val windowWidth = (displayMetrics.widthPixels).toDouble() / density
-  private val windowHeight = (displayMetrics.heightPixels).toDouble() / density
-
   // detection props
-  private var autoScale = false
+  private var autoMode = false
   private var faceDetector: FaceDetector? = null
   private var runLandmarks = false
   private var runClassifications = false
   private var runContours = false
   private var trackingEnabled = false
+  private var windowWidth = 1.0
+  private var windowHeight = 1.0
+  private var cameraFacing: Position = Position.FRONT
+  private val orientationManager = VisionCameraFaceDetectorOrientation(proxy.context)
   private var enableTensor = false
 
   init {
     // handle auto scaling
-    autoScale = options?.get("autoScale").toString() == "true"
+    autoMode = options?.get("autoMode").toString() == "true"
 
     // handle enable/disable tensor
     enableTensor = options?.get("enableTensor").toString() == "true"
@@ -56,7 +55,13 @@ class VisionCameraFaceDetectionPlugin(
     var landmarkModeValue = FaceDetectorOptions.LANDMARK_MODE_NONE
     var classificationModeValue = FaceDetectorOptions.CLASSIFICATION_MODE_NONE
     var contourModeValue = FaceDetectorOptions.CONTOUR_MODE_NONE
-    var minFaceSize = 0.15f
+
+    windowWidth = (options?.get("windowWidth") ?: 1.0) as Double
+    windowHeight = (options?.get("windowHeight") ?: 1.0) as Double
+
+    if (options?.get("cameraFacing").toString() == "back") {
+      cameraFacing = Position.BACK
+    }
 
     if (options?.get("performanceMode").toString() == "accurate") {
       performanceModeValue = FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE
@@ -77,25 +82,19 @@ class VisionCameraFaceDetectionPlugin(
       contourModeValue = FaceDetectorOptions.CONTOUR_MODE_ALL
     }
 
-    val minFaceSizeParam = options?.get("minFaceSize").toString()
-    if (
-      minFaceSizeParam != "null" &&
-      minFaceSizeParam != minFaceSize.toString()
-    ) {
-      minFaceSize = minFaceSizeParam.toFloat()
-    }
-
+    val minFaceSize: Double = (options?.get("minFaceSize") ?: 0.15) as Double
     val optionsBuilder = FaceDetectorOptions.Builder()
       .setPerformanceMode(performanceModeValue)
       .setLandmarkMode(landmarkModeValue)
       .setContourMode(contourModeValue)
       .setClassificationMode(classificationModeValue)
-      .setMinFaceSize(minFaceSize)
+      .setMinFaceSize(minFaceSize.toFloat())
 
     if (options?.get("trackingEnabled").toString() == "true") {
       trackingEnabled = true
       optionsBuilder.enableTracking()
     }
+
     faceDetector = FaceDetection.getClient(
       optionsBuilder.build()
     )
@@ -105,43 +104,72 @@ class VisionCameraFaceDetectionPlugin(
     boundingBox: Rect,
     sourceWidth: Double,
     sourceHeight: Double,
-    orientation: Orientation,
     scaleX: Double,
     scaleY: Double
   ): Map<String, Any> {
     val bounds: MutableMap<String, Any> = HashMap()
     val width = boundingBox.width().toDouble() * scaleX
     val height = boundingBox.height().toDouble() * scaleY
-    val x = boundingBox.left.toDouble() * scaleX
-    val y = boundingBox.top.toDouble() * scaleY
+    val x = boundingBox.left.toDouble()
+    val y = boundingBox.top.toDouble()
 
-    when (orientation) {
-      Orientation.PORTRAIT -> {
-        // device is landscape left
-        bounds["x"] = (-y + sourceWidth * scaleX) - width
-        bounds["y"] = (-x + sourceHeight * scaleY) - height
-      }
+    bounds["width"] = width
+    bounds["height"] = height
+    bounds["x"] = x * scaleX
+    bounds["y"] = y * scaleY
 
-      Orientation.LANDSCAPE_LEFT -> {
-        // device is portrait
-        bounds["x"] = (-x + sourceWidth * scaleX) - width
-        bounds["y"] = y
-      }
+    if(!autoMode) return bounds
 
-      Orientation.PORTRAIT_UPSIDE_DOWN -> {
+    // using front camera
+    if(cameraFacing == Position.FRONT) {
+      when (orientationManager.orientation) {
+        // device is portrait
+        Surface.ROTATION_0 -> {
+          bounds["x"] = ((-x * scaleX) + sourceWidth * scaleX) - width
+          bounds["y"] = y * scaleY
+        }
         // device is landscape right
-        bounds["x"] = y
-        bounds["y"] = x
+        Surface.ROTATION_270 -> {
+          bounds["x"] = y * scaleX
+          bounds["y"] = x * scaleY
+        }
+        // device is upside down
+        Surface.ROTATION_180 -> {
+          bounds["x"] = x * scaleX
+          bounds["y"] = ((-y * scaleY) + sourceHeight * scaleY) - height
+        }
+        // device is landscape left
+        Surface.ROTATION_90 -> {
+          bounds["x"] = ((-y * scaleX) + sourceWidth * scaleX) - width
+          bounds["y"] = ((-x * scaleY) + sourceHeight * scaleY) - height
+        }
       }
+      return bounds
+    }
 
-      Orientation.LANDSCAPE_RIGHT -> {
-        // device is upside down
-        bounds["x"] = x
-        bounds["y"] = (-y + sourceHeight * scaleY) - height
+    // using back camera
+    when (orientationManager.orientation) {
+      // device is portrait
+      Surface.ROTATION_0 -> {
+        bounds["x"] = x * scaleX
+        bounds["y"] = y * scaleY
+      }
+      // device is landscape right
+      Surface.ROTATION_270 -> {
+        bounds["x"] = y * scaleX
+        bounds["y"] = ((-x * scaleY) + sourceHeight * scaleY) - height
+      }
+      // device is upside down
+      Surface.ROTATION_180 -> {
+        bounds["x"] =((-x * scaleX) + sourceWidth * scaleX) - width
+        bounds["y"] = ((-y * scaleY) + sourceHeight * scaleY) - height
+      }
+      // device is landscape left
+      Surface.ROTATION_90 -> {
+        bounds["x"] = ((-y * scaleX) + sourceWidth * scaleX) - width
+        bounds["y"] = x * scaleY
       }
     }
-    bounds["width"] = width
-    bounds["height"] = height
     return bounds
   }
 
@@ -178,23 +206,16 @@ class VisionCameraFaceDetectionPlugin(
     for (i in faceLandmarksTypesStrings.indices) {
       val landmark = face.getLandmark(faceLandmarksTypes[i])
       val landmarkName = faceLandmarksTypesStrings[i]
-      Log.d(
-        TAG,
-        "Getting '$landmarkName' landmark"
-      )
-      if (landmark == null) {
-        Log.d(
-          TAG,
-          "Landmark '$landmarkName' is null - going next"
-        )
-        continue
-      }
+
+      if (landmark == null) continue
+
       val point = landmark.position
       val currentPointsMap: MutableMap<String, Double> = HashMap()
       currentPointsMap["x"] = point.x.toDouble() * scaleX
       currentPointsMap["y"] = point.y.toDouble() * scaleY
       faceLandmarksTypesMap[landmarkName] = currentPointsMap
     }
+
     return faceLandmarksTypesMap
   }
 
@@ -241,42 +262,34 @@ class VisionCameraFaceDetectionPlugin(
     for (i in faceContoursTypesStrings.indices) {
       val contour = face.getContour(faceContoursTypes[i])
       val contourName = faceContoursTypesStrings[i]
-      Log.d(
-        TAG,
-        "Getting '$contourName' contour"
-      )
-      if (contour == null) {
-        Log.d(
-          TAG,
-          "Face contour '$contourName' is null - going next"
-        )
-        continue
-      }
+
+      if (contour == null) continue
+
       val points = contour.points
-      val pointsMap: MutableMap<String, Map<String, Double>> = HashMap()
+      val pointsMap: MutableList<Map<String, Double>> = mutableListOf()
       for (j in points.indices) {
         val currentPointsMap: MutableMap<String, Double> = HashMap()
         currentPointsMap["x"] = points[j].x.toDouble() * scaleX
         currentPointsMap["y"] = points[j].y.toDouble() * scaleY
-        pointsMap[j.toString()] = currentPointsMap
+        pointsMap.add(currentPointsMap)
       }
+
       faceContoursTypesMap[contourName] = pointsMap
     }
     return faceContoursTypesMap
   }
 
-  private fun getOrientation(
-    orientation: Orientation
-  ): Int {
-    return when (orientation) {
-      // device is landscape left
-      Orientation.PORTRAIT -> 0
+  private fun getImageOrientation(): Int {
+    return when (orientationManager.orientation) {
       // device is portrait
-      Orientation.LANDSCAPE_LEFT -> 270
+      Surface.ROTATION_0 -> if(cameraFacing == Position.FRONT) 270 else 90
       // device is landscape right
-      Orientation.PORTRAIT_UPSIDE_DOWN -> 180
-      // device is upside-down
-      Orientation.LANDSCAPE_RIGHT -> 90
+      Surface.ROTATION_270 -> if(cameraFacing == Position.FRONT) 180 else 180
+      // device is upside down
+      Surface.ROTATION_180 -> if(cameraFacing == Position.FRONT) 90 else 270
+      // device is landscape left
+      Surface.ROTATION_90 -> if(cameraFacing == Position.FRONT) 0 else 0
+      else -> 0
     }
   }
 
@@ -285,19 +298,19 @@ class VisionCameraFaceDetectionPlugin(
     params: Map<String, Any>?
   ): Any {
     val result = ArrayList<Map<String, Any>>()
+
     try {
-      val orientation = getOrientation(frame.orientation)
-      val image = InputImage.fromMediaImage(frame.image, orientation)
+      val image = InputImage.fromMediaImage(frame.image, getImageOrientation())
       // we need to invert sizes as frame is always -90deg rotated
       val width = image.height.toDouble()
       val height = image.width.toDouble()
-      val scaleX = if (autoScale) windowWidth / width else 1.0
-      val scaleY = if (autoScale) windowHeight / height else 1.0
+      val scaleX = if(autoMode) windowWidth / width else 1.0
+      val scaleY = if(autoMode) windowHeight / height else 1.0
       val task = faceDetector!!.process(image)
       val faces = Tasks.await(task)
-      faces.forEach { face ->
+      faces.forEach{face ->
         val map: MutableMap<String, Any> = HashMap()
-        val arrayData: MutableList<Any> = ArrayList()
+        val arrayData: MutableList<Double> = ArrayList()
         if (enableTensor) {
           val bmpFrameResult = ImageConvertUtils.getInstance().getUpRightBitmap(image)
           val bmpFaceResult =
@@ -333,11 +346,13 @@ class VisionCameraFaceDetectionPlugin(
             scaleY
           )
         }
+
         if (runClassifications) {
           map["leftEyeOpenProbability"] = face.leftEyeOpenProbability?.toDouble() ?: -1
           map["rightEyeOpenProbability"] = face.rightEyeOpenProbability?.toDouble() ?: -1
           map["smilingProbability"] = face.smilingProbability?.toDouble() ?: -1
         }
+
         if (runContours) {
           map["contours"] = processFaceContours(
             face,
@@ -345,9 +360,11 @@ class VisionCameraFaceDetectionPlugin(
             scaleY
           )
         }
+
         if (trackingEnabled) {
           map["trackingId"] = face.trackingId ?: -1
         }
+
         map["rollAngle"] = face.headEulerAngleZ.toDouble()
         map["pitchAngle"] = face.headEulerAngleX.toDouble()
         map["yawAngle"] = face.headEulerAngleY.toDouble()
@@ -355,7 +372,6 @@ class VisionCameraFaceDetectionPlugin(
           face.boundingBox,
           width,
           height,
-          frame.orientation,
           scaleX,
           scaleY
         )
@@ -366,6 +382,7 @@ class VisionCameraFaceDetectionPlugin(
     } catch (e: FrameInvalidError) {
       Log.e(TAG, "Frame invalid error: ", e)
     }
+
     return result
   }
 }

package/android/src/main/java/com/visioncamerafacedetection/VisionCameraFaceDetectorOrientation.kt

@@ -0,0 +1,57 @@
+package com.visioncamerafacedetection
+
+import android.content.Context
+import android.util.Log
+import android.view.OrientationEventListener
+import android.view.Surface
+
+private const val TAG = "FaceDetectorOrientation"
+class VisionCameraFaceDetectorOrientation(private val context: Context) {
+    var orientation = Surface.ROTATION_0
+    private var orientationListener: OrientationEventListener? = null
+
+    init {
+        if (orientationListener == null) {
+            Log.d(TAG, "Assigning new device orientation listener")
+            orientationListener = object : OrientationEventListener(context) {
+                override fun onOrientationChanged(rotationDegrees: Int) {
+                    orientation = degreesToSurfaceRotation(rotationDegrees)
+                }
+            }
+        }
+
+        orientation = Surface.ROTATION_0
+        startDeviceOrientationListener()
+    }
+
+    protected fun finalize() {
+        stopDeviceOrientationListener()
+    }
+
+    private fun startDeviceOrientationListener() {
+        stopDeviceOrientationListener()
+
+        if (
+            orientationListener != null &&
+            orientationListener!!.canDetectOrientation()
+        ) {
+            Log.d(TAG, "Enabling device orientation listener")
+            orientationListener!!.enable()
+        }
+    }
+
+    private fun stopDeviceOrientationListener() {
+        if (orientationListener != null) {
+            Log.d(TAG, "Disabled device orientation listener")
+            orientationListener!!.disable()
+        }
+    }
+
+    private fun degreesToSurfaceRotation(degrees: Int): Int =
+        when (degrees) {
+            in 45..135 -> Surface.ROTATION_270
+            in 135..225 -> Surface.ROTATION_180
+            in 225..315 -> Surface.ROTATION_90
+            else -> Surface.ROTATION_0
+        }
+}