react-native-nitro-pose-exercises 1.0.8 → 1.1.1

package/android/src/main/java/com/margelo/nitro/nitroposeexercises/NitroPoseExercises.kt

@@ -1,16 +1,19 @@
 package com.margelo.nitro.nitroposeexercises
 
 import android.graphics.Bitmap
+import android.graphics.Matrix
+import android.media.Image
 import androidx.annotation.Keep
 import com.facebook.proguard.annotations.DoNotStrip
-import com.google.mediapipe.framework.image.BitmapImageBuilder
-import com.google.mediapipe.tasks.core.BaseOptions
-import com.google.mediapipe.tasks.vision.core.RunningMode
-import com.google.mediapipe.tasks.vision.poselandmarker.PoseLandmarker
-import com.google.mediapipe.tasks.vision.poselandmarker.PoseLandmarkerOptions
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.pose.PoseDetection
+import com.google.mlkit.vision.pose.PoseDetector
+import com.google.mlkit.vision.pose.PoseLandmark
+import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions
 import com.margelo.nitro.NitroModules
 import com.margelo.nitro.core.Promise
 import com.margelo.nitro.camera.HybridFrameSpec
+import com.margelo.nitro.camera.NativeFrame
 import kotlin.math.acos
 import kotlin.math.max
 import kotlin.math.min
@@ -20,10 +23,52 @@ import kotlin.math.sqrt
 @DoNotStrip
 class HybridPoseExercise : HybridNitroPoseExercisesSpec() {
 
-  // ─── MediaPipe ──────────────────────────────────────────────
-  private var poseLandmarker: PoseLandmarker? = null
+  // ─── ML Kit ─────────────────────────────────────────────────
+  private var poseDetector: PoseDetector? = null
   private var isInitialized = false
 
+  // ─── Cached Landmarks (ML Kit is async, we cache last result) ──
+  private var cachedLandmarks: Array<Landmark> = emptyArray()
+  private val landmarkLock = Any()
+
+  // ─── Landmark Index Mapping ─────────────────────────────────
+  // ML Kit PoseLandmark type → MediaPipe index that JS configs expect
+  private val mlKitToMediaPipeMap = mapOf(
+    PoseLandmark.NOSE to 0,
+    PoseLandmark.LEFT_EYE_INNER to 1,
+    PoseLandmark.LEFT_EYE to 2,
+    PoseLandmark.LEFT_EYE_OUTER to 3,
+    PoseLandmark.RIGHT_EYE_INNER to 4,
+    PoseLandmark.RIGHT_EYE to 5,
+    PoseLandmark.RIGHT_EYE_OUTER to 6,
+    PoseLandmark.LEFT_EAR to 7,
+    PoseLandmark.RIGHT_EAR to 8,
+    PoseLandmark.LEFT_MOUTH to 9,
+    PoseLandmark.RIGHT_MOUTH to 10,
+    PoseLandmark.LEFT_SHOULDER to 11,
+    PoseLandmark.RIGHT_SHOULDER to 12,
+    PoseLandmark.LEFT_ELBOW to 13,
+    PoseLandmark.RIGHT_ELBOW to 14,
+    PoseLandmark.LEFT_WRIST to 15,
+    PoseLandmark.RIGHT_WRIST to 16,
+    PoseLandmark.LEFT_PINKY to 17,
+    PoseLandmark.RIGHT_PINKY to 18,
+    PoseLandmark.LEFT_INDEX to 19,
+    PoseLandmark.RIGHT_INDEX to 20,
+    PoseLandmark.LEFT_THUMB to 21,
+    PoseLandmark.RIGHT_THUMB to 22,
+    PoseLandmark.LEFT_HIP to 23,
+    PoseLandmark.RIGHT_HIP to 24,
+    PoseLandmark.LEFT_KNEE to 25,
+    PoseLandmark.RIGHT_KNEE to 26,
+    PoseLandmark.LEFT_ANKLE to 27,
+    PoseLandmark.RIGHT_ANKLE to 28,
+    PoseLandmark.LEFT_HEEL to 29,
+    PoseLandmark.RIGHT_HEEL to 30,
+    PoseLandmark.LEFT_FOOT_INDEX to 31,
+    PoseLandmark.RIGHT_FOOT_INDEX to 32,
+  )
+
   // ─── Exercise Config ────────────────────────────────────────
   private var exerciseConfig: ExerciseConfig? = null
 
@@ -46,8 +91,6 @@ class HybridPoseExercise : HybridNitroPoseExercisesSpec() {
   private var sessionStartTime: Long = System.currentTimeMillis()
   private var targetReps: Double = 0.0
   private var countdownSeconds: Double = 0.0
-  private var frameCount: Int = 0
-  private val processEveryNFrames: Int = 3
 
   // ─── Form Tracking ──────────────────────────────────────────
   private var lastFormFeedbackTime = mutableMapOf<String, Long>()
@@ -60,6 +103,10 @@ class HybridPoseExercise : HybridNitroPoseExercisesSpec() {
   // ─── Pose Tracking ──────────────────────────────────────────
   private var poseWasLost = false
 
+  // ─── Frame Throttle ─────────────────────────────────────────
+  private var frameCount: Int = 0
+  private val processEveryNFrames: Int = 3
+
   // ─── Callbacks ──────────────────────────────────────────────
   override var onRepComplete: ((data: RepData) -> Unit)? = null
   override var onPhaseChange: ((phase: ExercisePhase) -> Unit)? = null
@@ -77,31 +124,21 @@ class HybridPoseExercise : HybridNitroPoseExercisesSpec() {
   // ═══════════════════════════════════════════════════════════
 
   override fun initialize(modelPath: String): Promise<Unit> {
+    // No model file needed — ML Kit downloads/bundles its own model
     return Promise.async {
-      val context = NitroModules.applicationContext
-        ?: throw Error("No ApplicationContext set!")
-
-      val baseOptions = BaseOptions.builder()
-        .setModelAssetPath(modelPath)
-        .build()
-
-      val options = PoseLandmarkerOptions.builder()
-        .setBaseOptions(baseOptions)
-        .setRunningMode(RunningMode.IMAGE)
-        .setNumPoses(1)
-        .setMinPoseDetectionConfidence(0.5f)
-        .setMinPosePresenceConfidence(0.5f)
-        .setMinTrackingConfidence(0.5f)
+      val options = PoseDetectorOptions.Builder()
+        .setDetectorMode(PoseDetectorOptions.STREAM_MODE)
         .build()
 
-      poseLandmarker = PoseLandmarker.createFromOptions(context, options)
+      poseDetector = PoseDetection.getClient(options)
       isInitialized = true
+      println("[PoseExercise] Initialized with ML Kit Pose Detection (no model file needed)")
     }
   }
 
   override fun release() {
-    poseLandmarker?.close()
-    poseLandmarker = null
+    poseDetector?.close()
+    poseDetector = null
     isInitialized = false
     _status = SessionStatus.IDLE
     resetSession()
@@ -151,105 +188,88 @@ class HybridPoseExercise : HybridNitroPoseExercisesSpec() {
   }
 
   // ═══════════════════════════════════════════════════════════
-  // Frame Processing
+  // Frame Processing (ML Kit — async with cached results)
   // ═══════════════════════════════════════════════════════════
 
-override fun processFrame(frame: HybridFrameSpec) {
+  override fun processFrame(frame: HybridFrameSpec) {
     if (_status != SessionStatus.ACTIVE && _status != SessionStatus.COUNTDOWN) return
-    if (!isInitialized || poseLandmarker == null) return
+    if (!isInitialized || poseDetector == null) return
 
+    // Frame throttle
     frameCount++
     if (frameCount % processEveryNFrames != 0) return
 
+    // Get the ImageProxy from VisionCamera frame
+    val nativeFrame = frame as? NativeFrame ?: return
+    val imageProxy = nativeFrame.image ?: return
+
     try {
-        val nativeBuffer = frame.getNativeBuffer()
-        val width = frame.getWidth().toInt()
-        val height = frame.getHeight().toInt()
+      // ML Kit takes InputImage directly from ImageProxy — no color conversion needed
+      @androidx.camera.core.ExperimentalGetImage
+      val mediaImage: Image = imageProxy.image ?: return
+      val inputImage = InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees)
 
-        // On Android, nativeBuffer.pointer is an AHardwareBuffer*
-        // Convert to Bitmap via Android's HardwareBuffer API
-        val hardwareBuffer = android.hardware.HardwareBuffer.wrap(nativeBuffer.pointer)
-        val bitmap = Bitmap.wrapHardwareBuffer(hardwareBuffer, null)
-            ?: throw Exception("Failed to wrap HardwareBuffer to Bitmap")
+      // ML Kit is async — fire detection, cache result, use cached landmarks for current frame
+      poseDetector!!.process(inputImage)
+        .addOnSuccessListener { pose ->
+          val poseLandmarks = pose.allPoseLandmarks
 
-        // MediaPipe needs ARGB_8888, not hardware bitmap
-        val softBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, false)
+          if (poseLandmarks.isNotEmpty()) {
+            if (poseWasLost) {
+              poseWasLost = false
+              onPoseRegained?.invoke()
+            }
 
-        val mpImage = com.google.mediapipe.framework.image.BitmapImageBuilder(softBitmap).build()
-        val result = poseLandmarker!!.detect(mpImage)
+            // Build landmark array mapped to MediaPipe indices (34 slots)
+            val landmarkArray = Array(34) { Landmark(x = 0.0, y = 0.0, z = 0.0, visibility = 0.0) }
 
-        if (result.landmarks().isNotEmpty()) {
-            val poseLandmarks = result.landmarks()[0]
+            val imageWidth = inputImage.width.toDouble()
+            val imageHeight = inputImage.height.toDouble()
 
-            if (poseWasLost) {
-                poseWasLost = false
-                onPoseRegained?.invoke()
+            for (poseLandmark in poseLandmarks) {
+              val mediaPipeIndex = mlKitToMediaPipeMap[poseLandmark.landmarkType] ?: continue
+              if (mediaPipeIndex >= 34) continue
+
+              // Normalize coordinates to 0-1 range
+              landmarkArray[mediaPipeIndex] = Landmark(
+                x = poseLandmark.position.x.toDouble() / imageWidth,
+                y = poseLandmark.position.y.toDouble() / imageHeight,
+                z = poseLandmark.position3D.z.toDouble(),
+                visibility = poseLandmark.inFrameLikelihood.toDouble()
+              )
             }
 
-            _landmarks = poseLandmarks.map { lm ->
-                Landmark(
-                    x = lm.x().toDouble(),
-                    y = lm.y().toDouble(),
-                    z = lm.z().toDouble(),
-                    visibility = (lm.visibility().orElse(0f)).toDouble()
-                )
-            }.toTypedArray()
-
-            if (_status == SessionStatus.ACTIVE) {
-                processExerciseLogic()
+            synchronized(landmarkLock) {
+              cachedLandmarks = landmarkArray
             }
-        } else {
+          } else {
             if (!poseWasLost) {
-                poseWasLost = true
-                onPoseLost?.invoke()
+              poseWasLost = true
+              onPoseLost?.invoke()
+            }
+            synchronized(landmarkLock) {
+              cachedLandmarks = emptyArray()
             }
-            _landmarks = emptyArray()
+          }
+        }
+        .addOnFailureListener { e ->
+          println("[PoseExercise] ML Kit error: ${e.message}")
         }
 
-        softBitmap.recycle()
-        bitmap.recycle()
-        nativeBuffer.release()
-
-    } catch (e: Exception) {
-        // MediaPipe detection failed — skip this frame
-    }
-}
-
-  // ═══════════════════════════════════════════════════════════
-  // ImageProxy to Bitmap conversion
-  // ═══════════════════════════════════════════════════════════
-
-  private fun imageProxyToBitmap(imageProxy: androidx.camera.core.ImageProxy): Bitmap {
-    val buffer = imageProxy.planes[0].buffer
-    val bytes = ByteArray(buffer.remaining())
-    buffer.get(bytes)
-
-    val yuvImage = android.graphics.YuvImage(
-      bytes,
-      android.graphics.ImageFormat.NV21,
-      imageProxy.width,
-      imageProxy.height,
-      null
-    )
+      // Use cached landmarks for exercise logic (from previous frame's detection)
+      val currentLandmarks: Array<Landmark>
+      synchronized(landmarkLock) {
+        currentLandmarks = cachedLandmarks.copyOf()
+      }
 
-    val out = java.io.ByteArrayOutputStream()
-    yuvImage.compressToJpeg(
-      android.graphics.Rect(0, 0, imageProxy.width, imageProxy.height),
-      100,
-      out
-    )
+      _landmarks = currentLandmarks
 
-    val jpegBytes = out.toByteArray()
-    val bitmap = android.graphics.BitmapFactory.decodeByteArray(jpegBytes, 0, jpegBytes.size)
+      if (currentLandmarks.isNotEmpty() && _status == SessionStatus.ACTIVE) {
+        processExerciseLogic()
+      }
 
-    // Apply rotation if needed
-    val rotation = imageProxy.imageInfo.rotationDegrees
-    return if (rotation != 0) {
-      val matrix = Matrix()
-      matrix.postRotate(rotation.toFloat())
-      Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
-    } else {
-      bitmap
+    } catch (e: Exception) {
+      println("[PoseExercise] Frame processing error: ${e.message}")
     }
   }
 
@@ -261,7 +281,6 @@ override fun processFrame(frame: HybridFrameSpec) {
     val config = exerciseConfig ?: return
     if (_landmarks.isEmpty()) return
 
-    // 1. Calculate all angles
     val currentAngles = mutableMapOf<String, Double>()
     val angleSnapshots = mutableListOf<AngleSnapshot>()
 
@@ -272,6 +291,9 @@ override fun processFrame(frame: HybridFrameSpec) {
 
       if (a >= _landmarks.size || b >= _landmarks.size || c >= _landmarks.size) continue
 
+      // Only calculate if all three landmarks have reasonable confidence
+      if (_landmarks[a].visibility < 0.3 || _landmarks[b].visibility < 0.3 || _landmarks[c].visibility < 0.3) continue
+
       val angle = calculateAngle(_landmarks[a], _landmarks[b], _landmarks[c])
       currentAngles[angleDef.name] = angle
       angleSnapshots.add(AngleSnapshot(name = angleDef.name, value = angle))
@@ -279,7 +301,6 @@ override fun processFrame(frame: HybridFrameSpec) {
 
     repAngleSnapshots = angleSnapshots.toTypedArray()
 
-    // 2. Determine current phase
     val detectedPhase = determinePhase(currentAngles, config)
 
     if (detectedPhase != _currentPhase && detectedPhase != ExercisePhase.UNKNOWN) {
@@ -288,10 +309,8 @@ override fun processFrame(frame: HybridFrameSpec) {
       handlePhaseTransition(detectedPhase, config)
     }
 
-    // 3. Check form rules
     checkFormRules(currentAngles, config)
 
-    // 4. Handle hold-based exercises
     if (config.type == ExerciseType.HOLD) {
       handleHoldProgress(currentAngles, config)
     }
@@ -345,37 +364,37 @@ override fun processFrame(frame: HybridFrameSpec) {
     if (phaseHistory.size >= repSeq.size) {
       val tail = phaseHistory.takeLast(repSeq.size)
 
-    if (tail == repSeq.toList()) {
+      if (tail == repSeq.toList()) {
         val now = System.currentTimeMillis()
         val repDuration = (now - repStartTime).toDouble()
 
         // Minimum 800ms per rep
         if (repDuration < 800) {
-            phaseHistory.clear()
-            phaseHistory.add(newPhase)
-            return
+          phaseHistory.clear()
+          phaseHistory.add(newPhase)
+          return
         }
 
         // Don't count rep if form is terrible
         if (repFormScore <= 30) {
-            onFormFeedback?.invoke(FormFeedback(
-                ruleName = "poorForm",
-                message = "Fix your form before continuing",
-                severity = FormSeverity.ERROR
-            ))
-            repFormScore = 100.0
-            phaseHistory.clear()
-            phaseHistory.add(newPhase)
-            return
+          onFormFeedback?.invoke(FormFeedback(
+            ruleName = "poorForm",
+            message = "Fix your form before continuing",
+            severity = FormSeverity.ERROR
+          ))
+          repFormScore = 100.0
+          phaseHistory.clear()
+          phaseHistory.add(newPhase)
+          return
         }
 
         _repCount += 1.0
 
         val repData = RepData(
-            repNumber = _repCount,
-            durationMs = repDuration,
-            formScore = repFormScore,
-            angles = repAngleSnapshots
+          repNumber = _repCount,
+          durationMs = repDuration,
+          formScore = repFormScore,
+          angles = repAngleSnapshots
         )
 
         allRepDurations.add(repDuration)
@@ -389,9 +408,9 @@ override fun processFrame(frame: HybridFrameSpec) {
         phaseHistory.add(newPhase)
 
         if (targetReps > 0 && _repCount >= targetReps) {
-            completeSession()
+          completeSession()
         }
-    }
+      }
     }
 
     val maxHistory = repSeq.size * 2
@@ -536,5 +555,9 @@ override fun processFrame(frame: HybridFrameSpec) {
     poseWasLost = false
     targetReps = 0.0
     countdownSeconds = 0.0
+    frameCount = 0
+    synchronized(landmarkLock) {
+      cachedLandmarks = emptyArray()
+    }
   }
 }