react-native-facefusion 0.1.0

package/android/src/main/java/com/facefusion/PreviewSurfaceHolder.kt

@@ -0,0 +1,104 @@
+package com.facefusion
+
+import android.graphics.Bitmap
+import android.graphics.Canvas
+import android.graphics.Rect
+import android.view.Surface
+
+/**
+ * The live-preview sink for [PhotoSwap] and [VideoSwap]: a global holder for whichever
+ * [FacefusionPreviewView] is currently mounted, if any.
+ *
+ * **Why a singleton rather than a parameter threaded through the swap call.** `g_pipe` is
+ * already a single C++ global — only one swap can run at a time ([PipeGuard]) — so there is
+ * never more than one frame stream to preview. A JS-side prop or a second TurboModule call
+ * to "attach" a surface would need to cross the bridge with a view reference; a plain object
+ * both sides can reach avoids that, and mirrors how [NativePipe] itself is one process-wide
+ * object rather than an instance per swap.
+ *
+ * **Never touches [NativePipe] or the vendored C++.** `bgr`/`w`/`h` here are exactly the
+ * bytes [VideoSwap] and [PhotoSwap] already hold in JVM memory after [NativePipe.processFrame]
+ * returns — this only has to get them onto a [Surface]'s [Canvas], which is a plain Android
+ * API. That is what keeps this out of the "never pass frames across the JS bridge" rule: no
+ * pixel ever reaches JS, only the mounted `<FacefusionPreview />` view does, and rendering
+ * happens entirely on the native side of it.
+ */
+object PreviewSurfaceHolder {
+
+  // Caps preview redraws independent of how fast the swap loop itself runs -- a 720p frame
+  // decoded to a Bitmap and blitted to a Canvas is not free, and the swap loop's own pace
+  // (tens of fps) is faster than a screen needs to look "live". ~15 fps is smooth to the eye
+  // and cheap enough not to be the thing that slows the swap down.
+  private const val MIN_INTERVAL_NANOS = 66_000_000L
+
+  @Volatile
+  private var surface: Surface? = null
+
+  @Volatile
+  private var lastDrawNanos = 0L
+
+  /** Called by [FacefusionPreviewView] once its [Surface] exists. */
+  fun attach(newSurface: Surface) {
+    surface = newSurface
+    lastDrawNanos = 0L
+  }
+
+  /** Called by [FacefusionPreviewView] when its [Surface] is torn down. Idempotent. */
+  fun detach(oldSurface: Surface) {
+    // Only clear if it's still the same surface -- a fast unmount+remount (view recycling)
+    // could otherwise have a stale detach() race ahead of the new attach() and wipe it.
+    if (surface === oldSurface) surface = null
+  }
+
+  /**
+   * Draws one BGR frame, letterboxed to fit the attached surface. A no-op — one [Volatile]
+   * read — when nothing is mounted, so [VideoSwap]'s loop pays nothing when no preview is
+   * showing. Also a no-op inside the throttle window.
+   *
+   * Safe to call from any thread; [VideoSwap] and [PhotoSwap] both call it from their own
+   * worker thread, never the UI thread.
+   */
+  fun draw(bgr: ByteArray, w: Int, h: Int) {
+    val target = surface ?: return
+    if (!target.isValid) return
+
+    val now = System.nanoTime()
+    if (now - lastDrawNanos < MIN_INTERVAL_NANOS) return
+
+    val argb = com.facefusion.mobile.NativePipe.bgrToArgb(bgr, w, h, w, h)
+    val bitmap = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888)
+    bitmap.setPixels(argb, 0, w, 0, 0, w, h)
+
+    // lockCanvas/unlockCanvasAndPost can throw if the surface is torn down concurrently with
+    // this call (SurfaceHolder.Callback runs on the UI thread, this runs on the swap
+    // worker) -- caught rather than synchronized against, since a dropped preview frame
+    // during teardown is harmless and a lock here would block the swap loop on the UI thread.
+    try {
+      val canvas: Canvas = target.lockCanvas(null)
+      try {
+        val dst = fitRect(w, h, canvas.width, canvas.height)
+        canvas.drawColor(android.graphics.Color.BLACK)
+        canvas.drawBitmap(bitmap, null, dst, null)
+      } finally {
+        target.unlockCanvasAndPost(canvas)
+      }
+      lastDrawNanos = now
+    } catch (_: Exception) {
+      // Surface gone or going -- next attach() (or the next frame, if it recovers) picks
+      // back up. Nothing here is worth crashing a swap over.
+    } finally {
+      bitmap.recycle()
+    }
+  }
+
+  /** Centers a `srcW`x`srcH` rect inside `dstW`x`dstH`, preserving aspect ratio. */
+  private fun fitRect(srcW: Int, srcH: Int, dstW: Int, dstH: Int): Rect {
+    if (srcW <= 0 || srcH <= 0 || dstW <= 0 || dstH <= 0) return Rect(0, 0, dstW, dstH)
+    val scale = minOf(dstW.toFloat() / srcW, dstH.toFloat() / srcH)
+    val w = (srcW * scale).toInt()
+    val h = (srcH * scale).toInt()
+    val left = (dstW - w) / 2
+    val top = (dstH - h) / 2
+    return Rect(left, top, left + w, top + h)
+  }
+}

package/android/src/main/java/com/facefusion/SourceFaces.kt

@@ -0,0 +1,125 @@
+package com.facefusion
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import com.facefusion.mobile.NativePipe
+
+/**
+ * One face found in an image.
+ *
+ * The box is in the pixel coordinates of **the bitmap that was actually analysed**, which
+ * is not the file on disk: a source photo is decoded capped at [BitmapDecode.SOURCE_MAX],
+ * a photo target at [BitmapDecode.TARGET_MAX], and a video target not capped at all (its
+ * own frame size). That is deliberate — it is the same space [PhotoSwap]/[VideoSwap] crop
+ * in, which is what lets a box picked here be handed straight back as
+ * `SwapOptions.targetFaceBox` with no conversion.
+ *
+ * It also means a caller cannot work out the scale on their own, so [imageWidth] and
+ * [imageHeight] travel with every face. Without them, drawing this box over the original
+ * file — the obvious thing to do with a detection API — silently lands in the wrong place
+ * on any photo bigger than the cap, and there is no way to detect that from the outside.
+ */
+data class DetectedFace(
+  val left: Float,
+  val top: Float,
+  val right: Float,
+  val bottom: Float,
+  /** Detector confidence, `0..1`. */
+  val score: Float,
+  /** Width of the analysed bitmap, the space [left]/[right] are in. */
+  val imageWidth: Int,
+  /** Height of the analysed bitmap, the space [top]/[bottom] are in. */
+  val imageHeight: Int,
+)
+
+/**
+ * Detects every face in a source photo, and crops to one of them for [PhotoSwap]/
+ * [VideoSwap] to use as the identity instead of always "the largest" — see ADR-0012
+ * (`docs/06-decisions/0012-source-face-picker-patches-ffjni.md`).
+ *
+ * [NativePipe.analyseFaces] is the one native symbol in this project that is not
+ * upstream's own — a small, tracked patch (`docs/02-upstream.md` "Patches"), because
+ * reaching the already-warm pipeline for a detect-only pass needed a new export inside
+ * `ffjni.cpp` itself. Everything below this point is our own layer, same as the rest of
+ * the app, and needed no further native changes: choosing a face is done by cropping the
+ * source bitmap and calling the existing, unmodified `setSource`, not by a new native
+ * "pick face N" entry point.
+ */
+object SourceFaces {
+
+  /** Mirrors [PhotoSwap.ModelsMissing] — same required models a swap needs. */
+  class ModelsMissing(message: String) : Exception(message)
+
+  /** How much wider/taller than the detected box to crop, so `setSource`'s own detector
+   *  sees the same kind of context it would in an uncropped photo instead of a box drawn
+   *  exactly to the model's own edges — a hair too tight can lose the chin or forehead. */
+  private const val MARGIN = 0.4f
+
+  fun detect(context: Context, sourcePath: String, cfg: SwapConfig): List<DetectedFace> {
+    NativePipe.loadError?.let {
+      throw IllegalStateException("libffnative.so did not load: $it")
+    }
+
+    val tier = ModelPaths.tier(context)
+    val missing = ModelPaths.missing(context, tier)
+    if (missing.isNotEmpty()) {
+      throw ModelsMissing(
+        "Models missing for $tier: ${missing.joinToString(", ")} — call downloadModels() first"
+      )
+    }
+
+    val bitmap = decode(sourcePath)
+    return PipeGuard.run(context, tier, cfg) {
+      val bgr = NativePipe.argbToBgr(pixelsOf(bitmap), bitmap.width, bitmap.height)
+      val flat = NativePipe.analyseFaces(bgr, bitmap.width, bitmap.height)
+        ?: throw IllegalStateException(NativePipe.lastError())
+      (flat.indices step 5).map { i ->
+        DetectedFace(
+          flat[i],
+          flat[i + 1],
+          flat[i + 2],
+          flat[i + 3],
+          flat[i + 4],
+          bitmap.width,
+          bitmap.height,
+        )
+      }
+    }
+  }
+
+  /**
+   * Crops [bitmap] to the box `[left, top, right, bottom]` (as returned by [detect] /
+   * [NativePipe.analyseFaces]), expanded by [MARGIN] on every side and clamped to the
+   * bitmap's own bounds. The result is what gets handed to [NativePipe.setSource] — since
+   * it contains only (or overwhelmingly) the chosen face, `setSource`'s own unmodified
+   * "pick the largest face" logic naturally lands on it.
+   */
+  fun cropToFace(bitmap: Bitmap, box: FloatArray): Bitmap {
+    require(box.size == 4) { "box must be [left, top, right, bottom]" }
+    val (boxLeft, boxTop, boxRight, boxBottom) = box
+    val w = boxRight - boxLeft
+    val h = boxBottom - boxTop
+    val left = (boxLeft - w * MARGIN).toInt().coerceIn(0, bitmap.width - 1)
+    val top = (boxTop - h * MARGIN).toInt().coerceIn(0, bitmap.height - 1)
+    val right = (boxRight + w * MARGIN).toInt().coerceIn(left + 1, bitmap.width)
+    val bottom = (boxBottom + h * MARGIN).toInt().coerceIn(top + 1, bitmap.height)
+    return Bitmap.createBitmap(bitmap, left, top, right - left, bottom - top)
+  }
+
+  // Duplicated from PhotoSwap rather than shared — see that file's note on the same choice.
+  /**
+   * Capped at [BitmapDecode.SOURCE_MAX] — **the same cap [PhotoSwap] decodes the source
+   * with, and that is load-bearing.** The boxes this returns are in the decoded image's
+   * pixel coordinates and come back as `SwapOptions.sourceFaceBox`; if the two paths
+   * decoded at different scales, a picked face would crop the wrong region.
+   */
+  private fun decode(path: String): Bitmap =
+    BitmapDecode.decode(path, BitmapDecode.SOURCE_MAX)
+
+  private fun pixelsOf(bitmap: Bitmap): IntArray {
+    val pixels = IntArray(bitmap.width * bitmap.height)
+    bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height)
+    return pixels
+  }
+}

package/android/src/main/java/com/facefusion/SwapConfig.kt

@@ -0,0 +1,31 @@
+package com.facefusion
+
+/**
+ * JS-tunable knobs for a swap, mirroring `ffpipe::Config`'s defaults for the `hyperswap`
+ * swapper ([ModelPaths.SWAPPER]) field for field.
+ *
+ * [maskPadding] is `[top, right, bottom, left]`, each `0..100`; out-of-range values are
+ * clamped again on the C++ side (`initEx` in `ffjni.cpp`), so this is a convenience default,
+ * not the only safety net.
+ */
+data class SwapConfig(
+  val swapperWeight: Float = 0.5f,
+  val maskBlur: Float = 0.3f,
+  val maskPadding: List<Int> = listOf(0, 0, 0, 0),
+  val detectorScore: Float = 0.5f,
+  val landmarkerScore: Float = 0.5f,
+  val pixelBoost: Int = 1,
+  val largestFaceOnly: Boolean = false,
+  val faceEnhance: Boolean = false,
+  val faceEnhancerBlend: Float = 0.8f,
+) {
+  /**
+   * Identifies this config for [PipeGuard]'s "does the pipeline need reloading" check.
+   * A plain string rather than relying on `data class` equality because the values that
+   * matter are the ones that cross into `initEx`, not Kotlin identity.
+   */
+  fun signature(): String = listOf(
+    swapperWeight, maskBlur, maskPadding.joinToString(","), detectorScore,
+    landmarkerScore, pixelBoost, largestFaceOnly, faceEnhance, faceEnhancerBlend,
+  ).joinToString("|")
+}

package/android/src/main/java/com/facefusion/TargetFaces.kt

@@ -0,0 +1,88 @@
+package com.facefusion
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.media.MediaMetadataRetriever
+import com.facefusion.mobile.NativePipe
+
+/**
+ * Detects every face in a *target* — a photo, or the first frame of a video — so a UI can
+ * let the user pick which one gets swapped, instead of every face found (or just the
+ * largest, via [SwapConfig.largestFaceOnly]). Pass the chosen box back as
+ * `targetFaceBox`; [PhotoSwap]/[VideoSwap] crop to it with [FaceCrop] the same way
+ * [SourceFaces] crops for `setSource`, so this needed no native change either.
+ *
+ * Structurally a twin of [SourceFaces.detect] — duplicated rather than shared, same as
+ * [VideoSwap]'s own note on why it doesn't share code with [PhotoSwap]: each is a small,
+ * self-contained entry point, and the one real difference (this one also has to handle a
+ * video target) doesn't belong in the source-only file.
+ */
+object TargetFaces {
+
+  /** Mirrors [PhotoSwap.ModelsMissing] — same required models a swap needs. */
+  class ModelsMissing(message: String) : Exception(message)
+
+  fun detect(context: Context, targetPath: String, cfg: SwapConfig): List<DetectedFace> {
+    NativePipe.loadError?.let {
+      throw IllegalStateException("libffnative.so did not load: $it")
+    }
+
+    val tier = ModelPaths.tier(context)
+    val missing = ModelPaths.missing(context, tier)
+    if (missing.isNotEmpty()) {
+      throw ModelsMissing(
+        "Models missing for $tier: ${missing.joinToString(", ")} — call downloadModels() first"
+      )
+    }
+
+    val bitmap = decode(targetPath)
+    return PipeGuard.run(context, tier, cfg) {
+      val bgr = NativePipe.argbToBgr(pixelsOf(bitmap), bitmap.width, bitmap.height)
+      val flat = NativePipe.analyseFaces(bgr, bitmap.width, bitmap.height)
+        ?: throw IllegalStateException(NativePipe.lastError())
+      (flat.indices step 5).map { i ->
+        DetectedFace(
+          flat[i],
+          flat[i + 1],
+          flat[i + 2],
+          flat[i + 3],
+          flat[i + 4],
+          bitmap.width,
+          bitmap.height,
+        )
+      }
+    }
+  }
+
+  /** A photo decodes directly; a video falls back to its first frame, already rotated
+   *  upright by [MediaMetadataRetriever] itself — the same orientation [VideoSwap]'s own
+   *  loop presents to `processFrame`, which is what makes a box detected here line up with
+   *  what that loop sees per frame, with no rotation math needed on this side at all. */
+  private fun decode(path: String): Bitmap {
+    // Capped at [BitmapDecode.TARGET_MAX] -- the same cap [PhotoSwap] decodes a photo
+    // target with, which is what keeps a box picked here aligned with what the swap
+    // actually crops. `decodeOrNull`, not `decode`, because "not a still image" is how the
+    // video branch below is reached.
+    BitmapDecode.decodeOrNull(path, BitmapDecode.TARGET_MAX)?.let { return it }
+
+    // A video frame arrives at the clip's own resolution, uncapped -- and deliberately so:
+    // VideoSwap's loop processes frames at that resolution too, so capping here would
+    // misalign every box against the frames the swap actually sees. Codec limits keep this
+    // bounded in practice where a photo's megapixels are not.
+    val retriever = MediaMetadataRetriever()
+    try {
+      retriever.setDataSource(path)
+      return retriever.frameAtTime
+        ?: throw IllegalArgumentException("Could not decode image or video frame: $path")
+    } finally {
+      retriever.release()
+    }
+  }
+
+  private fun pixelsOf(bitmap: Bitmap): IntArray {
+    val pixels = IntArray(bitmap.width * bitmap.height)
+    bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height)
+    return pixels
+  }
+}