react-native-facefusion 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/LICENSE +20 -0
  2. package/README.md +259 -0
  3. package/android/build.gradle +129 -0
  4. package/android/src/main/AndroidManifest.xml +66 -0
  5. package/android/src/main/cpp/CMakeLists.txt +73 -0
  6. package/android/src/main/java/com/facefusion/BitmapDecode.kt +112 -0
  7. package/android/src/main/java/com/facefusion/ContentGate.kt +86 -0
  8. package/android/src/main/java/com/facefusion/DeviceProbe.kt +158 -0
  9. package/android/src/main/java/com/facefusion/FaceCrop.kt +57 -0
  10. package/android/src/main/java/com/facefusion/FacefusionModule.kt +396 -0
  11. package/android/src/main/java/com/facefusion/FacefusionPackage.kt +37 -0
  12. package/android/src/main/java/com/facefusion/FacefusionPreviewView.kt +30 -0
  13. package/android/src/main/java/com/facefusion/FacefusionPreviewViewManager.kt +25 -0
  14. package/android/src/main/java/com/facefusion/GallerySave.kt +82 -0
  15. package/android/src/main/java/com/facefusion/ModelDownload.kt +351 -0
  16. package/android/src/main/java/com/facefusion/ModelPaths.kt +120 -0
  17. package/android/src/main/java/com/facefusion/PhotoSwap.kt +125 -0
  18. package/android/src/main/java/com/facefusion/PipeGuard.kt +74 -0
  19. package/android/src/main/java/com/facefusion/PreviewSurfaceHolder.kt +104 -0
  20. package/android/src/main/java/com/facefusion/SourceFaces.kt +125 -0
  21. package/android/src/main/java/com/facefusion/SwapConfig.kt +31 -0
  22. package/android/src/main/java/com/facefusion/TargetFaces.kt +88 -0
  23. package/android/src/main/java/com/facefusion/VideoSwap.kt +501 -0
  24. package/android/src/main/java/com/facefusion/VideoSwapService.kt +85 -0
  25. package/android/src/main/java/com/facefusion/mobile/NativePipe.kt +192 -0
  26. package/lib/module/FacefusionPreview.js +6 -0
  27. package/lib/module/FacefusionPreview.js.map +1 -0
  28. package/lib/module/FacefusionPreview.native.js +15 -0
  29. package/lib/module/FacefusionPreview.native.js.map +1 -0
  30. package/lib/module/NativeFacefusion.js +53 -0
  31. package/lib/module/NativeFacefusion.js.map +1 -0
  32. package/lib/module/detectSourceFaces.js +6 -0
  33. package/lib/module/detectSourceFaces.js.map +1 -0
  34. package/lib/module/detectSourceFaces.native.js +15 -0
  35. package/lib/module/detectSourceFaces.native.js.map +1 -0
  36. package/lib/module/detectTargetFaces.js +6 -0
  37. package/lib/module/detectTargetFaces.js.map +1 -0
  38. package/lib/module/detectTargetFaces.native.js +13 -0
  39. package/lib/module/detectTargetFaces.native.js.map +1 -0
  40. package/lib/module/index.js +11 -0
  41. package/lib/module/index.js.map +1 -0
  42. package/lib/module/models.js +16 -0
  43. package/lib/module/models.js.map +1 -0
  44. package/lib/module/models.native.js +43 -0
  45. package/lib/module/models.native.js.map +1 -0
  46. package/lib/module/package.json +1 -0
  47. package/lib/module/probeDevice.js +6 -0
  48. package/lib/module/probeDevice.js.map +1 -0
  49. package/lib/module/probeDevice.native.js +14 -0
  50. package/lib/module/probeDevice.native.js.map +1 -0
  51. package/lib/module/saveToGallery.js +6 -0
  52. package/lib/module/saveToGallery.js.map +1 -0
  53. package/lib/module/saveToGallery.native.js +15 -0
  54. package/lib/module/saveToGallery.native.js.map +1 -0
  55. package/lib/module/swapPhoto.js +6 -0
  56. package/lib/module/swapPhoto.js.map +1 -0
  57. package/lib/module/swapPhoto.native.js +16 -0
  58. package/lib/module/swapPhoto.native.js.map +1 -0
  59. package/lib/module/swapVideo.js +12 -0
  60. package/lib/module/swapVideo.js.map +1 -0
  61. package/lib/module/swapVideo.native.js +29 -0
  62. package/lib/module/swapVideo.native.js.map +1 -0
  63. package/lib/typescript/package.json +1 -0
  64. package/lib/typescript/src/FacefusionPreview.d.ts +4 -0
  65. package/lib/typescript/src/FacefusionPreview.d.ts.map +1 -0
  66. package/lib/typescript/src/FacefusionPreview.native.d.ts +142 -0
  67. package/lib/typescript/src/FacefusionPreview.native.d.ts.map +1 -0
  68. package/lib/typescript/src/NativeFacefusion.d.ts +291 -0
  69. package/lib/typescript/src/NativeFacefusion.d.ts.map +1 -0
  70. package/lib/typescript/src/detectSourceFaces.d.ts +3 -0
  71. package/lib/typescript/src/detectSourceFaces.d.ts.map +1 -0
  72. package/lib/typescript/src/detectSourceFaces.native.d.ts +11 -0
  73. package/lib/typescript/src/detectSourceFaces.native.d.ts.map +1 -0
  74. package/lib/typescript/src/detectTargetFaces.d.ts +3 -0
  75. package/lib/typescript/src/detectTargetFaces.d.ts.map +1 -0
  76. package/lib/typescript/src/detectTargetFaces.native.d.ts +9 -0
  77. package/lib/typescript/src/detectTargetFaces.native.d.ts.map +1 -0
  78. package/lib/typescript/src/index.d.ts +11 -0
  79. package/lib/typescript/src/index.d.ts.map +1 -0
  80. package/lib/typescript/src/models.d.ts +7 -0
  81. package/lib/typescript/src/models.d.ts.map +1 -0
  82. package/lib/typescript/src/models.native.d.ts +31 -0
  83. package/lib/typescript/src/models.native.d.ts.map +1 -0
  84. package/lib/typescript/src/probeDevice.d.ts +3 -0
  85. package/lib/typescript/src/probeDevice.d.ts.map +1 -0
  86. package/lib/typescript/src/probeDevice.native.d.ts +10 -0
  87. package/lib/typescript/src/probeDevice.native.d.ts.map +1 -0
  88. package/lib/typescript/src/saveToGallery.d.ts +2 -0
  89. package/lib/typescript/src/saveToGallery.d.ts.map +1 -0
  90. package/lib/typescript/src/saveToGallery.native.d.ts +9 -0
  91. package/lib/typescript/src/saveToGallery.native.d.ts.map +1 -0
  92. package/lib/typescript/src/swapPhoto.d.ts +3 -0
  93. package/lib/typescript/src/swapPhoto.d.ts.map +1 -0
  94. package/lib/typescript/src/swapPhoto.native.d.ts +12 -0
  95. package/lib/typescript/src/swapPhoto.native.d.ts.map +1 -0
  96. package/lib/typescript/src/swapVideo.d.ts +6 -0
  97. package/lib/typescript/src/swapVideo.d.ts.map +1 -0
  98. package/lib/typescript/src/swapVideo.native.d.ts +20 -0
  99. package/lib/typescript/src/swapVideo.native.d.ts.map +1 -0
  100. package/package.json +132 -0
  101. package/patches/ffjni-analyse-faces.patch +63 -0
  102. package/scripts/check-tarball.sh +68 -0
  103. package/scripts/fetch-upstream.sh +91 -0
  104. package/src/FacefusionPreview.native.tsx +18 -0
  105. package/src/FacefusionPreview.tsx +9 -0
  106. package/src/NativeFacefusion.ts +324 -0
  107. package/src/detectSourceFaces.native.tsx +17 -0
  108. package/src/detectSourceFaces.tsx +10 -0
  109. package/src/detectTargetFaces.native.tsx +15 -0
  110. package/src/detectTargetFaces.tsx +10 -0
  111. package/src/index.tsx +25 -0
  112. package/src/models.native.tsx +45 -0
  113. package/src/models.tsx +23 -0
  114. package/src/probeDevice.native.tsx +13 -0
  115. package/src/probeDevice.tsx +7 -0
  116. package/src/saveToGallery.native.tsx +16 -0
  117. package/src/saveToGallery.tsx +9 -0
  118. package/src/swapPhoto.native.tsx +20 -0
  119. package/src/swapPhoto.tsx +12 -0
  120. package/src/swapVideo.native.tsx +40 -0
  121. package/src/swapVideo.tsx +31 -0
  122. package/third_party/facefusion-mobile/NOTICE +49 -0
@@ -0,0 +1,501 @@
1
+ package com.facefusion
2
+
3
+ import android.content.Context
4
+ import android.graphics.Bitmap
5
+ import android.graphics.BitmapFactory
6
+ import android.media.Image
7
+ import android.media.MediaCodec
8
+ import android.media.MediaCodecInfo
9
+ import android.media.MediaExtractor
10
+ import android.media.MediaFormat
11
+ import android.media.MediaMuxer
12
+ import com.facefusion.mobile.NativePipe
13
+ import java.io.File
14
+ import java.io.IOException
15
+ import java.nio.ByteBuffer
16
+
17
+ /** A tick of progress through [VideoSwap.run]. */
18
+ data class VideoSwapProgress(
19
+ /** Frames decoded, swapped and encoded so far. */
20
+ val frameIndex: Int,
21
+ /** Estimated from the container's duration and frame rate — not exact, since a
22
+ * variable-frame-rate source has no true count until the last frame arrives. `0` when
23
+ * it could not be estimated. */
24
+ val estimatedFrameCount: Int,
25
+ /** Wall-clock frames/second across the loop so far, not upstream's per-graph figure. */
26
+ val fps: Double,
27
+ )
28
+
29
+ /** The result of one video swap. */
30
+ data class VideoSwapResult(
31
+ val outputPath: String,
32
+ /** Frames decoded, swapped and encoded — the real count, unlike [VideoSwapProgress]'s
33
+ * estimate. */
34
+ val frameCount: Int,
35
+ /** Of [frameCount], how many had at least one face swapped. */
36
+ val faceFrameCount: Int,
37
+ /** The tier that ran — see [ModelPaths.tier]. */
38
+ val tier: String,
39
+ /** Wall-clock frames/second across the decode-swap-encode loop only: excludes opening
40
+ * the extractor/codecs and the final mux/audio-copy pass. Measured, per rule 11 — never
41
+ * reported as upstream's per-graph figure, which this is not. */
42
+ val fps: Double,
43
+ /** Whether the source clip had an audio track, which — if true — was copied to
44
+ * [outputPath] unmodified. */
45
+ val hasAudio: Boolean,
46
+ )
47
+
48
+ /**
49
+ * Swaps a face from [VideoSwap.run]'s `sourcePath` into every frame of a video, writing the
50
+ * result to `outputPath`.
51
+ *
52
+ * Shaped like [PhotoSwap]: paths in, path out, and this is the only other place in the
53
+ * Kotlin layer that touches pixels. What's different from a still is everything about
54
+ * getting pixels in and out of [NativePipe] — MediaExtractor → MediaCodec(decode) →
55
+ * [NativePipe.processFrame] → MediaCodec(encode) → MediaMuxer, all synchronous on the
56
+ * caller's thread, one frame at a time. The three converters this needed —
57
+ * [NativePipe.yuvToBgr], [NativePipe.rotateBgr], [NativePipe.bgrToImagePlanes] — were
58
+ * already exported by upstream's vendored, unmodified `ffjni.cpp` (ADR-0004): nothing here
59
+ * changes the C++, only wires symbols that were sitting unused since Phase 3.
60
+ *
61
+ * **Two passes, not one interleaved stream.** The whole video track is decoded, swapped and
62
+ * encoded first, buffering the (small — a few MB for a short clip) encoded output in memory;
63
+ * only then is the muxer opened, because the muxer needs the encoder's *actual* output
64
+ * `MediaFormat` (only known once the encoder starts producing) before any track can be
65
+ * added, and needs every track added before `start()`. Interleaving a live audio copy with
66
+ * that would mean holding audio samples until the video format shows up. Buffering encoded
67
+ * video instead is simpler and correct for the clip lengths this project targets; a
68
+ * multi-minute clip would want a streaming interleave instead — not needed yet.
69
+ */
70
+ object VideoSwap {
71
+
72
+ /** Mirrors [PhotoSwap.ModelsMissing] — the tier's required models are not on disk. */
73
+ class ModelsMissing(message: String) : Exception(message)
74
+
75
+ /** Thrown when [cancel] was called mid-run. The partial [outputPath] is deleted first. */
76
+ class Cancelled : IOException("Cancelled")
77
+
78
+ private const val TIMEOUT_US = 10_000L
79
+ // Bounds the busy-wait for a free codec input buffer. At 10 ms per poll this is 5 s,
80
+ // generous for a codec that is merely busy and a hard stop for one that is stuck --
81
+ // without a bound, a stalled codec would hang the job past where [cancel] can reach it,
82
+ // since the check for it lives in this same wait.
83
+ private const val MAX_STALL_POLLS = 500
84
+
85
+ @Volatile
86
+ private var cancelled = false
87
+
88
+ /** Asks the run in flight to stop. Idempotent, and safe to call when nothing is running. */
89
+ fun cancel() {
90
+ cancelled = true
91
+ }
92
+
93
+ fun run(
94
+ context: Context,
95
+ sourcePath: String,
96
+ targetPath: String,
97
+ outputPath: String,
98
+ cfg: SwapConfig,
99
+ /** `[left, top, right, bottom]` from [SourceFaces.detect], or `null` for the default
100
+ * "largest face in the source" that [NativePipe.setSource] already picks on its own. */
101
+ sourceFaceBox: FloatArray? = null,
102
+ /** `[left, top, right, bottom]` from [TargetFaces.detect], in the clip's upright
103
+ * (post-rotation-correction) coordinate space, or `null` to swap every face found in
104
+ * every frame, same as before this option existed. Picked once and held fixed for the
105
+ * whole clip — Saad's own call: re-detecting the face every frame to track it as it
106
+ * moves would cost a full extra detector pass per frame, on an already-slow NPU
107
+ * pipeline, for a demo app where a locked crop is the right tradeoff. See [FaceCrop]. */
108
+ targetFaceBox: FloatArray? = null,
109
+ /** Caps how many of the source's frames actually get swapped and encoded — the rest are
110
+ * decoded and dropped. `null` or `>=` the source's own frame rate processes every
111
+ * frame, unchanged from before this option existed. Trades output smoothness for wall-
112
+ * clock swap time: half the frames is roughly half the NPU + encode work. */
113
+ targetFps: Int? = null,
114
+ onProgress: (VideoSwapProgress) -> Unit,
115
+ ): VideoSwapResult {
116
+ cancelled = false
117
+
118
+ NativePipe.loadError?.let {
119
+ throw IllegalStateException("libffnative.so did not load: $it")
120
+ }
121
+
122
+ val tier = ModelPaths.tier(context)
123
+ val missing = ModelPaths.missing(context, tier)
124
+ if (missing.isNotEmpty()) {
125
+ throw ModelsMissing(
126
+ "Models missing for $tier: ${missing.joinToString(", ")} — call downloadModels() first"
127
+ )
128
+ }
129
+
130
+ try {
131
+ return PipeGuard.run(context, tier, cfg) {
132
+ val decodedSource = decodeBitmap(sourcePath)
133
+ val source = sourceFaceBox?.let { SourceFaces.cropToFace(decodedSource, it) } ?: decodedSource
134
+ val sourceBgr = NativePipe.argbToBgr(pixelsOf(source), source.width, source.height)
135
+ if (!NativePipe.setSource(sourceBgr, source.width, source.height)) {
136
+ throw IllegalStateException(NativePipe.lastError())
137
+ }
138
+ encode(targetPath, outputPath, tier, targetFaceBox, targetFps, onProgress)
139
+ }
140
+ } catch (e: Cancelled) {
141
+ File(outputPath).delete()
142
+ throw e
143
+ }
144
+ }
145
+
146
+ private fun encode(
147
+ targetPath: String,
148
+ outputPath: String,
149
+ tier: String,
150
+ targetFaceBox: FloatArray?,
151
+ targetFps: Int?,
152
+ onProgress: (VideoSwapProgress) -> Unit,
153
+ ): VideoSwapResult {
154
+ val probe = MediaExtractor().apply { setDataSource(targetPath) }
155
+ val videoTrack = findTrack(probe, "video/")
156
+ ?: throw IllegalArgumentException("No video track in $targetPath")
157
+ val videoFormat = probe.getTrackFormat(videoTrack)
158
+ val mime = videoFormat.getString(MediaFormat.KEY_MIME)
159
+ ?: throw IllegalArgumentException("Video track has no MIME type")
160
+ val width = videoFormat.getInteger(MediaFormat.KEY_WIDTH)
161
+ val height = videoFormat.getInteger(MediaFormat.KEY_HEIGHT)
162
+ // MediaCodec never applies this -- it is a container flag a player honours and a
163
+ // decoder does not, so a portrait clip decodes as sideways landscape frames. Corrected
164
+ // per frame below with rotateBgr, then restored on the output track with the same hint.
165
+ val rotation = if (videoFormat.containsKey(MediaFormat.KEY_ROTATION)) {
166
+ videoFormat.getInteger(MediaFormat.KEY_ROTATION)
167
+ } else 0
168
+ // Some encoders stored this as a float, not the documented int -- runCatching rather
169
+ // than a crash over a number that only ever feeds a progress estimate and the output
170
+ // bitrate heuristic below.
171
+ val frameRate = runCatching {
172
+ if (videoFormat.containsKey(MediaFormat.KEY_FRAME_RATE)) {
173
+ videoFormat.getInteger(MediaFormat.KEY_FRAME_RATE).coerceAtLeast(1)
174
+ } else 30
175
+ }.getOrDefault(30)
176
+ // `targetFps` only drops frames, never adds them -- out of range (<=0, or >= the
177
+ // source's own rate) means "every frame", the behaviour before this option existed.
178
+ val effectiveFps = targetFps?.takeIf { it in 1 until frameRate } ?: frameRate
179
+ val estimatedFrameCount = if (videoFormat.containsKey(MediaFormat.KEY_DURATION)) {
180
+ ((videoFormat.getLong(MediaFormat.KEY_DURATION) / 1_000_000.0) * effectiveFps)
181
+ .toInt().coerceAtLeast(0)
182
+ } else 0
183
+ val audioTrack = findTrack(probe, "audio/")
184
+ val audioFormat = audioTrack?.let { probe.getTrackFormat(it) }
185
+ probe.release()
186
+
187
+ // Fixed for the whole clip -- rotation doesn't change frame to frame, so neither does
188
+ // the upright size processFrame sees, which is what a target-face crop rect is in.
189
+ val (uprightW, uprightH) = if (rotation == 90 || rotation == 270) height to width else width to height
190
+ val targetRect = targetFaceBox?.let { FaceCrop.rect(it, uprightW, uprightH) }
191
+
192
+ val extractor = MediaExtractor().apply { setDataSource(targetPath) }
193
+ extractor.selectTrack(videoTrack)
194
+
195
+ val decoder = MediaCodec.createDecoderByType(mime)
196
+ decoder.configure(videoFormat, null, null, 0)
197
+ decoder.start()
198
+
199
+ val encFormat = MediaFormat.createVideoFormat(mime, width, height).apply {
200
+ setInteger(
201
+ MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible
202
+ )
203
+ // A simple bits-per-pixel heuristic (~4x the pixel count) rather than a fixed number:
204
+ // it scales with resolution instead of being tuned for 720p alone and silently wrong
205
+ // elsewhere. Floored so a tiny/low-fps clip doesn't get an unusably low bitrate.
206
+ setInteger(MediaFormat.KEY_BIT_RATE, (width * height * 4).coerceAtLeast(2_000_000))
207
+ setInteger(MediaFormat.KEY_FRAME_RATE, frameRate)
208
+ setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
209
+ }
210
+ val encoder = MediaCodec.createEncoderByType(mime)
211
+ encoder.configure(encFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
212
+ encoder.start()
213
+
214
+ val encodedChunks = mutableListOf<Pair<ByteArray, MediaCodec.BufferInfo>>()
215
+ var outputFormat: MediaFormat? = null
216
+ var inputDone = false
217
+ var frameIndex = 0
218
+ var faceFrameCount = 0
219
+ // Bresenham-style frame-rate reduction: keeps whichever decoded frames land closest to
220
+ // an even spread at `effectiveFps`, rather than a fixed "every Nth" stride that would
221
+ // drift against a variable-frame-rate source.
222
+ var decodedCount = 0
223
+ var keptCount = 0
224
+ val info = MediaCodec.BufferInfo()
225
+ val loopStart = System.nanoTime()
226
+ // One sample per second of PRESENTATION time, not per decoded frame -- matches
227
+ // upstream's own video sampling rate for the content gate (docs/02-upstream.md).
228
+ val contentSampler = ContentGate.VideoSampler()
229
+ var lastSampledSecond = -1
230
+
231
+ try {
232
+ var encoderDone = false
233
+ while (!encoderDone) {
234
+ throwIfCancelled()
235
+
236
+ if (!inputDone) {
237
+ val inIndex = decoder.dequeueInputBuffer(TIMEOUT_US)
238
+ if (inIndex >= 0) {
239
+ val buf = decoder.getInputBuffer(inIndex)!!
240
+ val sampleSize = extractor.readSampleData(buf, 0)
241
+ if (sampleSize < 0) {
242
+ decoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
243
+ inputDone = true
244
+ } else {
245
+ decoder.queueInputBuffer(inIndex, 0, sampleSize, extractor.sampleTime, 0)
246
+ extractor.advance()
247
+ }
248
+ }
249
+ }
250
+
251
+ var draining = true
252
+ while (draining) {
253
+ val outIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_US)
254
+ if (outIndex < 0) {
255
+ draining = false
256
+ continue
257
+ }
258
+ if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
259
+ decoder.releaseOutputBuffer(outIndex, false)
260
+ feedEncoderEos(encoder)
261
+ draining = false
262
+ continue
263
+ }
264
+ if (info.size <= 0) {
265
+ decoder.releaseOutputBuffer(outIndex, false)
266
+ continue
267
+ }
268
+
269
+ decodedCount++
270
+ val keep = effectiveFps >= frameRate || (decodedCount * effectiveFps) / frameRate > keptCount
271
+ if (!keep) {
272
+ // Dropped before ever touching an Image or the NPU -- this is the entire saving
273
+ // targetFps buys: no processFrame call, no encode, just hand the buffer back.
274
+ decoder.releaseOutputBuffer(outIndex, false)
275
+ continue
276
+ }
277
+ keptCount++
278
+
279
+ val image = decoder.getOutputImage(outIndex)
280
+ ?: throw IllegalStateException("Decoder did not return a YUV image for $targetPath")
281
+ var bgr = bgrFromImage(image, width, height)
282
+ image.close()
283
+ decoder.releaseOutputBuffer(outIndex, false)
284
+
285
+ // Content-gate sample, on the raw (pre-rotation-correction) frame -- orientation
286
+ // does not matter for this check, and it's the same bytes already in hand.
287
+ val presentedSecond = (info.presentationTimeUs / 1_000_000L).toInt()
288
+ if (presentedSecond != lastSampledSecond) {
289
+ lastSampledSecond = presentedSecond
290
+ contentSampler.sample(bgr, width, height)
291
+ }
292
+
293
+ val (fw, fh) = if (rotation == 90 || rotation == 270) height to width else width to height
294
+ if (rotation != 0) {
295
+ bgr = NativePipe.rotateBgr(bgr, width, height, rotation)
296
+ ?: throw IllegalStateException(NativePipe.lastError())
297
+ }
298
+ val faces = if (targetRect != null) {
299
+ // Same crop-swap-paste trick PhotoSwap uses: processFrame only ever sees the
300
+ // chosen face's region, so it can't touch anything outside it.
301
+ val cropped = FaceCrop.crop(bgr, fw, targetRect)
302
+ val cw = targetRect[2] - targetRect[0]
303
+ val ch = targetRect[3] - targetRect[1]
304
+ val count = NativePipe.processFrame(cropped, cw, ch)
305
+ if (count > 0) FaceCrop.paste(bgr, fw, cropped, targetRect)
306
+ count
307
+ } else {
308
+ NativePipe.processFrame(bgr, fw, fh)
309
+ }
310
+ if (faces < 0) throw IllegalStateException(NativePipe.lastError())
311
+ if (faces > 0) faceFrameCount++
312
+ if (rotation != 0) {
313
+ bgr = NativePipe.rotateBgr(bgr, fw, fh, (360 - rotation) % 360)
314
+ ?: throw IllegalStateException(NativePipe.lastError())
315
+ }
316
+ // Same orientation the encoder is about to receive, so the preview matches the
317
+ // output file rather than the decoder's raw (possibly sideways) frame.
318
+ PreviewSurfaceHolder.draw(bgr, width, height)
319
+ feedEncoderFrame(encoder, bgr, width, height, info.presentationTimeUs)
320
+
321
+ frameIndex++
322
+ val elapsedS = (System.nanoTime() - loopStart) / 1_000_000_000.0
323
+ onProgress(
324
+ VideoSwapProgress(frameIndex, estimatedFrameCount, if (elapsedS > 0) frameIndex / elapsedS else 0.0)
325
+ )
326
+ }
327
+
328
+ var encoderDraining = true
329
+ while (encoderDraining) {
330
+ val outIndex = encoder.dequeueOutputBuffer(info, TIMEOUT_US)
331
+ when {
332
+ outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> outputFormat = encoder.outputFormat
333
+ outIndex < 0 -> encoderDraining = false
334
+ else -> {
335
+ if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG == 0 && info.size > 0) {
336
+ val buf = encoder.getOutputBuffer(outIndex)!!
337
+ buf.position(info.offset)
338
+ buf.limit(info.offset + info.size)
339
+ val chunk = ByteArray(info.size)
340
+ buf.get(chunk)
341
+ val chunkInfo = MediaCodec.BufferInfo()
342
+ .apply { set(0, chunk.size, info.presentationTimeUs, info.flags) }
343
+ encodedChunks += chunk to chunkInfo
344
+ }
345
+ val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
346
+ encoder.releaseOutputBuffer(outIndex, false)
347
+ if (isEos) {
348
+ encoderDone = true
349
+ encoderDraining = false
350
+ }
351
+ }
352
+ }
353
+ }
354
+ }
355
+ } finally {
356
+ decoder.stop()
357
+ decoder.release()
358
+ encoder.stop()
359
+ encoder.release()
360
+ extractor.release()
361
+ }
362
+
363
+ // Applied here, before the muxer ever opens outputPath -- a refusal must never leave a
364
+ // partial or full file behind, the same "refuse before doing the work" shape as the
365
+ // single-frame check in PhotoSwap.
366
+ contentSampler.finish()
367
+
368
+ val elapsedS = (System.nanoTime() - loopStart) / 1_000_000_000.0
369
+ val fps = if (elapsedS > 0) frameIndex / elapsedS else 0.0
370
+
371
+ mux(outputPath, rotation, outputFormat, encodedChunks, targetPath, audioTrack, audioFormat)
372
+
373
+ return VideoSwapResult(outputPath, frameIndex, faceFrameCount, tier, fps, audioFormat != null)
374
+ }
375
+
376
+ private fun mux(
377
+ outputPath: String,
378
+ rotation: Int,
379
+ videoFormat: MediaFormat?,
380
+ encodedChunks: List<Pair<ByteArray, MediaCodec.BufferInfo>>,
381
+ targetPath: String,
382
+ audioTrack: Int?,
383
+ audioFormat: MediaFormat?,
384
+ ) {
385
+ val format = videoFormat
386
+ ?: throw IllegalStateException("Encoder never reported an output format")
387
+ File(outputPath).parentFile?.mkdirs()
388
+ val muxer = MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
389
+ try {
390
+ if (rotation != 0) muxer.setOrientationHint(rotation)
391
+ val videoMuxTrack = muxer.addTrack(format)
392
+ val audioMuxTrack = if (audioFormat != null) muxer.addTrack(audioFormat) else -1
393
+ muxer.start()
394
+
395
+ for ((bytes, chunkInfo) in encodedChunks) {
396
+ muxer.writeSampleData(videoMuxTrack, ByteBuffer.wrap(bytes), chunkInfo)
397
+ }
398
+
399
+ if (audioMuxTrack >= 0 && audioTrack != null) {
400
+ copyAudio(targetPath, audioTrack, muxer, audioMuxTrack)
401
+ }
402
+ } finally {
403
+ muxer.stop()
404
+ muxer.release()
405
+ }
406
+ }
407
+
408
+ /** Raw compressed-sample copy — no decode, no encode, since only video pixels change. */
409
+ private fun copyAudio(targetPath: String, audioTrack: Int, muxer: MediaMuxer, muxTrack: Int) {
410
+ val extractor = MediaExtractor().apply { setDataSource(targetPath) }
411
+ try {
412
+ extractor.selectTrack(audioTrack)
413
+ val buf = ByteBuffer.allocateDirect(1 shl 20)
414
+ val info = MediaCodec.BufferInfo()
415
+ while (true) {
416
+ throwIfCancelled()
417
+ buf.clear()
418
+ val size = extractor.readSampleData(buf, 0)
419
+ if (size < 0) break
420
+ info.set(0, size, extractor.sampleTime, extractor.sampleFlags)
421
+ muxer.writeSampleData(muxTrack, buf, info)
422
+ extractor.advance()
423
+ }
424
+ } finally {
425
+ extractor.release()
426
+ }
427
+ }
428
+
429
+ private fun feedEncoderFrame(encoder: MediaCodec, bgr: ByteArray, w: Int, h: Int, ptsUs: Long) {
430
+ val index = dequeueInputBufferBlocking(encoder)
431
+ val image = encoder.getInputImage(index)
432
+ ?: throw IllegalStateException("Encoder does not support Image-based input")
433
+ val planes = image.planes
434
+ val ok = NativePipe.bgrToImagePlanes(
435
+ bgr, w, h,
436
+ planes[0].buffer, planes[0].rowStride, planes[0].pixelStride,
437
+ planes[1].buffer, planes[1].rowStride, planes[1].pixelStride,
438
+ planes[2].buffer, planes[2].rowStride, planes[2].pixelStride,
439
+ )
440
+ image.close()
441
+ if (!ok) throw IllegalStateException(NativePipe.lastError())
442
+ encoder.queueInputBuffer(index, 0, w * h * 3 / 2, ptsUs, 0)
443
+ }
444
+
445
+ private fun feedEncoderEos(encoder: MediaCodec) {
446
+ val index = dequeueInputBufferBlocking(encoder)
447
+ encoder.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
448
+ }
449
+
450
+ private fun dequeueInputBufferBlocking(encoder: MediaCodec): Int {
451
+ repeat(MAX_STALL_POLLS) {
452
+ throwIfCancelled()
453
+ val index = encoder.dequeueInputBuffer(TIMEOUT_US)
454
+ if (index >= 0) return index
455
+ }
456
+ throw IllegalStateException("Encoder did not free an input buffer in time")
457
+ }
458
+
459
+ private fun bgrFromImage(image: Image, w: Int, h: Int): ByteArray {
460
+ val planes = image.planes
461
+ return NativePipe.yuvToBgr(
462
+ planeBytes(planes[0].buffer), planes[0].rowStride,
463
+ planeBytes(planes[1].buffer), planes[1].rowStride, planes[1].pixelStride,
464
+ planeBytes(planes[2].buffer), planes[2].rowStride, planes[2].pixelStride,
465
+ w, h,
466
+ )
467
+ }
468
+
469
+ private fun planeBytes(buffer: ByteBuffer): ByteArray {
470
+ val dup = buffer.duplicate()
471
+ val bytes = ByteArray(dup.remaining())
472
+ dup.get(bytes)
473
+ return bytes
474
+ }
475
+
476
+ private fun findTrack(extractor: MediaExtractor, mimePrefix: String): Int? {
477
+ for (i in 0 until extractor.trackCount) {
478
+ val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue
479
+ if (mime.startsWith(mimePrefix)) return i
480
+ }
481
+ return null
482
+ }
483
+
484
+ private fun throwIfCancelled() {
485
+ if (cancelled) throw Cancelled()
486
+ }
487
+
488
+ // Duplicated from PhotoSwap rather than shared: each swap object is self-contained, the
489
+ // same shape upstream's own files are in, and this is ten lines.
490
+ private fun decodeBitmap(path: String): Bitmap {
491
+ val options = BitmapFactory.Options().apply { inPreferredConfig = Bitmap.Config.ARGB_8888 }
492
+ return BitmapFactory.decodeFile(path, options)
493
+ ?: throw IllegalArgumentException("Could not decode image: $path")
494
+ }
495
+
496
+ private fun pixelsOf(bitmap: Bitmap): IntArray {
497
+ val pixels = IntArray(bitmap.width * bitmap.height)
498
+ bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height)
499
+ return pixels
500
+ }
501
+ }
@@ -0,0 +1,85 @@
1
+ package com.facefusion
2
+
3
+ import android.app.Notification
4
+ import android.app.NotificationChannel
5
+ import android.app.NotificationManager
6
+ import android.app.Service
7
+ import android.content.Intent
8
+ import android.content.pm.ServiceInfo
9
+ import android.os.Build
10
+ import android.os.IBinder
11
+
12
+ /**
13
+ * A notification shell, not a worker.
14
+ *
15
+ * Android's actual rule for a job that runs minutes in the background is about *process
16
+ * priority*, not which thread does the work: a foreground service with an active
17
+ * notification keeps the whole process from being treated as killable. So this component
18
+ * does nothing but call [startForeground] and sit there — the real decode/swap/encode loop
19
+ * still runs on [FacefusionModule]'s own `videoWorker` thread, in the same process, which is
20
+ * what keeps [VideoSwap] a plain Kotlin object with a normal return value instead of needing
21
+ * a callback channel back from a Service to a TurboModule's `Promise`.
22
+ *
23
+ * `foregroundServiceType="mediaProcessing"` (declared in the manifest) is required from
24
+ * targetSdk 34 for any foreground service — chosen over `dataSync` as the type that actually
25
+ * describes what this is. Below API 34 the type argument does not exist yet, so
26
+ * [start] branches on it.
27
+ */
28
+ class VideoSwapService : Service() {
29
+
30
+ override fun onBind(intent: Intent?): IBinder? = null
31
+
32
+ override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
33
+ ensureChannel(this)
34
+ val notification = buildNotification(this, "Swapping video…")
35
+ if (Build.VERSION.SDK_INT >= 34) {
36
+ startForeground(NOTIFICATION_ID, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROCESSING)
37
+ } else {
38
+ startForeground(NOTIFICATION_ID, notification)
39
+ }
40
+ return START_NOT_STICKY
41
+ }
42
+
43
+ companion object {
44
+ private const val CHANNEL_ID = "facefusion_video_swap"
45
+ private const val NOTIFICATION_ID = 1001
46
+
47
+ fun start(context: android.content.Context) {
48
+ ensureChannel(context)
49
+ context.startForegroundService(Intent(context, VideoSwapService::class.java))
50
+ }
51
+
52
+ fun stop(context: android.content.Context) {
53
+ context.stopService(Intent(context, VideoSwapService::class.java))
54
+ }
55
+
56
+ /** Updates the already-shown notification in place — no need to go back through the
57
+ * Service's `onStartCommand`, since [NotificationManager.notify] on the same id just
58
+ * replaces what's displayed. Throttled by the caller, same cadence as the JS event. */
59
+ fun updateProgress(context: android.content.Context, text: String) {
60
+ val nm = context.getSystemService(NotificationManager::class.java) ?: return
61
+ nm.notify(NOTIFICATION_ID, buildNotification(context, text))
62
+ }
63
+
64
+ private fun ensureChannel(context: android.content.Context) {
65
+ if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return
66
+ val nm = context.getSystemService(NotificationManager::class.java) ?: return
67
+ if (nm.getNotificationChannel(CHANNEL_ID) != null) return
68
+ nm.createNotificationChannel(
69
+ NotificationChannel(CHANNEL_ID, "Video swap", NotificationManager.IMPORTANCE_LOW)
70
+ )
71
+ }
72
+
73
+ // A library ships no drawable resources of its own worth trusting for a status-bar
74
+ // icon, so this borrows the CONSUMER app's own launcher icon -- always present, unlike
75
+ // any icon id this package could hardcode. Full-colour icons render squared-off on some
76
+ // versions rather than as a proper alpha-mask glyph; cosmetic, not a functional issue.
77
+ private fun buildNotification(context: android.content.Context, text: String): Notification =
78
+ Notification.Builder(context, CHANNEL_ID)
79
+ .setContentTitle("FaceFusion")
80
+ .setContentText(text)
81
+ .setSmallIcon(context.applicationInfo.icon)
82
+ .setOngoing(true)
83
+ .build()
84
+ }
85
+ }