react-native-facefusion 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -0
- package/README.md +259 -0
- package/android/build.gradle +129 -0
- package/android/src/main/AndroidManifest.xml +66 -0
- package/android/src/main/cpp/CMakeLists.txt +73 -0
- package/android/src/main/java/com/facefusion/BitmapDecode.kt +112 -0
- package/android/src/main/java/com/facefusion/ContentGate.kt +86 -0
- package/android/src/main/java/com/facefusion/DeviceProbe.kt +158 -0
- package/android/src/main/java/com/facefusion/FaceCrop.kt +57 -0
- package/android/src/main/java/com/facefusion/FacefusionModule.kt +396 -0
- package/android/src/main/java/com/facefusion/FacefusionPackage.kt +37 -0
- package/android/src/main/java/com/facefusion/FacefusionPreviewView.kt +30 -0
- package/android/src/main/java/com/facefusion/FacefusionPreviewViewManager.kt +25 -0
- package/android/src/main/java/com/facefusion/GallerySave.kt +82 -0
- package/android/src/main/java/com/facefusion/ModelDownload.kt +351 -0
- package/android/src/main/java/com/facefusion/ModelPaths.kt +120 -0
- package/android/src/main/java/com/facefusion/PhotoSwap.kt +125 -0
- package/android/src/main/java/com/facefusion/PipeGuard.kt +74 -0
- package/android/src/main/java/com/facefusion/PreviewSurfaceHolder.kt +104 -0
- package/android/src/main/java/com/facefusion/SourceFaces.kt +125 -0
- package/android/src/main/java/com/facefusion/SwapConfig.kt +31 -0
- package/android/src/main/java/com/facefusion/TargetFaces.kt +88 -0
- package/android/src/main/java/com/facefusion/VideoSwap.kt +501 -0
- package/android/src/main/java/com/facefusion/VideoSwapService.kt +85 -0
- package/android/src/main/java/com/facefusion/mobile/NativePipe.kt +192 -0
- package/lib/module/FacefusionPreview.js +6 -0
- package/lib/module/FacefusionPreview.js.map +1 -0
- package/lib/module/FacefusionPreview.native.js +15 -0
- package/lib/module/FacefusionPreview.native.js.map +1 -0
- package/lib/module/NativeFacefusion.js +53 -0
- package/lib/module/NativeFacefusion.js.map +1 -0
- package/lib/module/detectSourceFaces.js +6 -0
- package/lib/module/detectSourceFaces.js.map +1 -0
- package/lib/module/detectSourceFaces.native.js +15 -0
- package/lib/module/detectSourceFaces.native.js.map +1 -0
- package/lib/module/detectTargetFaces.js +6 -0
- package/lib/module/detectTargetFaces.js.map +1 -0
- package/lib/module/detectTargetFaces.native.js +13 -0
- package/lib/module/detectTargetFaces.native.js.map +1 -0
- package/lib/module/index.js +11 -0
- package/lib/module/index.js.map +1 -0
- package/lib/module/models.js +16 -0
- package/lib/module/models.js.map +1 -0
- package/lib/module/models.native.js +43 -0
- package/lib/module/models.native.js.map +1 -0
- package/lib/module/package.json +1 -0
- package/lib/module/probeDevice.js +6 -0
- package/lib/module/probeDevice.js.map +1 -0
- package/lib/module/probeDevice.native.js +14 -0
- package/lib/module/probeDevice.native.js.map +1 -0
- package/lib/module/saveToGallery.js +6 -0
- package/lib/module/saveToGallery.js.map +1 -0
- package/lib/module/saveToGallery.native.js +15 -0
- package/lib/module/saveToGallery.native.js.map +1 -0
- package/lib/module/swapPhoto.js +6 -0
- package/lib/module/swapPhoto.js.map +1 -0
- package/lib/module/swapPhoto.native.js +16 -0
- package/lib/module/swapPhoto.native.js.map +1 -0
- package/lib/module/swapVideo.js +12 -0
- package/lib/module/swapVideo.js.map +1 -0
- package/lib/module/swapVideo.native.js +29 -0
- package/lib/module/swapVideo.native.js.map +1 -0
- package/lib/typescript/package.json +1 -0
- package/lib/typescript/src/FacefusionPreview.d.ts +4 -0
- package/lib/typescript/src/FacefusionPreview.d.ts.map +1 -0
- package/lib/typescript/src/FacefusionPreview.native.d.ts +142 -0
- package/lib/typescript/src/FacefusionPreview.native.d.ts.map +1 -0
- package/lib/typescript/src/NativeFacefusion.d.ts +291 -0
- package/lib/typescript/src/NativeFacefusion.d.ts.map +1 -0
- package/lib/typescript/src/detectSourceFaces.d.ts +3 -0
- package/lib/typescript/src/detectSourceFaces.d.ts.map +1 -0
- package/lib/typescript/src/detectSourceFaces.native.d.ts +11 -0
- package/lib/typescript/src/detectSourceFaces.native.d.ts.map +1 -0
- package/lib/typescript/src/detectTargetFaces.d.ts +3 -0
- package/lib/typescript/src/detectTargetFaces.d.ts.map +1 -0
- package/lib/typescript/src/detectTargetFaces.native.d.ts +9 -0
- package/lib/typescript/src/detectTargetFaces.native.d.ts.map +1 -0
- package/lib/typescript/src/index.d.ts +11 -0
- package/lib/typescript/src/index.d.ts.map +1 -0
- package/lib/typescript/src/models.d.ts +7 -0
- package/lib/typescript/src/models.d.ts.map +1 -0
- package/lib/typescript/src/models.native.d.ts +31 -0
- package/lib/typescript/src/models.native.d.ts.map +1 -0
- package/lib/typescript/src/probeDevice.d.ts +3 -0
- package/lib/typescript/src/probeDevice.d.ts.map +1 -0
- package/lib/typescript/src/probeDevice.native.d.ts +10 -0
- package/lib/typescript/src/probeDevice.native.d.ts.map +1 -0
- package/lib/typescript/src/saveToGallery.d.ts +2 -0
- package/lib/typescript/src/saveToGallery.d.ts.map +1 -0
- package/lib/typescript/src/saveToGallery.native.d.ts +9 -0
- package/lib/typescript/src/saveToGallery.native.d.ts.map +1 -0
- package/lib/typescript/src/swapPhoto.d.ts +3 -0
- package/lib/typescript/src/swapPhoto.d.ts.map +1 -0
- package/lib/typescript/src/swapPhoto.native.d.ts +12 -0
- package/lib/typescript/src/swapPhoto.native.d.ts.map +1 -0
- package/lib/typescript/src/swapVideo.d.ts +6 -0
- package/lib/typescript/src/swapVideo.d.ts.map +1 -0
- package/lib/typescript/src/swapVideo.native.d.ts +20 -0
- package/lib/typescript/src/swapVideo.native.d.ts.map +1 -0
- package/package.json +132 -0
- package/patches/ffjni-analyse-faces.patch +63 -0
- package/scripts/check-tarball.sh +68 -0
- package/scripts/fetch-upstream.sh +91 -0
- package/src/FacefusionPreview.native.tsx +18 -0
- package/src/FacefusionPreview.tsx +9 -0
- package/src/NativeFacefusion.ts +324 -0
- package/src/detectSourceFaces.native.tsx +17 -0
- package/src/detectSourceFaces.tsx +10 -0
- package/src/detectTargetFaces.native.tsx +15 -0
- package/src/detectTargetFaces.tsx +10 -0
- package/src/index.tsx +25 -0
- package/src/models.native.tsx +45 -0
- package/src/models.tsx +23 -0
- package/src/probeDevice.native.tsx +13 -0
- package/src/probeDevice.tsx +7 -0
- package/src/saveToGallery.native.tsx +16 -0
- package/src/saveToGallery.tsx +9 -0
- package/src/swapPhoto.native.tsx +20 -0
- package/src/swapPhoto.tsx +12 -0
- package/src/swapVideo.native.tsx +40 -0
- package/src/swapVideo.tsx +31 -0
- package/third_party/facefusion-mobile/NOTICE +49 -0
|
@@ -0,0 +1,501 @@
|
|
|
1
|
+
package com.facefusion
|
|
2
|
+
|
|
3
|
+
import android.content.Context
|
|
4
|
+
import android.graphics.Bitmap
|
|
5
|
+
import android.graphics.BitmapFactory
|
|
6
|
+
import android.media.Image
|
|
7
|
+
import android.media.MediaCodec
|
|
8
|
+
import android.media.MediaCodecInfo
|
|
9
|
+
import android.media.MediaExtractor
|
|
10
|
+
import android.media.MediaFormat
|
|
11
|
+
import android.media.MediaMuxer
|
|
12
|
+
import com.facefusion.mobile.NativePipe
|
|
13
|
+
import java.io.File
|
|
14
|
+
import java.io.IOException
|
|
15
|
+
import java.nio.ByteBuffer
|
|
16
|
+
|
|
17
|
+
/** A tick of progress through [VideoSwap.run]. */
|
|
18
|
+
data class VideoSwapProgress(
|
|
19
|
+
/** Frames decoded, swapped and encoded so far. */
|
|
20
|
+
val frameIndex: Int,
|
|
21
|
+
/** Estimated from the container's duration and frame rate — not exact, since a
|
|
22
|
+
* variable-frame-rate source has no true count until the last frame arrives. `0` when
|
|
23
|
+
* it could not be estimated. */
|
|
24
|
+
val estimatedFrameCount: Int,
|
|
25
|
+
/** Wall-clock frames/second across the loop so far, not upstream's per-graph figure. */
|
|
26
|
+
val fps: Double,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
/** The result of one video swap. */
|
|
30
|
+
data class VideoSwapResult(
|
|
31
|
+
val outputPath: String,
|
|
32
|
+
/** Frames decoded, swapped and encoded — the real count, unlike [VideoSwapProgress]'s
|
|
33
|
+
* estimate. */
|
|
34
|
+
val frameCount: Int,
|
|
35
|
+
/** Of [frameCount], how many had at least one face swapped. */
|
|
36
|
+
val faceFrameCount: Int,
|
|
37
|
+
/** The tier that ran — see [ModelPaths.tier]. */
|
|
38
|
+
val tier: String,
|
|
39
|
+
/** Wall-clock frames/second across the decode-swap-encode loop only: excludes opening
|
|
40
|
+
* the extractor/codecs and the final mux/audio-copy pass. Measured, per rule 11 — never
|
|
41
|
+
* reported as upstream's per-graph figure, which this is not. */
|
|
42
|
+
val fps: Double,
|
|
43
|
+
/** Whether the source clip had an audio track, which — if true — was copied to
|
|
44
|
+
* [outputPath] unmodified. */
|
|
45
|
+
val hasAudio: Boolean,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Swaps a face from [VideoSwap.run]'s `sourcePath` into every frame of a video, writing the
|
|
50
|
+
* result to `outputPath`.
|
|
51
|
+
*
|
|
52
|
+
* Shaped like [PhotoSwap]: paths in, path out, and this is the only other place in the
|
|
53
|
+
* Kotlin layer that touches pixels. What's different from a still is everything about
|
|
54
|
+
* getting pixels in and out of [NativePipe] — MediaExtractor → MediaCodec(decode) →
|
|
55
|
+
* [NativePipe.processFrame] → MediaCodec(encode) → MediaMuxer, all synchronous on the
|
|
56
|
+
* caller's thread, one frame at a time. The three converters this needed —
|
|
57
|
+
* [NativePipe.yuvToBgr], [NativePipe.rotateBgr], [NativePipe.bgrToImagePlanes] — were
|
|
58
|
+
* already exported by upstream's vendored, unmodified `ffjni.cpp` (ADR-0004): nothing here
|
|
59
|
+
* changes the C++, only wires symbols that were sitting unused since Phase 3.
|
|
60
|
+
*
|
|
61
|
+
* **Two passes, not one interleaved stream.** The whole video track is decoded, swapped and
|
|
62
|
+
* encoded first, buffering the (small — a few MB for a short clip) encoded output in memory;
|
|
63
|
+
* only then is the muxer opened, because the muxer needs the encoder's *actual* output
|
|
64
|
+
* `MediaFormat` (only known once the encoder starts producing) before any track can be
|
|
65
|
+
* added, and needs every track added before `start()`. Interleaving a live audio copy with
|
|
66
|
+
* that would mean holding audio samples until the video format shows up. Buffering encoded
|
|
67
|
+
* video instead is simpler and correct for the clip lengths this project targets; a
|
|
68
|
+
* multi-minute clip would want a streaming interleave instead — not needed yet.
|
|
69
|
+
*/
|
|
70
|
+
object VideoSwap {
|
|
71
|
+
|
|
72
|
+
/** Mirrors [PhotoSwap.ModelsMissing] — the tier's required models are not on disk. */
|
|
73
|
+
class ModelsMissing(message: String) : Exception(message)
|
|
74
|
+
|
|
75
|
+
/** Thrown when [cancel] was called mid-run. The partial [outputPath] is deleted first. */
|
|
76
|
+
class Cancelled : IOException("Cancelled")
|
|
77
|
+
|
|
78
|
+
private const val TIMEOUT_US = 10_000L
|
|
79
|
+
// Bounds the busy-wait for a free codec input buffer. At 10 ms per poll this is 5 s,
|
|
80
|
+
// generous for a codec that is merely busy and a hard stop for one that is stuck --
|
|
81
|
+
// without a bound, a stalled codec would hang the job past where [cancel] can reach it,
|
|
82
|
+
// since the check for it lives in this same wait.
|
|
83
|
+
private const val MAX_STALL_POLLS = 500
|
|
84
|
+
|
|
85
|
+
@Volatile
|
|
86
|
+
private var cancelled = false
|
|
87
|
+
|
|
88
|
+
/** Asks the run in flight to stop. Idempotent, and safe to call when nothing is running. */
|
|
89
|
+
fun cancel() {
|
|
90
|
+
cancelled = true
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
fun run(
|
|
94
|
+
context: Context,
|
|
95
|
+
sourcePath: String,
|
|
96
|
+
targetPath: String,
|
|
97
|
+
outputPath: String,
|
|
98
|
+
cfg: SwapConfig,
|
|
99
|
+
/** `[left, top, right, bottom]` from [SourceFaces.detect], or `null` for the default
|
|
100
|
+
* "largest face in the source" that [NativePipe.setSource] already picks on its own. */
|
|
101
|
+
sourceFaceBox: FloatArray? = null,
|
|
102
|
+
/** `[left, top, right, bottom]` from [TargetFaces.detect], in the clip's upright
|
|
103
|
+
* (post-rotation-correction) coordinate space, or `null` to swap every face found in
|
|
104
|
+
* every frame, same as before this option existed. Picked once and held fixed for the
|
|
105
|
+
* whole clip — Saad's own call: re-detecting the face every frame to track it as it
|
|
106
|
+
* moves would cost a full extra detector pass per frame, on an already-slow NPU
|
|
107
|
+
* pipeline, for a demo app where a locked crop is the right tradeoff. See [FaceCrop]. */
|
|
108
|
+
targetFaceBox: FloatArray? = null,
|
|
109
|
+
/** Caps how many of the source's frames actually get swapped and encoded — the rest are
|
|
110
|
+
* decoded and dropped. `null` or `>=` the source's own frame rate processes every
|
|
111
|
+
* frame, unchanged from before this option existed. Trades output smoothness for wall-
|
|
112
|
+
* clock swap time: half the frames is roughly half the NPU + encode work. */
|
|
113
|
+
targetFps: Int? = null,
|
|
114
|
+
onProgress: (VideoSwapProgress) -> Unit,
|
|
115
|
+
): VideoSwapResult {
|
|
116
|
+
cancelled = false
|
|
117
|
+
|
|
118
|
+
NativePipe.loadError?.let {
|
|
119
|
+
throw IllegalStateException("libffnative.so did not load: $it")
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
val tier = ModelPaths.tier(context)
|
|
123
|
+
val missing = ModelPaths.missing(context, tier)
|
|
124
|
+
if (missing.isNotEmpty()) {
|
|
125
|
+
throw ModelsMissing(
|
|
126
|
+
"Models missing for $tier: ${missing.joinToString(", ")} — call downloadModels() first"
|
|
127
|
+
)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
try {
|
|
131
|
+
return PipeGuard.run(context, tier, cfg) {
|
|
132
|
+
val decodedSource = decodeBitmap(sourcePath)
|
|
133
|
+
val source = sourceFaceBox?.let { SourceFaces.cropToFace(decodedSource, it) } ?: decodedSource
|
|
134
|
+
val sourceBgr = NativePipe.argbToBgr(pixelsOf(source), source.width, source.height)
|
|
135
|
+
if (!NativePipe.setSource(sourceBgr, source.width, source.height)) {
|
|
136
|
+
throw IllegalStateException(NativePipe.lastError())
|
|
137
|
+
}
|
|
138
|
+
encode(targetPath, outputPath, tier, targetFaceBox, targetFps, onProgress)
|
|
139
|
+
}
|
|
140
|
+
} catch (e: Cancelled) {
|
|
141
|
+
File(outputPath).delete()
|
|
142
|
+
throw e
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
private fun encode(
|
|
147
|
+
targetPath: String,
|
|
148
|
+
outputPath: String,
|
|
149
|
+
tier: String,
|
|
150
|
+
targetFaceBox: FloatArray?,
|
|
151
|
+
targetFps: Int?,
|
|
152
|
+
onProgress: (VideoSwapProgress) -> Unit,
|
|
153
|
+
): VideoSwapResult {
|
|
154
|
+
val probe = MediaExtractor().apply { setDataSource(targetPath) }
|
|
155
|
+
val videoTrack = findTrack(probe, "video/")
|
|
156
|
+
?: throw IllegalArgumentException("No video track in $targetPath")
|
|
157
|
+
val videoFormat = probe.getTrackFormat(videoTrack)
|
|
158
|
+
val mime = videoFormat.getString(MediaFormat.KEY_MIME)
|
|
159
|
+
?: throw IllegalArgumentException("Video track has no MIME type")
|
|
160
|
+
val width = videoFormat.getInteger(MediaFormat.KEY_WIDTH)
|
|
161
|
+
val height = videoFormat.getInteger(MediaFormat.KEY_HEIGHT)
|
|
162
|
+
// MediaCodec never applies this -- it is a container flag a player honours and a
|
|
163
|
+
// decoder does not, so a portrait clip decodes as sideways landscape frames. Corrected
|
|
164
|
+
// per frame below with rotateBgr, then restored on the output track with the same hint.
|
|
165
|
+
val rotation = if (videoFormat.containsKey(MediaFormat.KEY_ROTATION)) {
|
|
166
|
+
videoFormat.getInteger(MediaFormat.KEY_ROTATION)
|
|
167
|
+
} else 0
|
|
168
|
+
// Some encoders stored this as a float, not the documented int -- runCatching rather
|
|
169
|
+
// than a crash over a number that only ever feeds a progress estimate and the output
|
|
170
|
+
// bitrate heuristic below.
|
|
171
|
+
val frameRate = runCatching {
|
|
172
|
+
if (videoFormat.containsKey(MediaFormat.KEY_FRAME_RATE)) {
|
|
173
|
+
videoFormat.getInteger(MediaFormat.KEY_FRAME_RATE).coerceAtLeast(1)
|
|
174
|
+
} else 30
|
|
175
|
+
}.getOrDefault(30)
|
|
176
|
+
// `targetFps` only drops frames, never adds them -- out of range (<=0, or >= the
|
|
177
|
+
// source's own rate) means "every frame", the behaviour before this option existed.
|
|
178
|
+
val effectiveFps = targetFps?.takeIf { it in 1 until frameRate } ?: frameRate
|
|
179
|
+
val estimatedFrameCount = if (videoFormat.containsKey(MediaFormat.KEY_DURATION)) {
|
|
180
|
+
((videoFormat.getLong(MediaFormat.KEY_DURATION) / 1_000_000.0) * effectiveFps)
|
|
181
|
+
.toInt().coerceAtLeast(0)
|
|
182
|
+
} else 0
|
|
183
|
+
val audioTrack = findTrack(probe, "audio/")
|
|
184
|
+
val audioFormat = audioTrack?.let { probe.getTrackFormat(it) }
|
|
185
|
+
probe.release()
|
|
186
|
+
|
|
187
|
+
// Fixed for the whole clip -- rotation doesn't change frame to frame, so neither does
|
|
188
|
+
// the upright size processFrame sees, which is what a target-face crop rect is in.
|
|
189
|
+
val (uprightW, uprightH) = if (rotation == 90 || rotation == 270) height to width else width to height
|
|
190
|
+
val targetRect = targetFaceBox?.let { FaceCrop.rect(it, uprightW, uprightH) }
|
|
191
|
+
|
|
192
|
+
val extractor = MediaExtractor().apply { setDataSource(targetPath) }
|
|
193
|
+
extractor.selectTrack(videoTrack)
|
|
194
|
+
|
|
195
|
+
val decoder = MediaCodec.createDecoderByType(mime)
|
|
196
|
+
decoder.configure(videoFormat, null, null, 0)
|
|
197
|
+
decoder.start()
|
|
198
|
+
|
|
199
|
+
val encFormat = MediaFormat.createVideoFormat(mime, width, height).apply {
|
|
200
|
+
setInteger(
|
|
201
|
+
MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible
|
|
202
|
+
)
|
|
203
|
+
// A simple bits-per-pixel heuristic (~4x the pixel count) rather than a fixed number:
|
|
204
|
+
// it scales with resolution instead of being tuned for 720p alone and silently wrong
|
|
205
|
+
// elsewhere. Floored so a tiny/low-fps clip doesn't get an unusably low bitrate.
|
|
206
|
+
setInteger(MediaFormat.KEY_BIT_RATE, (width * height * 4).coerceAtLeast(2_000_000))
|
|
207
|
+
setInteger(MediaFormat.KEY_FRAME_RATE, frameRate)
|
|
208
|
+
setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
|
|
209
|
+
}
|
|
210
|
+
val encoder = MediaCodec.createEncoderByType(mime)
|
|
211
|
+
encoder.configure(encFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
|
|
212
|
+
encoder.start()
|
|
213
|
+
|
|
214
|
+
val encodedChunks = mutableListOf<Pair<ByteArray, MediaCodec.BufferInfo>>()
|
|
215
|
+
var outputFormat: MediaFormat? = null
|
|
216
|
+
var inputDone = false
|
|
217
|
+
var frameIndex = 0
|
|
218
|
+
var faceFrameCount = 0
|
|
219
|
+
// Bresenham-style frame-rate reduction: keeps whichever decoded frames land closest to
|
|
220
|
+
// an even spread at `effectiveFps`, rather than a fixed "every Nth" stride that would
|
|
221
|
+
// drift against a variable-frame-rate source.
|
|
222
|
+
var decodedCount = 0
|
|
223
|
+
var keptCount = 0
|
|
224
|
+
val info = MediaCodec.BufferInfo()
|
|
225
|
+
val loopStart = System.nanoTime()
|
|
226
|
+
// One sample per second of PRESENTATION time, not per decoded frame -- matches
|
|
227
|
+
// upstream's own video sampling rate for the content gate (docs/02-upstream.md).
|
|
228
|
+
val contentSampler = ContentGate.VideoSampler()
|
|
229
|
+
var lastSampledSecond = -1
|
|
230
|
+
|
|
231
|
+
try {
|
|
232
|
+
var encoderDone = false
|
|
233
|
+
while (!encoderDone) {
|
|
234
|
+
throwIfCancelled()
|
|
235
|
+
|
|
236
|
+
if (!inputDone) {
|
|
237
|
+
val inIndex = decoder.dequeueInputBuffer(TIMEOUT_US)
|
|
238
|
+
if (inIndex >= 0) {
|
|
239
|
+
val buf = decoder.getInputBuffer(inIndex)!!
|
|
240
|
+
val sampleSize = extractor.readSampleData(buf, 0)
|
|
241
|
+
if (sampleSize < 0) {
|
|
242
|
+
decoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
|
243
|
+
inputDone = true
|
|
244
|
+
} else {
|
|
245
|
+
decoder.queueInputBuffer(inIndex, 0, sampleSize, extractor.sampleTime, 0)
|
|
246
|
+
extractor.advance()
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
var draining = true
|
|
252
|
+
while (draining) {
|
|
253
|
+
val outIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_US)
|
|
254
|
+
if (outIndex < 0) {
|
|
255
|
+
draining = false
|
|
256
|
+
continue
|
|
257
|
+
}
|
|
258
|
+
if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
|
|
259
|
+
decoder.releaseOutputBuffer(outIndex, false)
|
|
260
|
+
feedEncoderEos(encoder)
|
|
261
|
+
draining = false
|
|
262
|
+
continue
|
|
263
|
+
}
|
|
264
|
+
if (info.size <= 0) {
|
|
265
|
+
decoder.releaseOutputBuffer(outIndex, false)
|
|
266
|
+
continue
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
decodedCount++
|
|
270
|
+
val keep = effectiveFps >= frameRate || (decodedCount * effectiveFps) / frameRate > keptCount
|
|
271
|
+
if (!keep) {
|
|
272
|
+
// Dropped before ever touching an Image or the NPU -- this is the entire saving
|
|
273
|
+
// targetFps buys: no processFrame call, no encode, just hand the buffer back.
|
|
274
|
+
decoder.releaseOutputBuffer(outIndex, false)
|
|
275
|
+
continue
|
|
276
|
+
}
|
|
277
|
+
keptCount++
|
|
278
|
+
|
|
279
|
+
val image = decoder.getOutputImage(outIndex)
|
|
280
|
+
?: throw IllegalStateException("Decoder did not return a YUV image for $targetPath")
|
|
281
|
+
var bgr = bgrFromImage(image, width, height)
|
|
282
|
+
image.close()
|
|
283
|
+
decoder.releaseOutputBuffer(outIndex, false)
|
|
284
|
+
|
|
285
|
+
// Content-gate sample, on the raw (pre-rotation-correction) frame -- orientation
|
|
286
|
+
// does not matter for this check, and it's the same bytes already in hand.
|
|
287
|
+
val presentedSecond = (info.presentationTimeUs / 1_000_000L).toInt()
|
|
288
|
+
if (presentedSecond != lastSampledSecond) {
|
|
289
|
+
lastSampledSecond = presentedSecond
|
|
290
|
+
contentSampler.sample(bgr, width, height)
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
val (fw, fh) = if (rotation == 90 || rotation == 270) height to width else width to height
|
|
294
|
+
if (rotation != 0) {
|
|
295
|
+
bgr = NativePipe.rotateBgr(bgr, width, height, rotation)
|
|
296
|
+
?: throw IllegalStateException(NativePipe.lastError())
|
|
297
|
+
}
|
|
298
|
+
val faces = if (targetRect != null) {
|
|
299
|
+
// Same crop-swap-paste trick PhotoSwap uses: processFrame only ever sees the
|
|
300
|
+
// chosen face's region, so it can't touch anything outside it.
|
|
301
|
+
val cropped = FaceCrop.crop(bgr, fw, targetRect)
|
|
302
|
+
val cw = targetRect[2] - targetRect[0]
|
|
303
|
+
val ch = targetRect[3] - targetRect[1]
|
|
304
|
+
val count = NativePipe.processFrame(cropped, cw, ch)
|
|
305
|
+
if (count > 0) FaceCrop.paste(bgr, fw, cropped, targetRect)
|
|
306
|
+
count
|
|
307
|
+
} else {
|
|
308
|
+
NativePipe.processFrame(bgr, fw, fh)
|
|
309
|
+
}
|
|
310
|
+
if (faces < 0) throw IllegalStateException(NativePipe.lastError())
|
|
311
|
+
if (faces > 0) faceFrameCount++
|
|
312
|
+
if (rotation != 0) {
|
|
313
|
+
bgr = NativePipe.rotateBgr(bgr, fw, fh, (360 - rotation) % 360)
|
|
314
|
+
?: throw IllegalStateException(NativePipe.lastError())
|
|
315
|
+
}
|
|
316
|
+
// Same orientation the encoder is about to receive, so the preview matches the
|
|
317
|
+
// output file rather than the decoder's raw (possibly sideways) frame.
|
|
318
|
+
PreviewSurfaceHolder.draw(bgr, width, height)
|
|
319
|
+
feedEncoderFrame(encoder, bgr, width, height, info.presentationTimeUs)
|
|
320
|
+
|
|
321
|
+
frameIndex++
|
|
322
|
+
val elapsedS = (System.nanoTime() - loopStart) / 1_000_000_000.0
|
|
323
|
+
onProgress(
|
|
324
|
+
VideoSwapProgress(frameIndex, estimatedFrameCount, if (elapsedS > 0) frameIndex / elapsedS else 0.0)
|
|
325
|
+
)
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
var encoderDraining = true
|
|
329
|
+
while (encoderDraining) {
|
|
330
|
+
val outIndex = encoder.dequeueOutputBuffer(info, TIMEOUT_US)
|
|
331
|
+
when {
|
|
332
|
+
outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> outputFormat = encoder.outputFormat
|
|
333
|
+
outIndex < 0 -> encoderDraining = false
|
|
334
|
+
else -> {
|
|
335
|
+
if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG == 0 && info.size > 0) {
|
|
336
|
+
val buf = encoder.getOutputBuffer(outIndex)!!
|
|
337
|
+
buf.position(info.offset)
|
|
338
|
+
buf.limit(info.offset + info.size)
|
|
339
|
+
val chunk = ByteArray(info.size)
|
|
340
|
+
buf.get(chunk)
|
|
341
|
+
val chunkInfo = MediaCodec.BufferInfo()
|
|
342
|
+
.apply { set(0, chunk.size, info.presentationTimeUs, info.flags) }
|
|
343
|
+
encodedChunks += chunk to chunkInfo
|
|
344
|
+
}
|
|
345
|
+
val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
|
|
346
|
+
encoder.releaseOutputBuffer(outIndex, false)
|
|
347
|
+
if (isEos) {
|
|
348
|
+
encoderDone = true
|
|
349
|
+
encoderDraining = false
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
} finally {
|
|
356
|
+
decoder.stop()
|
|
357
|
+
decoder.release()
|
|
358
|
+
encoder.stop()
|
|
359
|
+
encoder.release()
|
|
360
|
+
extractor.release()
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// Applied here, before the muxer ever opens outputPath -- a refusal must never leave a
|
|
364
|
+
// partial or full file behind, the same "refuse before doing the work" shape as the
|
|
365
|
+
// single-frame check in PhotoSwap.
|
|
366
|
+
contentSampler.finish()
|
|
367
|
+
|
|
368
|
+
val elapsedS = (System.nanoTime() - loopStart) / 1_000_000_000.0
|
|
369
|
+
val fps = if (elapsedS > 0) frameIndex / elapsedS else 0.0
|
|
370
|
+
|
|
371
|
+
mux(outputPath, rotation, outputFormat, encodedChunks, targetPath, audioTrack, audioFormat)
|
|
372
|
+
|
|
373
|
+
return VideoSwapResult(outputPath, frameIndex, faceFrameCount, tier, fps, audioFormat != null)
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
private fun mux(
|
|
377
|
+
outputPath: String,
|
|
378
|
+
rotation: Int,
|
|
379
|
+
videoFormat: MediaFormat?,
|
|
380
|
+
encodedChunks: List<Pair<ByteArray, MediaCodec.BufferInfo>>,
|
|
381
|
+
targetPath: String,
|
|
382
|
+
audioTrack: Int?,
|
|
383
|
+
audioFormat: MediaFormat?,
|
|
384
|
+
) {
|
|
385
|
+
val format = videoFormat
|
|
386
|
+
?: throw IllegalStateException("Encoder never reported an output format")
|
|
387
|
+
File(outputPath).parentFile?.mkdirs()
|
|
388
|
+
val muxer = MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
|
|
389
|
+
try {
|
|
390
|
+
if (rotation != 0) muxer.setOrientationHint(rotation)
|
|
391
|
+
val videoMuxTrack = muxer.addTrack(format)
|
|
392
|
+
val audioMuxTrack = if (audioFormat != null) muxer.addTrack(audioFormat) else -1
|
|
393
|
+
muxer.start()
|
|
394
|
+
|
|
395
|
+
for ((bytes, chunkInfo) in encodedChunks) {
|
|
396
|
+
muxer.writeSampleData(videoMuxTrack, ByteBuffer.wrap(bytes), chunkInfo)
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
if (audioMuxTrack >= 0 && audioTrack != null) {
|
|
400
|
+
copyAudio(targetPath, audioTrack, muxer, audioMuxTrack)
|
|
401
|
+
}
|
|
402
|
+
} finally {
|
|
403
|
+
muxer.stop()
|
|
404
|
+
muxer.release()
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** Raw compressed-sample copy — no decode, no encode, since only video pixels change. */
|
|
409
|
+
private fun copyAudio(targetPath: String, audioTrack: Int, muxer: MediaMuxer, muxTrack: Int) {
|
|
410
|
+
val extractor = MediaExtractor().apply { setDataSource(targetPath) }
|
|
411
|
+
try {
|
|
412
|
+
extractor.selectTrack(audioTrack)
|
|
413
|
+
val buf = ByteBuffer.allocateDirect(1 shl 20)
|
|
414
|
+
val info = MediaCodec.BufferInfo()
|
|
415
|
+
while (true) {
|
|
416
|
+
throwIfCancelled()
|
|
417
|
+
buf.clear()
|
|
418
|
+
val size = extractor.readSampleData(buf, 0)
|
|
419
|
+
if (size < 0) break
|
|
420
|
+
info.set(0, size, extractor.sampleTime, extractor.sampleFlags)
|
|
421
|
+
muxer.writeSampleData(muxTrack, buf, info)
|
|
422
|
+
extractor.advance()
|
|
423
|
+
}
|
|
424
|
+
} finally {
|
|
425
|
+
extractor.release()
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
private fun feedEncoderFrame(encoder: MediaCodec, bgr: ByteArray, w: Int, h: Int, ptsUs: Long) {
|
|
430
|
+
val index = dequeueInputBufferBlocking(encoder)
|
|
431
|
+
val image = encoder.getInputImage(index)
|
|
432
|
+
?: throw IllegalStateException("Encoder does not support Image-based input")
|
|
433
|
+
val planes = image.planes
|
|
434
|
+
val ok = NativePipe.bgrToImagePlanes(
|
|
435
|
+
bgr, w, h,
|
|
436
|
+
planes[0].buffer, planes[0].rowStride, planes[0].pixelStride,
|
|
437
|
+
planes[1].buffer, planes[1].rowStride, planes[1].pixelStride,
|
|
438
|
+
planes[2].buffer, planes[2].rowStride, planes[2].pixelStride,
|
|
439
|
+
)
|
|
440
|
+
image.close()
|
|
441
|
+
if (!ok) throw IllegalStateException(NativePipe.lastError())
|
|
442
|
+
encoder.queueInputBuffer(index, 0, w * h * 3 / 2, ptsUs, 0)
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
private fun feedEncoderEos(encoder: MediaCodec) {
|
|
446
|
+
val index = dequeueInputBufferBlocking(encoder)
|
|
447
|
+
encoder.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
private fun dequeueInputBufferBlocking(encoder: MediaCodec): Int {
|
|
451
|
+
repeat(MAX_STALL_POLLS) {
|
|
452
|
+
throwIfCancelled()
|
|
453
|
+
val index = encoder.dequeueInputBuffer(TIMEOUT_US)
|
|
454
|
+
if (index >= 0) return index
|
|
455
|
+
}
|
|
456
|
+
throw IllegalStateException("Encoder did not free an input buffer in time")
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
private fun bgrFromImage(image: Image, w: Int, h: Int): ByteArray {
|
|
460
|
+
val planes = image.planes
|
|
461
|
+
return NativePipe.yuvToBgr(
|
|
462
|
+
planeBytes(planes[0].buffer), planes[0].rowStride,
|
|
463
|
+
planeBytes(planes[1].buffer), planes[1].rowStride, planes[1].pixelStride,
|
|
464
|
+
planeBytes(planes[2].buffer), planes[2].rowStride, planes[2].pixelStride,
|
|
465
|
+
w, h,
|
|
466
|
+
)
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
private fun planeBytes(buffer: ByteBuffer): ByteArray {
|
|
470
|
+
val dup = buffer.duplicate()
|
|
471
|
+
val bytes = ByteArray(dup.remaining())
|
|
472
|
+
dup.get(bytes)
|
|
473
|
+
return bytes
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
private fun findTrack(extractor: MediaExtractor, mimePrefix: String): Int? {
|
|
477
|
+
for (i in 0 until extractor.trackCount) {
|
|
478
|
+
val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue
|
|
479
|
+
if (mime.startsWith(mimePrefix)) return i
|
|
480
|
+
}
|
|
481
|
+
return null
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
private fun throwIfCancelled() {
|
|
485
|
+
if (cancelled) throw Cancelled()
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// Duplicated from PhotoSwap rather than shared: each swap object is self-contained, the
|
|
489
|
+
// same shape upstream's own files are in, and this is ten lines.
|
|
490
|
+
private fun decodeBitmap(path: String): Bitmap {
|
|
491
|
+
val options = BitmapFactory.Options().apply { inPreferredConfig = Bitmap.Config.ARGB_8888 }
|
|
492
|
+
return BitmapFactory.decodeFile(path, options)
|
|
493
|
+
?: throw IllegalArgumentException("Could not decode image: $path")
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
private fun pixelsOf(bitmap: Bitmap): IntArray {
|
|
497
|
+
val pixels = IntArray(bitmap.width * bitmap.height)
|
|
498
|
+
bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height)
|
|
499
|
+
return pixels
|
|
500
|
+
}
|
|
501
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
package com.facefusion
|
|
2
|
+
|
|
3
|
+
import android.app.Notification
|
|
4
|
+
import android.app.NotificationChannel
|
|
5
|
+
import android.app.NotificationManager
|
|
6
|
+
import android.app.Service
|
|
7
|
+
import android.content.Intent
|
|
8
|
+
import android.content.pm.ServiceInfo
|
|
9
|
+
import android.os.Build
|
|
10
|
+
import android.os.IBinder
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* A notification shell, not a worker.
|
|
14
|
+
*
|
|
15
|
+
* Android's actual rule for a job that runs minutes in the background is about *process
|
|
16
|
+
* priority*, not which thread does the work: a foreground service with an active
|
|
17
|
+
* notification keeps the whole process from being treated as killable. So this component
|
|
18
|
+
* does nothing but call [startForeground] and sit there — the real decode/swap/encode loop
|
|
19
|
+
* still runs on [FacefusionModule]'s own `videoWorker` thread, in the same process, which is
|
|
20
|
+
* what keeps [VideoSwap] a plain Kotlin object with a normal return value instead of needing
|
|
21
|
+
* a callback channel back from a Service to a TurboModule's `Promise`.
|
|
22
|
+
*
|
|
23
|
+
* `foregroundServiceType="mediaProcessing"` (declared in the manifest) is required from
|
|
24
|
+
* targetSdk 34 for any foreground service — chosen over `dataSync` as the type that actually
|
|
25
|
+
* describes what this is. Below API 34 the type argument does not exist yet, so
|
|
26
|
+
* [start] branches on it.
|
|
27
|
+
*/
|
|
28
|
+
class VideoSwapService : Service() {
|
|
29
|
+
|
|
30
|
+
override fun onBind(intent: Intent?): IBinder? = null
|
|
31
|
+
|
|
32
|
+
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
|
33
|
+
ensureChannel(this)
|
|
34
|
+
val notification = buildNotification(this, "Swapping video…")
|
|
35
|
+
if (Build.VERSION.SDK_INT >= 34) {
|
|
36
|
+
startForeground(NOTIFICATION_ID, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROCESSING)
|
|
37
|
+
} else {
|
|
38
|
+
startForeground(NOTIFICATION_ID, notification)
|
|
39
|
+
}
|
|
40
|
+
return START_NOT_STICKY
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
companion object {
|
|
44
|
+
private const val CHANNEL_ID = "facefusion_video_swap"
|
|
45
|
+
private const val NOTIFICATION_ID = 1001
|
|
46
|
+
|
|
47
|
+
fun start(context: android.content.Context) {
|
|
48
|
+
ensureChannel(context)
|
|
49
|
+
context.startForegroundService(Intent(context, VideoSwapService::class.java))
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
fun stop(context: android.content.Context) {
|
|
53
|
+
context.stopService(Intent(context, VideoSwapService::class.java))
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Updates the already-shown notification in place — no need to go back through the
|
|
57
|
+
* Service's `onStartCommand`, since [NotificationManager.notify] on the same id just
|
|
58
|
+
* replaces what's displayed. Throttled by the caller, same cadence as the JS event. */
|
|
59
|
+
fun updateProgress(context: android.content.Context, text: String) {
|
|
60
|
+
val nm = context.getSystemService(NotificationManager::class.java) ?: return
|
|
61
|
+
nm.notify(NOTIFICATION_ID, buildNotification(context, text))
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
private fun ensureChannel(context: android.content.Context) {
|
|
65
|
+
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return
|
|
66
|
+
val nm = context.getSystemService(NotificationManager::class.java) ?: return
|
|
67
|
+
if (nm.getNotificationChannel(CHANNEL_ID) != null) return
|
|
68
|
+
nm.createNotificationChannel(
|
|
69
|
+
NotificationChannel(CHANNEL_ID, "Video swap", NotificationManager.IMPORTANCE_LOW)
|
|
70
|
+
)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// A library ships no drawable resources of its own worth trusting for a status-bar
|
|
74
|
+
// icon, so this borrows the CONSUMER app's own launcher icon -- always present, unlike
|
|
75
|
+
// any icon id this package could hardcode. Full-colour icons render squared-off on some
|
|
76
|
+
// versions rather than as a proper alpha-mask glyph; cosmetic, not a functional issue.
|
|
77
|
+
private fun buildNotification(context: android.content.Context, text: String): Notification =
|
|
78
|
+
Notification.Builder(context, CHANNEL_ID)
|
|
79
|
+
.setContentTitle("FaceFusion")
|
|
80
|
+
.setContentText(text)
|
|
81
|
+
.setSmallIcon(context.applicationInfo.icon)
|
|
82
|
+
.setOngoing(true)
|
|
83
|
+
.build()
|
|
84
|
+
}
|
|
85
|
+
}
|