react-native-webrtc-kaleidoscope 1.1.0 → 2.0.0

package/android/src/main/java/com/simiancraft/kaleidoscope/effects/BlurFactory.kt

@@ -4,8 +4,8 @@
 //   1. Render the input OES camera texture through an OES->2D passthrough
 //      shader into a cached intermediate FBO (the "original 2D" copy).
 //   2. Produce a mask via Mask.produce (which downsamples the original 2D,
-//      reads it back to a Bitmap, runs MLKit Selfie Segmentation, uploads
-//      the confidence map as a 2D GL texture).
+//      reads it back to a Bitmap, runs MediaPipe segmentation off-thread via
+//      SegmentationEngine, uploads the confidence map as a 2D GL texture).
 //   3. Run two separable Gaussian blur passes on the "original 2D" copy
 //      using ping-pong FBOs.
 //   4. Composite original + blurred + mask into a fresh output texture via
@@ -16,10 +16,11 @@
 // Every observable failure logs to adb logcat under Kaleidoscope.Blur and
 // returns null so upstream forwards the original frame instead of crashing.
 //
-// Replaces the CPU implementation (manual Kotlin YUV/ARGB conversion +
-// MLKit + RenderScript). The CPU path was ~5-10 FPS at 720p; the GPU path
-// now runs MLKit on a worker thread (see Mask.kt) with a last-known-mask
-// cache, so the render thread is no longer gated on segmentation.
+// Replaces the original CPU implementation (manual Kotlin YUV/ARGB
+// conversion). The CPU path was ~5-10 FPS at 720p; the GPU path now runs
+// MediaPipe segmentation on a shared worker thread (see Mask.kt and
+// SegmentationEngine.kt) with a last-known-mask cache, so the render thread is
+// no longer gated on segmentation.
 
 package com.simiancraft.kaleidoscope.effects
 
@@ -32,7 +33,9 @@ import com.oney.WebRTCModule.videoEffects.VideoFrameProcessorFactoryInterface
 import com.simiancraft.kaleidoscope.EffectTuning
 import com.simiancraft.kaleidoscope.gpu.Egl
 import com.simiancraft.kaleidoscope.gpu.Fbo
+import com.simiancraft.kaleidoscope.gpu.FramePipeline
 import com.simiancraft.kaleidoscope.gpu.GlDebug
+import com.simiancraft.kaleidoscope.gpu.Ingest
 import com.simiancraft.kaleidoscope.gpu.GlProgram
 import com.simiancraft.kaleidoscope.gpu.Shaders
 import com.simiancraft.kaleidoscope.segmentation.Mask
@@ -54,10 +57,14 @@ import org.webrtc.YuvConverter
 class BlurFactory(
   private val context: Context,
 ) : VideoFrameProcessorFactoryInterface {
-  override fun build(): VideoFrameProcessor = BlurProcessor()
+  override fun build(): VideoFrameProcessor = BlurProcessor(context)
 }
 
-private class BlurProcessor : VideoFrameProcessor {
+private class BlurProcessor(private val context: Context) : VideoFrameProcessor {
+  // process() is only ever invoked on the single SurfaceTextureHelper capture
+  // thread (VideoEffectProcessor.onFrameCaptured), so this never actually
+  // contends. Retained as cheap uncontended insurance and as an explicit marker
+  // that the GL state below is single-threaded; it is NOT a cross-thread guard.
   private val lock = Any()
 
   private var oesToTwoD: GlProgram? = null
@@ -71,31 +78,46 @@ private class BlurProcessor : VideoFrameProcessor {
   private var blurBFbo: Fbo? = null
   private var cachedWidth = 0
   private var cachedHeight = 0
+  // R1: the blur ping-pong runs at a downscaled resolution; these hold its
+  // dimensions so the blur passes set uAxis in the downscaled texel space.
+  private var blurW = 0
+  private var blurH = 0
 
-  private val mask = Mask()
+  private val mask = Mask(context)
   private var yuvConverter: YuvConverter? = null
 
-  // 9-tap separable Gaussian kernel. tapSpacing is fixed; sigma comes from
-  // EffectTuning at frame time and the kernel is rebuilt on the CPU only
-  // when sigma changes. At sigma=8 the kernel covers ~+/-16 pixels with a
-  // smooth falloff.
+  // R3: one-frame GPU pipeline. process() hands each rendered texture here and
+  // gets the previous frame's GPU-complete texture back, so the capture thread
+  // never blocks on the current frame's GPU work (the old per-frame glFinish).
+  private val pipeline = FramePipeline()
+
+  // Linear-sampled separable Gaussian: 5 entries (center + 4 bilinear pairs).
+  // sigma comes from EffectTuning at frame time; the kernel is rebuilt on the
+  // CPU only when sigma changes. See src/web/blur-kernel.ts for the derivation.
   private val blurWeights = FloatArray(KERNEL_TAPS)
   private val blurOffsets = FloatArray(KERNEL_TAPS)
   private var cachedKernelSigma: Float = Float.NaN
 
   private fun ensureKernel(sigma: Float) {
     if (sigma == cachedKernelSigma) return
-    val tapSpacing = 2.0
-    val sigmaD = sigma.toDouble()
-    for (i in 0 until KERNEL_TAPS) {
-      blurOffsets[i] = (i * tapSpacing).toFloat()
-      val x = blurOffsets[i].toDouble()
-      blurWeights[i] = Math.exp(-(x * x) / (2.0 * sigmaD * sigmaD)).toFloat()
-    }
-    // Normalize: center contributes once, each side tap contributes twice
-    // because the shader samples vUv +/- offset and adds each.
+    val s = sigma.toDouble()
+    fun g(t: Double) = Math.exp(-(t * t) / (2.0 * s * s))
+    // Linear-sampled: center + 4 bilinear pairs of dense texels (1,2)(3,4)
+    // (5,6)(7,8); each pair is one fractional-offset fetch. Normalize so
+    // center + 2*sum(pairs) == 1 (the shader samples vUv +/- offset, adds each).
+    blurOffsets[0] = 0f
+    blurWeights[0] = g(0.0).toFloat()
     var sum = blurWeights[0]
-    for (i in 1 until KERNEL_TAPS) sum += 2f * blurWeights[i]
+    for (p in 1 until KERNEL_TAPS) {
+      val a = (2 * p - 1).toDouble()
+      val b = (2 * p).toDouble()
+      val wa = g(a)
+      val wb = g(b)
+      val w = wa + wb
+      blurOffsets[p] = ((a * wa + b * wb) / w).toFloat()
+      blurWeights[p] = w.toFloat()
+      sum += 2f * blurWeights[p]
+    }
     for (i in 0 until KERNEL_TAPS) blurWeights[i] = blurWeights[i] / sum
     cachedKernelSigma = sigma
   }
@@ -136,13 +158,19 @@ private class BlurProcessor : VideoFrameProcessor {
       Log.w(TAG, "TextureBuffer type is ${inputBuffer.type}; expected OES. Forwarding original.")
       return null
     }
-    val width = inputBuffer.width
-    val height = inputBuffer.height
-    if (width <= 0 || height <= 0) {
-      Log.w(TAG, "Degenerate dims ${width}x${height}; forwarding.")
+    val bufW = inputBuffer.width
+    val bufH = inputBuffer.height
+    if (bufW <= 0 || bufH <= 0) {
+      Log.w(TAG, "Degenerate dims ${bufW}x${bufH}; forwarding.")
       return null
     }
 
+    // Ingest normalization: the OES->2D pass below lands a DISPLAY-UPRIGHT frame
+    // (Ingest folds frame.rotation into the texture matrix), so every cached FBO
+    // and the output are sized in DISPLAY dims, and the frame goes out rotation 0.
+    val width = Ingest.displayWidth(bufW, bufH, frame.rotation)
+    val height = Ingest.displayHeight(bufW, bufH, frame.rotation)
+
     GlDebug.check("blur entry")
     val saved = Egl.save()
     var outputTextureId = 0
@@ -166,7 +194,8 @@ private class BlurProcessor : VideoFrameProcessor {
       origFbo.bind()
       oes.use()
       oes.setInt("uTex", 0)
-      val texMatrix = Egl.matrixToGl(inputBuffer.transformMatrix)
+      // Compose transformMatrix with the display rotation so the FBO lands upright.
+      val texMatrix = Ingest.composedTexMatrix(inputBuffer.transformMatrix, frame.rotation)
       GLES30.glUniformMatrix4fv(oes.uniformLocation("uTexMatrix"), 1, false, texMatrix, 0)
       GLES30.glDisable(GLES30.GL_DEPTH_TEST)
       GLES30.glDisable(GLES30.GL_BLEND)
@@ -180,7 +209,11 @@ private class BlurProcessor : VideoFrameProcessor {
         return null
       }
 
-      // ===== Pass 2: horizontal blur =====
+      // ===== Pass 1b: downsample original -> blurA (box-average) =====
+      // uAxis=0 collapses the kernel to its center tap (weights sum to 1), so
+      // this is a plain bilinear box-average into the downscaled target. Both
+      // blur passes then run in downscaled space; sampling the full-res
+      // original with downscaled-spaced offsets serrates the blur (see web fix).
       GLES30.glActiveTexture(GLES30.GL_TEXTURE0)
       GLES30.glBindTexture(GLES30.GL_TEXTURE_2D, origFbo.texture)
       blurA.bind()
@@ -188,19 +221,27 @@ private class BlurProcessor : VideoFrameProcessor {
       blur.setInt("uTex", 0)
       GLES30.glUniform1fv(blur.uniformLocation("uWeights"), KERNEL_TAPS, blurWeights, 0)
       GLES30.glUniform1fv(blur.uniformLocation("uOffsets"), KERNEL_TAPS, blurOffsets, 0)
-      GLES30.glUniform2f(blur.uniformLocation("uAxis"), 1.0f / width, 0.0f)
+      GLES30.glUniform2f(blur.uniformLocation("uAxis"), 0.0f, 0.0f)
       GLES30.glDrawArrays(GLES30.GL_TRIANGLE_STRIP, 0, 4)
-      GlDebug.check("blur horizontal pass")
+      GlDebug.check("blur downsample pass")
 
-      // ===== Pass 3: vertical blur =====
+      // ===== Pass 2: horizontal blur blurA -> blurB (downscaled) =====
       GLES30.glActiveTexture(GLES30.GL_TEXTURE0)
       GLES30.glBindTexture(GLES30.GL_TEXTURE_2D, blurA.texture)
       blurB.bind()
       blur.use()
       blur.setInt("uTex", 0)
-      GLES30.glUniform1fv(blur.uniformLocation("uWeights"), KERNEL_TAPS, blurWeights, 0)
-      GLES30.glUniform1fv(blur.uniformLocation("uOffsets"), KERNEL_TAPS, blurOffsets, 0)
-      GLES30.glUniform2f(blur.uniformLocation("uAxis"), 0.0f, 1.0f / height)
+      GLES30.glUniform2f(blur.uniformLocation("uAxis"), 1.0f / blurW, 0.0f)
+      GLES30.glDrawArrays(GLES30.GL_TRIANGLE_STRIP, 0, 4)
+      GlDebug.check("blur horizontal pass")
+
+      // ===== Pass 3: vertical blur blurB -> blurA (downscaled) =====
+      GLES30.glActiveTexture(GLES30.GL_TEXTURE0)
+      GLES30.glBindTexture(GLES30.GL_TEXTURE_2D, blurB.texture)
+      blurA.bind()
+      blur.use()
+      blur.setInt("uTex", 0)
+      GLES30.glUniform2f(blur.uniformLocation("uAxis"), 0.0f, 1.0f / blurH)
       GLES30.glDrawArrays(GLES30.GL_TRIANGLE_STRIP, 0, 4)
       GlDebug.check("blur vertical pass")
 
@@ -217,7 +258,8 @@ private class BlurProcessor : VideoFrameProcessor {
       composite.setInt("uOriginal", 0)
 
       GLES30.glActiveTexture(GLES30.GL_TEXTURE1)
-      GLES30.glBindTexture(GLES30.GL_TEXTURE_2D, blurB.texture)
+      // Final blurred result is in blurA after the vertical pass.
+      GLES30.glBindTexture(GLES30.GL_TEXTURE_2D, blurA.texture)
       composite.setInt("uBackground", 1)
 
       GLES30.glActiveTexture(GLES30.GL_TEXTURE2)
@@ -239,10 +281,6 @@ private class BlurProcessor : VideoFrameProcessor {
       GLES30.glDrawArrays(GLES30.GL_TRIANGLE_STRIP, 0, 4)
       GlDebug.check("blur composite pass")
 
-      // Synchronize before handing the output texture to the renderer's
-      // EGL context.
-      GLES30.glFinish()
-
       // Detach the texture from the FBO and free the FBO. The texture lives
       // with the VideoFrame and gets deleted in the release callback.
       GLES30.glFramebufferTexture2D(
@@ -257,28 +295,45 @@ private class BlurProcessor : VideoFrameProcessor {
       outputFboHandle = 0
       GlDebug.check("blur output cleanup")
 
+      // R3: fence this frame's GPU work and hand the PREVIOUS (GPU-complete)
+      // frame's texture downstream instead of glFinish-ing on this one. The
+      // pipeline now owns outputTextureId, so zero our local handle to keep the
+      // orphan-cleanup catch from double-freeing it.
+      val ready = pipeline.enqueue(
+        outputTextureId,
+        width,
+        height,
+        // Pixels are already display-upright after ingest; emit rotation 0.
+        0,
+        frame.timestampNs,
+        EffectTuning.debugTiming,
+        TAG,
+      )
+      outputTextureId = 0
+      // First frame: nothing to hand off yet. Forward the original frame once.
+      ready ?: return null
+
       val yc = yuvConverter ?: run {
         val c = YuvConverter()
         yuvConverter = c
         c
       }
 
-      val capturedTextureId = outputTextureId
+      val readyTextureId = ready.textureId
       val outputBuffer = TextureBufferImpl(
-        width,
-        height,
+        ready.width,
+        ready.height,
         VideoFrame.TextureBuffer.Type.RGB,
-        capturedTextureId,
+        readyTextureId,
         Matrix(),
         textureHelper.handler,
         yc,
         Runnable {
-          GLES30.glDeleteTextures(1, intArrayOf(capturedTextureId), 0)
+          GLES30.glDeleteTextures(1, intArrayOf(readyTextureId), 0)
         },
       )
-      outputTextureId = 0
 
-      VideoFrame(outputBuffer, frame.rotation, frame.timestampNs)
+      VideoFrame(outputBuffer, ready.rotation, ready.timestampNs)
     } catch (t: Throwable) {
       if (outputTextureId != 0) {
         try {
@@ -320,9 +375,18 @@ private class BlurProcessor : VideoFrameProcessor {
     originalFbo?.delete()
     blurAFbo?.delete()
     blurBFbo?.delete()
+    // R1: blur at quarter area (half each axis), floored so the short side
+    // stays >= 256px. The original stays full-res (foreground source + blur
+    // input); the composite upscales the downscaled blurred bg with GL_LINEAR
+    // for free, and the blur discards the detail the extra resolution carries.
+    val shortSide = minOf(width, height)
+    val target = maxOf(256, Math.round(shortSide * 0.5f))
+    val scale = target.toFloat() / shortSide
+    blurW = Math.round(width * scale).coerceAtLeast(2).let { if (it % 2 == 0) it else it - 1 }
+    blurH = Math.round(height * scale).coerceAtLeast(2).let { if (it % 2 == 0) it else it - 1 }
     originalFbo = Fbo(width, height)
-    blurAFbo = Fbo(width, height)
-    blurBFbo = Fbo(width, height)
+    blurAFbo = Fbo(blurW, blurH)
+    blurBFbo = Fbo(blurW, blurH)
     cachedWidth = width
     cachedHeight = height
     GlDebug.check("blur intermediates allocated")
@@ -331,6 +395,6 @@ private class BlurProcessor : VideoFrameProcessor {
   companion object {
     private const val TAG = "Kaleidoscope.Blur"
     private const val GL_TEXTURE_EXTERNAL_OES = 0x8D65
-    private const val KERNEL_TAPS = 9
+    private const val KERNEL_TAPS = 5
   }
 }

package/android/src/main/java/com/simiancraft/kaleidoscope/effects/TransformFactory.kt

@@ -0,0 +1,277 @@
+// Android transform effects — GPU pipeline. One factory class serves all four
+// geometric reorientation ops (flip-x, flip-y, rotate-cw, rotate-ccw); each
+// registration passes a different Orientation.Op.
+//
+// flip-x  = screen-horizontal mirror (left<->right, head stays up); keeps w x h.
+// flip-y  = screen-vertical flip (upside down, left/right unchanged); keeps w x h.
+// rotate-cw  = whole frame rotated 90 degrees clockwise; output dims swap to h x w.
+// rotate-ccw = 90 degrees counter-clockwise; output dims swap to h x w.
+//
+// flip-x replaces the old CPU "mirror" effect (the corrected screen-horizontal
+// mirror); MirrorFactory and its "mirror" registration are removed.
+//
+// Per frame:
+//   1. Render the input OES camera texture through the OES->2D passthrough into
+//      a cached DISPLAY-UPRIGHT "original 2D" FBO, exactly like BlurFactory /
+//      BackgroundImageFactory. Ingest folds the display rotation into the
+//      texture matrix, so the FBO is sized in display dims and already upright.
+//   2. Single TRANSFORM_FRAG pass: sample that upright 2D copy through the
+//      uUvTransform mat2 from Orientation.mat2For(op) into a fresh output
+//      texture sized display w x h (flips) or h x w (rotations).
+//   3. Wrap the fresh output texture in a TextureBufferImpl and return a
+//      VideoFrame at rotation 0 (pixels are already upright).
+//
+// Camera orientation lives ONLY in Ingest; the op here is pure screen space and
+// never consults frame.rotation. All failure paths log under
+// Kaleidoscope.Transform and return null so upstream forwards the original frame.
+
+package com.simiancraft.kaleidoscope.effects
+
+import android.graphics.Matrix
+import android.opengl.GLES30
+import android.util.Log
+import com.oney.WebRTCModule.videoEffects.VideoFrameProcessor
+import com.oney.WebRTCModule.videoEffects.VideoFrameProcessorFactoryInterface
+import com.simiancraft.kaleidoscope.EffectTuning
+import com.simiancraft.kaleidoscope.gpu.Egl
+import com.simiancraft.kaleidoscope.gpu.Fbo
+import com.simiancraft.kaleidoscope.gpu.FramePipeline
+import com.simiancraft.kaleidoscope.gpu.GlDebug
+import com.simiancraft.kaleidoscope.gpu.GlProgram
+import com.simiancraft.kaleidoscope.gpu.Ingest
+import com.simiancraft.kaleidoscope.gpu.Orientation
+import com.simiancraft.kaleidoscope.gpu.Shaders
+import org.webrtc.SurfaceTextureHelper
+import org.webrtc.TextureBufferImpl
+import org.webrtc.VideoFrame
+import org.webrtc.YuvConverter
+
+/**
+ * @param op Which screen-space reorientation this factory's processor applies.
+ */
+class TransformFactory(
+  private val op: Orientation.Op,
+) : VideoFrameProcessorFactoryInterface {
+  override fun build(): VideoFrameProcessor = TransformProcessor(op)
+}
+
+private class TransformProcessor(
+  private val op: Orientation.Op,
+) : VideoFrameProcessor {
+  // process() is only ever invoked on the single SurfaceTextureHelper capture
+  // thread (VideoEffectProcessor.onFrameCaptured), so this never actually
+  // contends. Retained as cheap uncontended insurance and as an explicit marker
+  // that the GL state below is single-threaded; it is NOT a cross-thread guard.
+  private val lock = Any()
+
+  private var oesToTwoD: GlProgram? = null
+  private var transformProgram: GlProgram? = null
+
+  // Cached display-oriented "original 2D" copy at full input resolution.
+  private var originalFbo: Fbo? = null
+  private var cachedWidth = 0
+  private var cachedHeight = 0
+
+  private var yuvConverter: YuvConverter? = null
+
+  // R3: one-frame GPU pipeline (see FramePipeline). Replaces the per-frame
+  // glFinish; only changes WHEN the frame is returned, not its orientation.
+  private val pipeline = FramePipeline()
+
+  override fun process(frame: VideoFrame, textureHelper: SurfaceTextureHelper?): VideoFrame? {
+    return synchronized(lock) { processOuter(frame, textureHelper) }
+  }
+
+  private fun processOuter(frame: VideoFrame, textureHelper: SurfaceTextureHelper?): VideoFrame? {
+    return try {
+      processInner(frame, textureHelper)
+    } catch (t: Throwable) {
+      Log.e(
+        TAG,
+        "process() threw; falling through to original frame. op=$op " +
+          "frame=${frame.buffer.width}x${frame.buffer.height} " +
+          "rotation=${frame.rotation} bufferClass=${frame.buffer.javaClass.simpleName}",
+        t,
+      )
+      null
+    }
+  }
+
+  private fun processInner(
+    frame: VideoFrame,
+    textureHelper: SurfaceTextureHelper?,
+  ): VideoFrame? {
+    if (textureHelper == null) {
+      Log.w(TAG, "textureHelper is null; falling through.")
+      return null
+    }
+    val inputBuffer = frame.buffer
+    if (inputBuffer !is VideoFrame.TextureBuffer) {
+      // Chained after a CPU effect emitting I420 — silently forward.
+      return null
+    }
+    if (inputBuffer.type != VideoFrame.TextureBuffer.Type.OES) {
+      Log.w(TAG, "TextureBuffer type is ${inputBuffer.type}; expected OES. Forwarding original.")
+      return null
+    }
+    val bufW = inputBuffer.width
+    val bufH = inputBuffer.height
+    if (bufW <= 0 || bufH <= 0) {
+      Log.w(TAG, "Degenerate dims ${bufW}x${bufH}; forwarding.")
+      return null
+    }
+
+    // Ingest normalization: the OES->2D pass lands a DISPLAY-UPRIGHT frame, so
+    // the original FBO is sized in DISPLAY dims and the op runs in pure screen
+    // space (Orientation no longer consults frame.rotation).
+    val width = Ingest.displayWidth(bufW, bufH, frame.rotation)
+    val height = Ingest.displayHeight(bufW, bufH, frame.rotation)
+
+    // Rotations swap the (display) output dimensions; flips keep them.
+    val swap = Orientation.swapsDimensions(op)
+    val outWidth = if (swap) height else width
+    val outHeight = if (swap) width else height
+
+    GlDebug.check("transform entry")
+    val saved = Egl.save()
+    var outputTextureId = 0
+    var outputFboHandle = 0
+    return try {
+      ensurePrograms()
+      ensureIntermediates(width, height)
+      val origFbo = originalFbo ?: error("originalFbo null after ensure")
+      val oes = oesToTwoD ?: error("oesToTwoD program null after ensure")
+      val transform = transformProgram ?: error("transformProgram null after ensure")
+
+      // ===== Pass 1: OES camera -> display-oriented "original 2D" =====
+      GLES30.glActiveTexture(GLES30.GL_TEXTURE0)
+      GLES30.glBindTexture(GL_TEXTURE_EXTERNAL_OES, inputBuffer.textureId)
+      origFbo.bind()
+      oes.use()
+      oes.setInt("uTex", 0)
+      // Compose transformMatrix with the display rotation so the FBO lands upright.
+      val texMatrix = Ingest.composedTexMatrix(inputBuffer.transformMatrix, frame.rotation)
+      GLES30.glUniformMatrix4fv(oes.uniformLocation("uTexMatrix"), 1, false, texMatrix, 0)
+      GLES30.glDisable(GLES30.GL_DEPTH_TEST)
+      GLES30.glDisable(GLES30.GL_BLEND)
+      GLES30.glDrawArrays(GLES30.GL_TRIANGLE_STRIP, 0, 4)
+      GlDebug.check("transform OES->2D")
+
+      // ===== Pass 2: geometric reorientation into a fresh output texture =====
+      // Output FBO is sized for the (possibly swapped) output dims. The
+      // uUvTransform mat2 reads the square [0,1] original UV back through the
+      // op; rotations map the unit square onto itself, so swapping the FBO's
+      // physical dims yields the rotated image with no UV clamping.
+      val outputFbo = Fbo(outWidth, outHeight)
+      outputTextureId = outputFbo.texture
+      outputFboHandle = outputFbo.framebuffer
+
+      outputFbo.bind()
+      transform.use()
+      GLES30.glActiveTexture(GLES30.GL_TEXTURE0)
+      GLES30.glBindTexture(GLES30.GL_TEXTURE_2D, origFbo.texture)
+      transform.setInt("uTex", 0)
+      // Input is already display-upright (ingest), so the op is pure screen
+      // space; frame.rotation is no longer consulted here.
+      transform.setMat2("uUvTransform", Orientation.mat2For(op))
+      GLES30.glDrawArrays(GLES30.GL_TRIANGLE_STRIP, 0, 4)
+      GlDebug.check("transform reorient pass")
+
+      // Detach the texture from the FBO and free the FBO; the texture lives
+      // with the VideoFrame and is deleted in the release callback.
+      GLES30.glFramebufferTexture2D(
+        GLES30.GL_FRAMEBUFFER,
+        GLES30.GL_COLOR_ATTACHMENT0,
+        GLES30.GL_TEXTURE_2D,
+        0,
+        0,
+      )
+      GLES30.glBindFramebuffer(GLES30.GL_FRAMEBUFFER, 0)
+      GLES30.glDeleteFramebuffers(1, intArrayOf(outputFboHandle), 0)
+      outputFboHandle = 0
+      GlDebug.check("transform output cleanup")
+
+      // R3: fence this frame and hand the previous GPU-complete frame off. The
+      // (possibly swapped) output dims travel with the frame, so the previous
+      // frame's own dims/rotation are wrapped unchanged; orientation behavior
+      // is untouched, only the return is deferred one frame.
+      val ready = pipeline.enqueue(
+        outputTextureId,
+        outWidth,
+        outHeight,
+        // Pixels are already display-upright after ingest; emit rotation 0.
+        0,
+        frame.timestampNs,
+        EffectTuning.debugTiming,
+        TAG,
+      )
+      outputTextureId = 0
+      ready ?: return null
+
+      val yc = yuvConverter ?: run {
+        val c = YuvConverter()
+        yuvConverter = c
+        c
+      }
+
+      val readyTextureId = ready.textureId
+      val outputBuffer = TextureBufferImpl(
+        ready.width,
+        ready.height,
+        VideoFrame.TextureBuffer.Type.RGB,
+        readyTextureId,
+        Matrix(),
+        textureHelper.handler,
+        yc,
+        Runnable {
+          GLES30.glDeleteTextures(1, intArrayOf(readyTextureId), 0)
+        },
+      )
+
+      VideoFrame(outputBuffer, ready.rotation, ready.timestampNs)
+    } catch (t: Throwable) {
+      if (outputTextureId != 0) {
+        try {
+          GLES30.glDeleteTextures(1, intArrayOf(outputTextureId), 0)
+        } catch (delErr: Throwable) {
+          Log.w(TAG, "failed to free orphan texture $outputTextureId", delErr)
+        }
+      }
+      if (outputFboHandle != 0) {
+        try {
+          GLES30.glDeleteFramebuffers(1, intArrayOf(outputFboHandle), 0)
+        } catch (delErr: Throwable) {
+          Log.w(TAG, "failed to free orphan FBO $outputFboHandle", delErr)
+        }
+      }
+      throw t
+    } finally {
+      Egl.restore(saved)
+    }
+  }
+
+  private fun ensurePrograms() {
+    if (oesToTwoD == null) {
+      oesToTwoD = GlProgram(Shaders.PASSTHROUGH_VERT, Shaders.OES_PASSTHROUGH_FRAG)
+      GlDebug.check("oesToTwoD program compile/link")
+    }
+    if (transformProgram == null) {
+      transformProgram = GlProgram(Shaders.PASSTHROUGH_VERT, Shaders.TRANSFORM_FRAG)
+      GlDebug.check("transform program compile/link")
+    }
+  }
+
+  private fun ensureIntermediates(width: Int, height: Int) {
+    if (cachedWidth == width && cachedHeight == height && originalFbo != null) return
+    originalFbo?.delete()
+    originalFbo = Fbo(width, height)
+    cachedWidth = width
+    cachedHeight = height
+    GlDebug.check("transform intermediates allocated")
+  }
+
+  companion object {
+    private const val TAG = "Kaleidoscope.Transform"
+    private const val GL_TEXTURE_EXTERNAL_OES = 0x8D65
+  }
+}