react-native-webrtc-kaleidoscope 2.0.0 → 2.1.1

package/android/src/main/java/com/simiancraft/kaleidoscope/EffectTuning.kt

@@ -6,83 +6,83 @@
 // This is the parameter-passing side-channel that lets us tune effect
 // uniforms without touching the upstream react-native-webrtc registry,
 // which only accepts flat-string effect names. The web side mirrors this
-// shape in src/web/tuning.ts; iOS mirrors it in
+// shape in web-driver/tuning.ts; iOS mirrors it in
 // ios/KaleidoscopeModule/EffectTuning.swift.
 
 package com.simiancraft.kaleidoscope
 
 internal object EffectTuning {
-  /**
-   * Gaussian sigma for the blur effect; higher = softer blur. Default 5;
-   * clamped to [0.5, 7] (the useful range before the linear-sampled kernel
-   * truncates and bands). Setting the property is the public mutation API.
-   */
-  @Volatile
-  var blurSigma: Float = 5f
-    set(value) {
-      field = value.coerceIn(0.5f, 7f)
-    }
+    /**
+     * Gaussian sigma for the blur effect; higher = softer blur. Default 5;
+     * clamped to [0.5, 7] (the useful range before the linear-sampled kernel
+     * truncates and bands). Setting the property is the public mutation API.
+     */
+    @Volatile
+    var blurSigma: Float = 5f
+        set(value) {
+            field = value.coerceIn(0.5f, 7f)
+        }
 
-  /**
-   * Mask smoothstep hardness for blur and background-image composites,
-   * in [0, 1]; 0 = soft halo, 1 = near-step. Default reproduces the
-   * historical smoothstep(0.34, 0.66) edge.
-   */
-  @Volatile
-  var maskHardness: Float = 0.5f
-    set(value) {
-      field = value.coerceIn(0f, 1f)
-    }
+    /**
+     * Mask smoothstep hardness for blur and the masked composite,
+     * in [0, 1]; 0 = soft halo, 1 = near-step. Default reproduces the
+     * historical smoothstep(0.34, 0.66) edge.
+     */
+    @Volatile
+    var maskHardness: Float = 0.5f
+        set(value) {
+            field = value.coerceIn(0f, 1f)
+        }
 
-  /**
-   * Mask smoothstep center for blur and background-image composites,
-   * in [0, 1]; the threshold at which a pixel's raw confidence flips from
-   * "background" to "person". Default 0.7 (dialed in post-optimization).
-   * Higher values reject
-   * low-confidence edges (chair backs, hair flyaway); lower values are
-   * more inclusive. Clamped to a workable range below to keep the
-   * smoothstep transition non-degenerate.
-   */
-  @Volatile
-  var maskThreshold: Float = 0.7f
-    set(value) {
-      field = value.coerceIn(0.05f, 0.95f)
-    }
+    /**
+     * Mask smoothstep center for blur and the masked composite,
+     * in [0, 1]; the threshold at which a pixel's raw confidence flips from
+     * "background" to "person". Default 0.7 (dialed in post-optimization).
+     * Higher values reject
+     * low-confidence edges (chair backs, hair flyaway); lower values are
+     * more inclusive. Clamped to a workable range below to keep the
+     * smoothstep transition non-degenerate.
+     */
+    @Volatile
+    var maskThreshold: Float = 0.7f
+        set(value) {
+            field = value.coerceIn(0.05f, 0.95f)
+        }
 
-  /**
-   * Debug GPU timing. When true, the GLES effect factories log per-frame
-   * GPU-completion latency under the "Perf" logcat tag (read a frame late via
-   * the frame-pipeline fence). Off by default; toggled from JS via the Expo
-   * Module's setDebugTiming so a tester can capture ground-truth numbers on a
-   * device build without rebuilding. Mirrors the iOS timing flag being added
-   * in parallel.
-   */
-  @Volatile
-  var debugTiming: Boolean = false
-    set(value) {
-      field = value
-    }
+    /**
+     * Debug GPU timing. When true, the GLES effect factories log per-frame
+     * GPU-completion latency under the "Perf" logcat tag (read a frame late via
+     * the frame-pipeline fence). Off by default; toggled from JS via the Expo
+     * Module's setDebugTiming so a tester can capture ground-truth numbers on a
+     * device build without rebuilding. Mirrors the iOS timing flag being added
+     * in parallel.
+     */
+    @Volatile
+    var debugTiming: Boolean = false
+        set(value) {
+            field = value
+        }
 
-  /**
-   * Segmentation input short-side (px). The mask is produced from an input
-   * downscaled to this; lower = cheaper segmentation, softer mask edge.
-   * Default 384; clamped [128, 1080]. The floor matches iOS and web
-   * (EffectTuning.swift, tuning.ts) so the JS setSegmentationTargetShortSide
-   * contract is identical across platforms; MediaPipe resizes the input to the
-   * model's own 256x256 internally, so a smaller input still segments. Tuned
-   * live from JS via setSegmentationTargetShortSide.
-   */
-  @Volatile
-  var targetShortSide: Int = 384
-    set(value) {
-      field = value.coerceIn(128, 1080)
-    }
+    /**
+     * Segmentation input short-side (px). The mask is produced from an input
+     * downscaled to this; lower = cheaper segmentation, softer mask edge.
+     * Default 384; clamped [128, 1080]. The floor matches iOS and web
+     * (EffectTuning.swift, tuning.ts) so the JS setSegmentationTargetShortSide
+     * contract is identical across platforms; MediaPipe resizes the input to the
+     * model's own 256x256 internally, so a smaller input still segments. Tuned
+     * live from JS via setSegmentationTargetShortSide.
+     */
+    @Volatile
+    var targetShortSide: Int = 384
+        set(value) {
+            field = value.coerceIn(128, 1080)
+        }
 
-  fun reset() {
-    blurSigma = 5f
-    maskHardness = 0.5f
-    maskThreshold = 0.7f
-    debugTiming = false
-    targetShortSide = 384
-  }
+    fun reset() {
+        blurSigma = 5f
+        maskHardness = 0.5f
+        maskThreshold = 0.7f
+        debugTiming = false
+        targetShortSide = 384
+    }
 }

package/android/src/main/java/com/simiancraft/kaleidoscope/KaleidoscopeModule.kt

@@ -15,37 +15,90 @@ import expo.modules.kotlin.modules.Module
 import expo.modules.kotlin.modules.ModuleDefinition
 
 class KaleidoscopeModule : Module() {
-  override fun definition() = ModuleDefinition {
-    Name("RnWebrtcKaleidoscope")
-
-    OnCreate {
-      val ctx = appContext.reactContext
-        ?: error("Kaleidoscope: no react context at OnCreate; cannot register Android effects")
-      Registration.registerAll(ctx)
-    }
-
-    Function("setBlurSigma") { value: Float ->
-      EffectTuning.blurSigma = value
-    }
-
-    Function("setMaskHardness") { value: Float ->
-      EffectTuning.maskHardness = value
-    }
-
-    Function("setMaskThreshold") { value: Float ->
-      EffectTuning.maskThreshold = value
-    }
-
-    Function("setDebugTiming") { value: Boolean ->
-      EffectTuning.debugTiming = value
-    }
-
-    Function("setSegmentationTargetShortSide") { value: Int ->
-      EffectTuning.targetShortSide = value
-    }
-
-    Function("resetEffectTuning") {
-      EffectTuning.reset()
-    }
-  }
+    // Application context captured at OnCreate. resolveImageUri uses this
+    // instead of the lazy appContext.reactContext (which can be transiently null
+    // at call time) so the thumbnail existence check is as robust as the
+    // composite factory, which holds its own captured context.
+    private var assetContext: android.content.Context? = null
+
+    override fun definition() =
+        ModuleDefinition {
+            Name("RnWebrtcKaleidoscope")
+
+            OnCreate {
+                val ctx =
+                    appContext.reactContext
+                        ?: error("Kaleidoscope: no react context at OnCreate; cannot register Android effects")
+                assetContext = ctx.applicationContext
+                Registration.registerAll(ctx)
+            }
+
+            Function("setBlurSigma") { value: Float ->
+                EffectTuning.blurSigma = value
+            }
+
+            Function("setMaskHardness") { value: Float ->
+                EffectTuning.maskHardness = value
+            }
+
+            Function("setMaskThreshold") { value: Float ->
+                EffectTuning.maskThreshold = value
+            }
+
+            // Composite layer-stack channel. JS serializes the active composite's ordered
+            // layer stack (shader, target, blend, image source id, uniforms) to a JSON
+            // string; CompositeLayers parses it once and the single registered "composite"
+            // processor composites whatever stack is current. Delivering it as a
+            // side-channel keeps the upstream registry's flat-string contract intact:
+            // "composite" is one registered name whose contents JS swaps; the per-layer
+            // uniforms ride inside that same payload.
+            Function("setCompositeLayers") { json: String ->
+                CompositeLayers.set(json)
+            }
+
+            // Resolve a displayable URI for a bundled composite image by its book id, for
+            // the picker's native thumbnails (react-native-webrtc-kaleidoscope/ui). The
+            // prebuild copies every referenced WebP into assets/images/<id>.webp
+            // (app.plugin.js, the same images the compositor reads); return Fresco's
+            // `asset://` URI when present, else null so the JS resolver falls back to the
+            // source.
+            //
+            // Must be `asset:///` (Fresco's LocalAssetFetchProducer reads it through the
+            // AssetManager), NOT `file:///android_asset/...`: the latter routes to Fresco's
+            // local-file fetcher, which opens a real FileInputStream on the virtual
+            // `/android_asset` path and fails silently, leaving the tile blank. The
+            // background-replace effect reads the same asset via assets.open() directly,
+            // which is why the full background renders while the thumbnail did not.
+            Function("resolveImageUri") { id: String ->
+                // Device-robust existence check. Two independent device-vs-emulator hazards
+                // both left library thumbnails blank on hardware while the emulator looked
+                // fine, so guard against both:
+                //  - method: assets.open(), NOT assets.list(). AssetManager.list() returns
+                //    empty on real-device AAPT2 release packaging even when the asset is
+                //    present; open() works (the image layer / composite path uses open()
+                //    and loads on device).
+                //  - context: the captured applicationContext, NOT the lazy
+                //    appContext.reactContext, which can be transiently null at call time.
+                val ctx = assetContext ?: appContext.reactContext?.applicationContext
+                val present =
+                    try {
+                        ctx?.assets?.open("images/$id.webp")?.use { true } == true
+                    } catch (t: Throwable) {
+                        false
+                    }
+                if (present) "asset:///images/$id.webp" else null
+            }
+
+            Function("setDebugTiming") { value: Boolean ->
+                EffectTuning.debugTiming = value
+            }
+
+            Function("setSegmentationTargetShortSide") { value: Int ->
+                EffectTuning.targetShortSide = value
+            }
+
+            Function("resetEffectTuning") {
+                EffectTuning.reset()
+            }
+        }
 }

package/android/src/main/java/com/simiancraft/kaleidoscope/Registration.kt

@@ -1,78 +1,37 @@
 // Frame-processor registration for Android. Called from
 // KaleidoscopeModule.OnCreate at Expo Module init time, before any track
-// requests an effect by name. The Context is needed by GPU effects so they
-// can read bundled assets: the background-image WebP presets and the
+// requests an effect by name. The Context is needed by the compositor so it can
+// read bundled assets: the composite image WebP files and the
 // selfie_segmenter.tflite model (loaded via SegmentationEngine).
+//
+// The art axis is one registered "composite" compositor: blur, background
+// images, and generative shaders are all layers inside a composite, delivered
+// from JS via setCompositeLayers (see CompositeLayers / KaleidoscopeModule). There
+// is no longer a per-effect factory per preset; the layer stack is data, swapped
+// from JS as the active composite changes, so adding a preset needs no Kotlin
+// change. Only the four geometric transforms stay statically named alongside it.
 
 package com.simiancraft.kaleidoscope
 
 import android.content.Context
 import com.oney.WebRTCModule.videoEffects.ProcessorProvider
-import com.simiancraft.kaleidoscope.effects.BackgroundImageFactory
-import com.simiancraft.kaleidoscope.effects.BlurFactory
+import com.simiancraft.kaleidoscope.effects.CompositeFactory
 import com.simiancraft.kaleidoscope.effects.TransformFactory
 import com.simiancraft.kaleidoscope.gpu.Orientation
 
 object Registration {
-  @JvmStatic
-  fun registerAll(context: Context) {
-    ProcessorProvider.addProcessor("blur", BlurFactory(context))
+    @JvmStatic
+    fun registerAll(context: Context) {
+        // Geometric reorientation effects. flip-x is the corrected screen-horizontal
+        // mirror (replaces the old "mirror" CPU effect). The rotation correction
+        // lives entirely in Orientation.kt; each registration just names its op.
+        ProcessorProvider.addProcessor("flip-x", TransformFactory(Orientation.Op.FLIP_X))
+        ProcessorProvider.addProcessor("flip-y", TransformFactory(Orientation.Op.FLIP_Y))
+        ProcessorProvider.addProcessor("rotate-cw", TransformFactory(Orientation.Op.ROTATE_CW))
+        ProcessorProvider.addProcessor("rotate-ccw", TransformFactory(Orientation.Op.ROTATE_CCW))
 
-    // Geometric reorientation effects. flip-x is the corrected screen-horizontal
-    // mirror (replaces the old "mirror" CPU effect). The rotation correction
-    // lives entirely in Orientation.kt; each registration just names its op.
-    ProcessorProvider.addProcessor("flip-x", TransformFactory(Orientation.Op.FLIP_X))
-    ProcessorProvider.addProcessor("flip-y", TransformFactory(Orientation.Op.FLIP_Y))
-    ProcessorProvider.addProcessor("rotate-cw", TransformFactory(Orientation.Op.ROTATE_CW))
-    ProcessorProvider.addProcessor("rotate-ccw", TransformFactory(Orientation.Op.ROTATE_CCW))
-
-    // Background-image variants — one factory per source preset. JS side
-    // emits "background-image-{source}" so each preset gets its own
-    // ProcessorProvider entry. Parameterized dispatch via uniforms lands
-    // when we extend the upstream rn-webrtc API surface.
-    ProcessorProvider.addProcessor(
-      "background-image-debug-resolutions",
-      BackgroundImageFactory(context, "debug-resolutions"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-dark-office",
-      BackgroundImageFactory(context, "dark-office"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-light-office",
-      BackgroundImageFactory(context, "light-office"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-home-light",
-      BackgroundImageFactory(context, "home-light"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-home-dark",
-      BackgroundImageFactory(context, "home-dark"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-nature-light",
-      BackgroundImageFactory(context, "nature-light"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-nature-dark",
-      BackgroundImageFactory(context, "nature-dark"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-stylized-light",
-      BackgroundImageFactory(context, "stylized-light"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-stylized-dark",
-      BackgroundImageFactory(context, "stylized-dark"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-simiancraft-light",
-      BackgroundImageFactory(context, "simiancraft-light"),
-    )
-    ProcessorProvider.addProcessor(
-      "background-image-simiancraft-dark",
-      BackgroundImageFactory(context, "simiancraft-dark"),
-    )
-  }
+        // The single art compositor; every art effect (image, blur, generative) is a
+        // layer in the stack JS delivers via setCompositeLayers.
+        ProcessorProvider.addProcessor("composite", CompositeFactory(context))
+    }
 }