react-native-webrtc-kaleidoscope 2.0.0 → 2.1.0

package/ios/KaleidoscopeModule/EffectTuning.swift

@@ -1,6 +1,6 @@
 // Mutable runtime parameters for the GLSL effects on iOS. Mirrors the
 // Android side at android/.../EffectTuning.kt and the web side at
-// src/web/tuning.ts.
+// web-driver/tuning.ts.
 //
 // The Expo Module's setBlurSigma / setMaskHardness JS functions update
 // these values via the bridge; per-frame processors read them on every
@@ -13,101 +13,101 @@
 import Foundation
 
 public enum EffectTuning {
-  // os_unfair_lock is the iOS 13+ recommended primitive for cheap
-  // serialization of cross-thread state. NSLock would also work; we use
-  // unfair_lock for vocabulary parity with the planned Segmenter pattern.
-  private static var unsafeLock = os_unfair_lock_s()
+    /// os_unfair_lock is the iOS 13+ recommended primitive for cheap
+    /// serialization of cross-thread state. NSLock would also work; we use
+    /// unfair_lock for vocabulary parity with the planned Segmenter pattern.
+    private static var unsafeLock = os_unfair_lock_s()
 
-  private static var _blurSigma: Float = 5.0
-  private static var _maskHardness: Float = 0.5
-  private static var _maskThreshold: Float = 0.7
-  // Short-side (in px) the camera buffer is downscaled to before MediaPipe
-  // segmentation runs. The produced mask is lower-res; the composite upsamples
-  // it with a LINEAR sampler, so the quality cost is small and the segmentation
-  // cost drop is large.
-  private static var _targetShortSide: Int = 384
-  // When true, the GPU/segmentation/ingest timing instrument logs under os_log
-  // category "Perf". Off by default; a debug build or a JS toggle turns it on
-  // to confirm the bottleneck on an EAS build (no attachable profiler there).
-  private static var _debugTiming = false
+    private static var _blurSigma: Float = 5.0
+    private static var _maskHardness: Float = 0.5
+    private static var _maskThreshold: Float = 0.7
+    /// Short-side (in px) the camera buffer is downscaled to before MediaPipe
+    /// segmentation runs. The produced mask is lower-res; the composite upsamples
+    /// it with a LINEAR sampler, so the quality cost is small and the segmentation
+    /// cost drop is large.
+    private static var _targetShortSide: Int = 384
+    /// When true, the GPU/segmentation/ingest timing instrument logs under os_log
+    /// category "Perf". Off by default; a debug build or a JS toggle turns it on
+    /// to confirm the bottleneck on an EAS build (no attachable profiler there).
+    private static var _debugTiming = false
 
-  public static var blurSigma: Float {
-    get {
-      os_unfair_lock_lock(&unsafeLock)
-      defer { os_unfair_lock_unlock(&unsafeLock) }
-      return _blurSigma
+    public static var blurSigma: Float {
+        get {
+            os_unfair_lock_lock(&unsafeLock)
+            defer { os_unfair_lock_unlock(&unsafeLock) }
+            return _blurSigma
+        }
+        set {
+            let clamped = min(max(newValue, 0.5), 7.0)
+            os_unfair_lock_lock(&unsafeLock)
+            _blurSigma = clamped
+            os_unfair_lock_unlock(&unsafeLock)
+        }
     }
-    set {
-      let clamped = min(max(newValue, 0.5), 7.0)
-      os_unfair_lock_lock(&unsafeLock)
-      _blurSigma = clamped
-      os_unfair_lock_unlock(&unsafeLock)
-    }
-  }
 
-  public static var maskHardness: Float {
-    get {
-      os_unfair_lock_lock(&unsafeLock)
-      defer { os_unfair_lock_unlock(&unsafeLock) }
-      return _maskHardness
-    }
-    set {
-      let clamped = min(max(newValue, 0.0), 1.0)
-      os_unfair_lock_lock(&unsafeLock)
-      _maskHardness = clamped
-      os_unfair_lock_unlock(&unsafeLock)
+    public static var maskHardness: Float {
+        get {
+            os_unfair_lock_lock(&unsafeLock)
+            defer { os_unfair_lock_unlock(&unsafeLock) }
+            return _maskHardness
+        }
+        set {
+            let clamped = min(max(newValue, 0.0), 1.0)
+            os_unfair_lock_lock(&unsafeLock)
+            _maskHardness = clamped
+            os_unfair_lock_unlock(&unsafeLock)
+        }
     }
-  }
 
-  public static var maskThreshold: Float {
-    get {
-      os_unfair_lock_lock(&unsafeLock)
-      defer { os_unfair_lock_unlock(&unsafeLock) }
-      return _maskThreshold
-    }
-    set {
-      let clamped = min(max(newValue, 0.05), 0.95)
-      os_unfair_lock_lock(&unsafeLock)
-      _maskThreshold = clamped
-      os_unfair_lock_unlock(&unsafeLock)
+    public static var maskThreshold: Float {
+        get {
+            os_unfair_lock_lock(&unsafeLock)
+            defer { os_unfair_lock_unlock(&unsafeLock) }
+            return _maskThreshold
+        }
+        set {
+            let clamped = min(max(newValue, 0.05), 0.95)
+            os_unfair_lock_lock(&unsafeLock)
+            _maskThreshold = clamped
+            os_unfair_lock_unlock(&unsafeLock)
+        }
     }
-  }
 
-  public static var targetShortSide: Int {
-    get {
-      os_unfair_lock_lock(&unsafeLock)
-      defer { os_unfair_lock_unlock(&unsafeLock) }
-      return _targetShortSide
+    public static var targetShortSide: Int {
+        get {
+            os_unfair_lock_lock(&unsafeLock)
+            defer { os_unfair_lock_unlock(&unsafeLock) }
+            return _targetShortSide
+        }
+        set {
+            let clamped = min(max(newValue, 128), 1080)
+            os_unfair_lock_lock(&unsafeLock)
+            _targetShortSide = clamped
+            os_unfair_lock_unlock(&unsafeLock)
+        }
     }
-    set {
-      let clamped = min(max(newValue, 128), 1080)
-      os_unfair_lock_lock(&unsafeLock)
-      _targetShortSide = clamped
-      os_unfair_lock_unlock(&unsafeLock)
-    }
-  }
 
-  public static var debugTiming: Bool {
-    get {
-      os_unfair_lock_lock(&unsafeLock)
-      defer { os_unfair_lock_unlock(&unsafeLock) }
-      return _debugTiming
-    }
-    set {
-      os_unfair_lock_lock(&unsafeLock)
-      _debugTiming = newValue
-      os_unfair_lock_unlock(&unsafeLock)
+    public static var debugTiming: Bool {
+        get {
+            os_unfair_lock_lock(&unsafeLock)
+            defer { os_unfair_lock_unlock(&unsafeLock) }
+            return _debugTiming
+        }
+        set {
+            os_unfair_lock_lock(&unsafeLock)
+            _debugTiming = newValue
+            os_unfair_lock_unlock(&unsafeLock)
+        }
     }
-  }
 
-  public static func reset() {
-    os_unfair_lock_lock(&unsafeLock)
-    _blurSigma = 5.0
-    _maskHardness = 0.5
-    _maskThreshold = 0.7
-    _targetShortSide = 384
-    // debugTiming is intentionally NOT reset here: it is an instrument toggle,
-    // not an effect parameter, so resetEffectTuning() should not silence it.
-    os_unfair_lock_unlock(&unsafeLock)
-  }
+    public static func reset() {
+        os_unfair_lock_lock(&unsafeLock)
+        _blurSigma = 5.0
+        _maskHardness = 0.5
+        _maskThreshold = 0.7
+        _targetShortSide = 384
+        // debugTiming is intentionally NOT reset here: it is an instrument toggle,
+        // not an effect parameter, so resetEffectTuning() should not silence it.
+        os_unfair_lock_unlock(&unsafeLock)
+    }
 }

package/ios/KaleidoscopeModule/KaleidoscopeModule.swift

@@ -6,39 +6,67 @@
 import ExpoModulesCore
 
 public class KaleidoscopeModule: Module {
-  public func definition() -> ModuleDefinition {
-    Name("RnWebrtcKaleidoscope")
+    public func definition() -> ModuleDefinition {
+        Name("RnWebrtcKaleidoscope")
 
-    OnCreate {
-      Registration.registerAll()
-    }
+        OnCreate {
+            Registration.registerAll()
+        }
 
-    Function("setBlurSigma") { (value: Float) in
-      EffectTuning.blurSigma = value
-    }
+        // The three numeric setters declare Double, not Float, even though the
+        // EffectTuning fields are Float. JS Number is natively a 64-bit double;
+        // ExpoModulesCore 1.12.26 on iOS bridges it cleanly to Swift Double but
+        // hits an Optional-wrap bug on Float (CastingException<Float> at
+        // DynamicRawType.swift:27 with 'Cannot cast Optional(0.6) to Float'),
+        // because the JSI bridge produces an Optional<Float> that the
+        // DynamicOptionalType wrapper does not unwrap before delegating to the
+        // wrapped raw type. The same Function shape on Android works with Float.
+        Function("setBlurSigma") { (value: Double) in
+            EffectTuning.blurSigma = Float(value)
+        }
 
-    Function("setMaskHardness") { (value: Float) in
-      EffectTuning.maskHardness = value
-    }
+        Function("setMaskHardness") { (value: Double) in
+            EffectTuning.maskHardness = Float(value)
+        }
 
-    Function("setMaskThreshold") { (value: Float) in
-      EffectTuning.maskThreshold = value
-    }
+        Function("setMaskThreshold") { (value: Double) in
+            EffectTuning.maskThreshold = Float(value)
+        }
 
-    // Segmentation perf control. A JS device-tier sets this to trade mask
-    // resolution for cost on lower-end devices (e.g. A11/iPhone X).
-    Function("setSegmentationTargetShortSide") { (value: Int) in
-      EffectTuning.targetShortSide = value
-    }
+        // Composite layer-stack channel. Mirrors android/.../KaleidoscopeModule.kt's
+        // setCompositeLayers: JS sends the active composite's ordered layer stack as a
+        // JSON string; CompositeLayers parses it into the snapshot the registered
+        // "composite" CompositeProcessor composites each frame.
+        Function("setCompositeLayers") { (json: String) in
+            CompositeLayers.set(json)
+        }
 
-    // Native perf instrument toggle; logs GPU/segmentation/ingest timings under
-    // the os_log "Perf" category. Off by default.
-    Function("setDebugTiming") { (value: Bool) in
-      EffectTuning.debugTiming = value
-    }
+        // Resolve a displayable file:// URI for a bundled background by its book id,
+        // for the picker's native thumbnails (react-native-webrtc-kaleidoscope/ui).
+        // Returns nil when the asset isn't bundled, so the JS resolver falls back to
+        // the source. Uses the same bundle lookup as an image layer, so
+        // the thumbnail and the rendered background resolve to the same file.
+        Function("resolveImageUri") { (id: String) -> String? in
+            BundledImage.bundledURL(for: id)?.absoluteString
+        }
+
+        // Segmentation perf control. A JS device-tier sets this to trade mask
+        // resolution for cost on lower-end devices (e.g. A11/iPhone X). Double
+        // and floor to Int for the same bridge-wrap reason as the Float setters
+        // above; JS Number is a double, and the Double path through ExpoModules
+        // does not hit the Optional<Float>-double-wrap bug.
+        Function("setSegmentationTargetShortSide") { (value: Double) in
+            EffectTuning.targetShortSide = Int(value)
+        }
+
+        // Native perf instrument toggle; logs GPU/segmentation/ingest timings under
+        // the os_log "Perf" category. Off by default.
+        Function("setDebugTiming") { (value: Bool) in
+            EffectTuning.debugTiming = value
+        }
 
-    Function("resetEffectTuning") {
-      EffectTuning.reset()
+        Function("resetEffectTuning") {
+            EffectTuning.reset()
+        }
     }
-  }
 }

package/ios/KaleidoscopeModule/Registration.swift

@@ -9,6 +9,18 @@
 // processor holds its own cached Metal/MediaPipe resources and is internally
 // thread-safe (see each processor's os_unfair_lock).
 //
+// The registry is now just TWO things (the effect-unification collapse, Phase C):
+//   - the four geometric transform ops (flip-x/flip-y/rotate-cw/rotate-ccw),
+//     sharing one TransformProcessor + transform.metalsrc, and
+//   - one "composite" compositor (CompositeProcessor): EVERY art effect (images,
+//     blur, generative shaders, masked subjects) is now a LAYER inside a
+//     composite, delivered out-of-band via setCompositeLayers (CompositeLayers) and
+//     composited per frame. The per-effect art processors (background-image-<id>,
+//     the bare generative names, blur) and their data-driven registration are
+//     gone; adding a preset is pure JS (a new Composite in the book), no Swift
+//     change and no native registration. Mirrors android/.../Registration.kt
+//     registering only the transforms + CompositeFactory under "composite".
+//
 // ProcessorProvider and the VideoFrameProcessorDelegate protocol come from
 // react-native-webrtc's Obj-C sources, imported as a Clang module (the consumer
 // enables `:modular_headers => true`; see Kaleidoscope.podspec and app.plugin.js).
@@ -20,67 +32,34 @@
 // types (RTCVideoFrame etc.) come from WebRTC.framework via `import WebRTC`.
 
 import Foundation
+import os.log
 import WebRTC
 #if canImport(livekit_react_native_webrtc)
-import livekit_react_native_webrtc
+    import livekit_react_native_webrtc
 #elseif canImport(react_native_webrtc)
-import react_native_webrtc
+    import react_native_webrtc
 #endif
 
 public enum Registration {
-  public static func registerAll() {
-    // Geometric reorientation ops. flip-x is the corrected screen-horizontal
-    // mirror that replaces the old "mirror" effect; the other three are new.
-    // All four share TransformProcessor + transform.metalsrc; the buffer-space
-    // rotation correction lives only in Orientation.swift.
-    ProcessorProvider.addProcessor(TransformProcessor(op: .flipX), forName: "flip-x")
-    ProcessorProvider.addProcessor(TransformProcessor(op: .flipY), forName: "flip-y")
-    ProcessorProvider.addProcessor(TransformProcessor(op: .rotateCW), forName: "rotate-cw")
-    ProcessorProvider.addProcessor(TransformProcessor(op: .rotateCCW), forName: "rotate-ccw")
-    ProcessorProvider.addProcessor(BlurProcessor(), forName: "blur")
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "debug-resolutions"),
-      forName: "background-image-debug-resolutions"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "dark-office"),
-      forName: "background-image-dark-office"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "light-office"),
-      forName: "background-image-light-office"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "home-light"),
-      forName: "background-image-home-light"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "home-dark"),
-      forName: "background-image-home-dark"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "nature-light"),
-      forName: "background-image-nature-light"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "nature-dark"),
-      forName: "background-image-nature-dark"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "stylized-light"),
-      forName: "background-image-stylized-light"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "stylized-dark"),
-      forName: "background-image-stylized-dark"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "simiancraft-light"),
-      forName: "background-image-simiancraft-light"
-    )
-    ProcessorProvider.addProcessor(
-      BackgroundImageProcessor(assetName: "simiancraft-dark"),
-      forName: "background-image-simiancraft-dark"
-    )
-  }
+    private static let log = OSLog(subsystem: "com.simiancraft.kaleidoscope", category: "Registration")
+
+    public static func registerAll() {
+        // Geometric reorientation ops. flip-x is the corrected screen-horizontal
+        // mirror that replaces the old "mirror" effect; the other three are new.
+        // All four share TransformProcessor + transform.metalsrc; the buffer-space
+        // rotation correction lives only in Orientation.swift. The registry-parity
+        // test pins these literals plus "composite".
+        ProcessorProvider.addProcessor(TransformProcessor(op: .flipX), forName: "flip-x")
+        ProcessorProvider.addProcessor(TransformProcessor(op: .flipY), forName: "flip-y")
+        ProcessorProvider.addProcessor(TransformProcessor(op: .rotateCW), forName: "rotate-cw")
+        ProcessorProvider.addProcessor(TransformProcessor(op: .rotateCCW), forName: "rotate-ccw")
+
+        // One composite compositor serves EVERY art effect; the active layer stack is
+        // data, delivered out-of-band via setCompositeLayers (CompositeLayers) and read per
+        // frame. A single registered instance, like the transform processors. Mirrors
+        // android/.../Registration.kt registering CompositeFactory under "composite".
+        ProcessorProvider.addProcessor(CompositeProcessor(), forName: "composite")
+
+        os_log("registered 4 transform op(s) + 1 composite compositor", log: log, type: .info)
+    }
 }