react-native-webrtc-kaleidoscope 2.0.0 → 2.1.0

package/src/web/segmenter.ts

@@ -1,107 +0,0 @@
-// MediaPipe Selfie Segmentation loader. Single shared instance across every
-// web effect that needs a person mask (blur, background-image, future
-// procedural backgrounds). The script is loaded once from a CDN; the
-// segmenter instance is cached for the lifetime of the page.
-//
-// CDN-loaded rather than bundled because @mediapipe/selfie_segmentation
-// ships a WASM blob and asset graph that does not play nicely with Metro;
-// we listed the npm package as an optionalDependency for typing only.
-
-const CDN_BASE = 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation';
-
-export type SegmenterResults = {
-  image: CanvasImageSource;
-  segmentationMask: CanvasImageSource;
-};
-
-type SegmenterOptions = { selfieMode?: boolean; modelSelection?: number };
-
-export type Segmenter = {
-  initialize(): Promise<void>;
-  setOptions(opts: SegmenterOptions): void;
-  onResults(cb: (r: SegmenterResults) => void): void;
-  send(input: { image: CanvasImageSource }): Promise<void>;
-  close(): Promise<void>;
-};
-
-type SegmenterCtor = new (config: { locateFile: (file: string) => string }) => Segmenter;
-
-let segmenterPromise: Promise<Segmenter> | null = null;
-
-// Decoupled mask source (mirrors Android Mask.kt / iOS Segmenter.swift): the
-// render path reads the most recent mask without ever awaiting a fresh one, and
-// a single in-flight segmentation refreshes it. `onResults` is registered once,
-// at load, below.
-let latestResults: SegmenterResults | null = null;
-let inFlight = false;
-let activeSegmenter: Segmenter | null = null;
-
-const loadScript = (src: string): Promise<void> =>
-  new Promise((resolve, reject) => {
-    const existing = document.querySelector(`script[src="${src}"]`);
-    if (existing) {
-      if (existing.getAttribute('data-loaded') === 'true') {
-        resolve();
-        return;
-      }
-      existing.addEventListener('load', () => resolve(), { once: true });
-      existing.addEventListener('error', () => reject(new Error(`failed to load ${src}`)), {
-        once: true,
-      });
-      return;
-    }
-    const script = document.createElement('script');
-    script.src = src;
-    script.crossOrigin = 'anonymous';
-    script.addEventListener('load', () => {
-      script.setAttribute('data-loaded', 'true');
-      resolve();
-    });
-    script.addEventListener('error', () => reject(new Error(`failed to load ${src}`)));
-    document.head.appendChild(script);
-  });
-
-export const loadSegmenter = (): Promise<Segmenter> => {
-  if (segmenterPromise) return segmenterPromise;
-  segmenterPromise = (async () => {
-    await loadScript(`${CDN_BASE}/selfie_segmentation.js`);
-    const SegCtor = (globalThis as unknown as { SelfieSegmentation?: SegmenterCtor })
-      .SelfieSegmentation;
-    if (!SegCtor) {
-      throw new Error(
-        'kaleidoscope: MediaPipe Selfie Segmentation script loaded but SelfieSegmentation global is missing',
-      );
-    }
-    const seg = new SegCtor({ locateFile: (file) => `${CDN_BASE}/${file}` });
-    seg.setOptions({ modelSelection: 1, selfieMode: false });
-    await seg.initialize();
-    // Register once; every completed segmentation updates the cache.
-    seg.onResults((r) => {
-      latestResults = r;
-    });
-    activeSegmenter = seg;
-    return seg;
-  })();
-  return segmenterPromise;
-};
-
-/** Most recent completed mask, or null before the first result. Non-blocking. */
-export const getLatestMask = (): SegmenterResults | null => latestResults;
-
-/**
- * Kick a new segmentation if none is in flight; returns immediately. The result
- * lands in `getLatestMask()` via the `onResults` handler registered at load.
- * No-op until the segmenter has finished loading. `inFlight` clears when `send`
- * settles, so a failed call cannot wedge the pipeline.
- */
-export const requestMaskIfIdle = (image: CanvasImageSource): void => {
-  const seg = activeSegmenter;
-  if (!seg || inFlight) return;
-  inFlight = true;
-  seg
-    .send({ image })
-    .catch(() => {})
-    .finally(() => {
-      inFlight = false;
-    });
-};

package/src/web/shaders.generated.ts

@@ -1,56 +0,0 @@
-// @generated by scripts/build-shaders.ts from shaders/. DO NOT EDIT.
-// Run `bun run build:shaders` to regenerate.
-
-export const PASSTHROUGH_VERT_SRC = `#version 300 es
-precision highp float;
-out highp vec2 vUv;
-void main() {
-  vec2 p = vec2(float((gl_VertexID & 1) << 1), float(gl_VertexID & 2));
-  vUv = p * 0.5;
-  gl_Position = vec4(p - 1.0, 0.0, 1.0);
-}
-`;
-
-export const COMPOSITE_FRAG_SRC = `#version 300 es
-precision mediump float;
-uniform sampler2D uOriginal;
-uniform sampler2D uBackground;
-uniform sampler2D uMask;
-uniform vec2 uBgUvScale;
-uniform vec2 uBgUvOffset;
-uniform vec2 uMaskUvScale;
-uniform vec2 uMaskUvOffset;
-uniform float uMaskLo;
-uniform float uMaskHi;
-in highp vec2 vUv;
-out vec4 oColor;
-void main() {
-  vec2 maskUv = vUv * uMaskUvScale + uMaskUvOffset;
-  float raw = texture(uMask, maskUv).r;
-  float safeHi = max(uMaskHi, uMaskLo + 0.001);
-  float m = smoothstep(uMaskLo, safeHi, raw);
-  vec3 orig = texture(uOriginal, vUv).rgb;
-  vec2 bgUv = clamp(vUv * uBgUvScale + uBgUvOffset, 0.0, 1.0);
-  vec3 bg = texture(uBackground, bgUv).rgb;
-  oColor = vec4(mix(bg, orig, m), 1.0);
-}
-`;
-
-export const BLUR_FRAG_SRC = `#version 300 es
-precision mediump float;
-uniform sampler2D uTex;
-uniform vec2 uAxis;
-uniform float uWeights[5];
-uniform float uOffsets[5];
-in highp vec2 vUv;
-out vec4 oColor;
-void main() {
-  vec4 color = texture(uTex, vUv) * uWeights[0];
-  for (int i = 1; i < 5; i++) {
-    vec2 off = uAxis * uOffsets[i];
-    color += texture(uTex, vUv + off) * uWeights[i];
-    color += texture(uTex, vUv - off) * uWeights[i];
-  }
-  oColor = color;
-}
-`;

package/src/web/shaders.ts

@@ -1,31 +0,0 @@
-// Web GLSL entry point. The shader source is generated from the canonical
-// `shaders/*.frag` / `*.vert` by `bun run build:shaders` into
-// `./shaders.generated`; this module re-exports it so web effect code imports
-// from one stable path. Do not hand-edit the generated file.
-//
-// The three shared shaders:
-//   - PASSTHROUGH_VERT_SRC: full-screen quad via gl_VertexID (no VAO/VBO);
-//     caller does gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4).
-//   - BLUR_FRAG_SRC: separable 1D Gaussian, 9-tap precomputed kernel
-//     (uWeights[9] / uOffsets[9], no per-pixel exp). The caller computes the
-//     kernel from sigma on the CPU and uploads the arrays; see
-//     src/web/effects/blur.ts (blurKernel).
-//   - COMPOSITE_FRAG_SRC: mix(background, original, mask). Identical body
-//     across web, Android, and (post-transpile) iOS.
-//
-// Texture-orientation convention (lives in the host uniforms, NOT the shader,
-// so the composite body stays identical across runtimes):
-//   - uOriginal / uBackground land with semantic "top of source" at GL v=1.
-//     Web uploads with UNPACK_FLIP_Y_WEBGL=true (bg staged through an
-//     OffscreenCanvas first, since flipY does not apply to ImageBitmap).
-//   - uMask is sampled at vUv * uMaskUvScale + uMaskUvOffset. Web direct-uploads
-//     the mask with flipY=false and V-flips in the sampling uniforms
-//     ((1,-1)/(0,1)); canvas-staging would mangle MediaPipe's soft confidence
-//     values via premultiplied alpha and break the hardness slider. Android's
-//     readback round-trip leaves the mask aligned, so its uniforms are identity.
-//   - uBgUvScale / uBgUvOffset cover-fit center-crop the background when its
-//     aspect ratio differs from the output; (1,1)/(0,0) for a full-size bg.
-//   - uMaskLo / uMaskHi parameterize the smoothstep, derived from a hardness
-//     factor via maskSmoothstepRange in src/web/tuning.ts (mirrors Android's
-//     MaskTuning.smoothstepRange).
-export { BLUR_FRAG_SRC, COMPOSITE_FRAG_SRC, PASSTHROUGH_VERT_SRC } from './shaders.generated';

package/src/web/tuning.ts

@@ -1,67 +0,0 @@
-// Mutable runtime state for the web GLSL effects. Mirrors
-// android/.../EffectTuning.kt and ios/.../EffectTuning.swift.
-//
-// `src/index.web.ts` exports setBlurSigma / setMaskHardness that write
-// here; per-frame transforms in `src/web/effects/*.ts` read here when
-// uploading uniforms. JS calls take effect on the next frame.
-
-const clamp = (value: number, lo: number, hi: number): number => Math.min(Math.max(value, lo), hi);
-
-class EffectTuningState {
-  blurSigma = 5;
-  maskHardness = 0.5;
-  maskThreshold = 0.7;
-  // Native-only knobs stored here for cross-platform API parity; the web
-  // MediaPipe pipeline does not currently consume them.
-  segmentationTargetShortSide = 384;
-  debugTiming = false;
-
-  setBlurSigma(value: number): void {
-    this.blurSigma = clamp(value, 0.5, 7);
-  }
-
-  setMaskHardness(value: number): void {
-    this.maskHardness = clamp(value, 0, 1);
-  }
-
-  setMaskThreshold(value: number): void {
-    this.maskThreshold = clamp(value, 0.05, 0.95);
-  }
-
-  setSegmentationTargetShortSide(value: number): void {
-    this.segmentationTargetShortSide = clamp(value, 128, 1080);
-  }
-
-  setDebugTiming(value: boolean): void {
-    this.debugTiming = value;
-  }
-
-  reset(): void {
-    this.blurSigma = 5;
-    this.maskHardness = 0.5;
-    this.maskThreshold = 0.7;
-    this.segmentationTargetShortSide = 384;
-    this.debugTiming = false;
-  }
-}
-
-export const tuning = new EffectTuningState();
-
-/**
- * Derive a smoothstep (lo, hi) range from a hardness factor in [0, 1] and
- * a threshold in [0.05, 0.95]. Matches `MaskTuning.smoothstepRange` on the
- * Android side; keep in sync.
- *
- * hardness controls width: 0 = soft halo (wide transition), 1 = near-step.
- * threshold controls the center: 0.5 = neutral, higher = reject low-
- * confidence pixels (rejects chair-edge regions), lower = more inclusive.
- */
-export const maskSmoothstepRange = (
-  hardness: number,
-  threshold: number,
-): readonly [number, number] => {
-  const clampedHardness = clamp(hardness, 0, 1);
-  const clampedThreshold = clamp(threshold, 0.05, 0.95);
-  const width = 0.6 * (1 - clampedHardness) + 0.02;
-  return [clampedThreshold - width * 0.5, clampedThreshold + width * 0.5] as const;
-};