react-native-skia-webgpu-video 0.1.0-beta.1 → 0.1.0-beta.3

package/scripts/skia-canvas-patch.json

@@ -1,5 +1,5 @@
 {
-  "version": "3.7.0",
+  "version": "3.7.1",
   "files": {
     "cpp/api/JsiSkCanvas.h": {
       "before": "91e6f97f34bef620740fabf7a6bb516c462a18daa72afb46ef5a84d1cd4f3158",

package/src/exportFrames.ts

@@ -18,13 +18,18 @@ import {
   reserveVideoMemory,
 } from "./memory";
 import type { VideoFrame } from "./types";
-import { blitYuvFrame, releaseYuvImports, yuvFrameOrigin } from "./yuvBlit";
+import {
+  blitNv12Frame,
+  blitYuvFrame,
+  releaseYuvImports,
+  yuvFrameOrigin,
+} from "./yuvBlit";
 import type { YuvImport } from "./yuvBlit";
 
-/** A decoded frame sampled in place, without an owned snapshot: on iOS its
- * IOSurface, on Android its producer's RGBA texture, which the decoder
- * buffer was just converted into. The decoder lease stays open while the
- * image is in use. */
+/** A decoded frame sampled in place, without an owned snapshot: a BGRA
+ * frame's IOSurface on iOS, or for a YUV frame (Android, iOS NV12) its
+ * producer's RGBA texture, which the decoder buffer was just converted into.
+ * The decoder lease stays open while the image is in use. */
 export type ExportFrameImport = {
   id: number;
   frame: VideoFrame;
@@ -37,7 +42,7 @@ export type ExportFrameImport = {
   aliases?: WeakSet<VideoFrame>;
 };
 
-/** Android: the texture a producer's decoded frames are converted into. */
+/** The texture a producer's YUV frames are converted into. */
 type ProducerTexture = { texture: GPUTexture; reservation: number };
 
 /** Per export session, keyed by producer. `enabled` turns off after a first
@@ -48,7 +53,7 @@ export type ExportFrameImports = {
   producers: Set<number>;
   enabled: boolean;
   imported: boolean;
-  /** Android: one RGBA texture per producer, rewritten by each frame. */
+  /** One RGBA texture per producer of YUV frames, rewritten by each frame. */
   textures: Map<number, ProducerTexture>;
   /** Android: decoder buffers imported once (see yuvBlit.ts). */
   yuv: Map<number, YuvImport[]>;
@@ -65,6 +70,8 @@ type NativeBufferTexture = {
   nativeBuffer: bigint;
   region?: { x: number; y: number; width: number; height: number };
   rotation?: number;
+  /** iOS: the decoder's NV12 buffer (`framePixelFormat: 'yuv'`), or BGRA. */
+  format?: "nv12" | "bgra8";
 };
 
 // Captured on the RN runtime. Android samples its ImageReader YUV frames
@@ -117,8 +124,8 @@ export const releaseExportFrame = (entry: ExportFrameImport): void => {
   entry.frame.dispose?.();
 };
 
-/** Android: releases a producer's texture and decoder buffer imports. Call
- * only once its last frame's GPU work has completed. */
+/** Releases a producer's texture and decoder buffer imports. Call only once
+ * its last frame's GPU work has completed. */
 const releaseProducerTexture = (
   imports: ExportFrameImports,
   producer: number,
@@ -133,10 +140,11 @@ const releaseProducerTexture = (
   }
 };
 
-/** Android: converts a decoded ImageReader frame into its producer's RGBA
- * texture, wrapped as a Skia image. The conversion is submitted ahead of the
- * frame's drawing, on the same queue; the export's checked flush waits for
- * both before the lease or the texture is reused. */
+/** Converts a decoded YUV frame (an Android ImageReader buffer, or an iOS
+ * NV12 one) into its producer's RGBA texture, wrapped as a Skia image. The
+ * conversion is submitted ahead of the frame's drawing, on the same queue;
+ * the export's checked flush waits for both before the lease or the texture
+ * is reused. */
 const importYuvExportFrame = (
   imports: ExportFrameImports,
   frame: VideoFrame,
@@ -158,12 +166,17 @@ const importYuvExportFrame = (
   try {
     entry.nativeFrame = createNativeVideoGpuFrame(source.nativeBuffer);
     const rotation = source.rotation ?? 0;
-    const origin = yuvFrameOrigin(
-      source.region!,
-      rotation,
-      entry.nativeFrame,
-      frame,
-    );
+    // iOS NV12: the whole buffer, in decoded orientation.
+    const nv12 = source.format === "nv12";
+    if (
+      nv12 &&
+      (entry.nativeFrame.width !== frame.width ||
+        entry.nativeFrame.height !== frame.height)
+    )
+      throw new Error("Native video buffer dimensions mismatch");
+    const origin = nv12
+      ? { x: 0, y: 0 }
+      : yuvFrameOrigin(source.region!, rotation, entry.nativeFrame, frame);
     const device = getVideoGpuDevice();
     target = imports.textures.get(producer);
     if (
@@ -199,17 +212,19 @@ const importYuvExportFrame = (
     }
     // Can throw after submitting: drain below before giving the lease back.
     submitted = true;
-    blitYuvFrame(
-      device,
-      imports.yuv,
-      producer,
-      entry.nativeFrame,
-      rotation,
-      origin,
-      frame.width,
-      frame.height,
-      target.texture,
-    );
+    if (nv12) blitNv12Frame(device, entry.nativeFrame, target.texture);
+    else
+      blitYuvFrame(
+        device,
+        imports.yuv,
+        producer,
+        entry.nativeFrame,
+        rotation,
+        origin,
+        frame.width,
+        frame.height,
+        target.texture,
+      );
     entry.image = Skia.Image.MakeImageFromGPUTexture(target.texture);
     if (!entry.image) throw new Error("Cannot wrap the decoded frame");
     return entry;
@@ -233,15 +248,18 @@ const importYuvExportFrame = (
 };
 
 /** Wraps a decoded BGRA CVPixelBuffer's IOSurface as a Skia image, read-only.
- * The decoder already finished writing it; no producer fence is needed. On
- * Android, converts the decoded YUV frame instead (see importYuvExportFrame). */
+ * The decoder already finished writing it; no producer fence is needed. A
+ * YUV frame (Android, iOS NV12) is converted instead (see importYuvExportFrame). */
 export const importExportFrame = (
   frame: VideoFrame,
   imports?: ExportFrameImports,
 ): ExportFrameImport => {
   "worklet";
-  if (convertsYuv) {
-    if (!imports) throw new Error("Android in-place frames need their imports");
+  if (
+    convertsYuv ||
+    (frame.texture as NativeBufferTexture).format === "nv12"
+  ) {
+    if (!imports) throw new Error("YUV in-place frames need their imports");
     return importYuvExportFrame(imports, frame);
   }
   const entry: ExportFrameImport = {

package/src/frameInterop.ts

@@ -15,7 +15,12 @@ import {
   releaseVideoMemory,
   reserveVideoMemory,
 } from "./memory";
-import { blitYuvFrame, releaseYuvImports, yuvFrameOrigin } from "./yuvBlit";
+import {
+  blitNv12Frame,
+  blitYuvFrame,
+  releaseYuvImports,
+  yuvFrameOrigin,
+} from "./yuvBlit";
 import type { YuvImport } from "./yuvBlit";
 
 type FrameTexture =
@@ -26,6 +31,8 @@ type FrameTexture =
        * this crop rectangle turned clockwise by `rotation` for display. */
       region?: { x: number; y: number; width: number; height: number };
       rotation?: number;
+      /** iOS: the decoder's NV12 buffer (`framePixelFormat: 'yuv'`), or BGRA. */
+      format?: "nv12" | "bgra8";
     }
   | {
       kind: "rgba";
@@ -441,6 +448,7 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
         nativeSource = createNativeVideoGpuFrame(texture.nativeBuffer);
         const region = texture.region;
         const rotation = texture.rotation ?? 0;
+        const nv12 = !region && texture.format === "nv12";
         let origin = { x: 0, y: 0 };
         if (region) {
           // An Android codec buffer: padded, YUV, in encoded orientation. The
@@ -449,6 +457,12 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
           // driver-defined YUV formats "bgra8", yet Dawn imports them as YUV:
           // the native side checked the format, so the label is not read.
           origin = yuvFrameOrigin(region, rotation, nativeSource, frame);
+        } else if (nv12) {
+          if (
+            nativeSource.width !== frame.width ||
+            nativeSource.height !== frame.height
+          )
+            throw new Error("Native video buffer dimensions mismatch");
         } else if (
           // RN WebGPU labels a single-plane Android RGBA AHB "bgra8". Dawn
           // reads the actual AHB format; the string distinguishes RGB from
@@ -478,6 +492,9 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
             frame.height,
             cache.uploadTexture!,
           );
+        } else if (nv12) {
+          // Decoded orientation, as below: drawVideoFrame applies rotation.
+          blitNv12Frame(device, nativeSource, cache.uploadTexture!);
         } else {
           // Keep decoded orientation: drawVideoFrame applies rotation once.
           device.queue.copyExternalImageToTexture(

package/src/types.ts

@@ -149,6 +149,26 @@ export type VideoPlayer = {
   on(name: "error", listener: (error: any) => void): () => void;
 };
 
+/**
+ * The pixel format decoded frames are handed over in, before they are drawn.
+ *
+ * - `yuv`: the decoder's own YUV 4:2:0 buffers, converted to RGB on the GPU
+ *   when a frame is drawn. A frame takes 1.5 bytes a pixel rather than 4
+ *   (12.4 MB rather than 33.2 MB in 4K), so the decoder holds more frames
+ *   ahead for the same memory. On Android only 8-bit SDR tracks decoded at
+ *   their own size; other tracks stay `rgba`.
+ * - `rgba`: frames converted to RGB as they are decoded (iOS: by VideoToolbox,
+ *   Android: by an OpenGL pass), which a GPU export samples in place.
+ * - `auto`: `yuv`, on both platforms.
+ *
+ * Camera footage (Rec.709, or HLG tone mapped to Rec.709) renders the same
+ * either way: both convert with the frame's YCbCr matrix and range. On iOS,
+ * `rgba` also converts a file tagged with other primaries or another SDR
+ * transfer function (Display P3, sRGB) to Rec.709; `yuv` takes its samples as
+ * Rec.709, as Android does in both formats.
+ */
+export type VideoFramePixelFormat = "auto" | "yuv" | "rgba";
+
 /**
  * Represents a video composition.
  */
@@ -171,6 +191,13 @@ export type VideoComposition = {
    * until its decoder has opened. The export always waits for it.
    */
   lazyDecoders?: boolean;
+  /**
+   * The pixel format the video items' frames are decoded into, for the
+   * preview and the export, see {@link VideoFramePixelFormat}.
+   *
+   * @default 'auto'
+   */
+  framePixelFormat?: VideoFramePixelFormat;
 };
 
 type VideoCompositionItemBase = {

package/src/videoCompositionPlayer.ts

@@ -80,7 +80,8 @@ type UseVideoCompositionPlayerOptions<T = undefined> = {
    * video only changes at its own frame rate. A drawer whose picture depends on
    * time only through the frames (or only at known moments, a transition say)
    * returns false elsewhere, and the player then redraws when a new frame is
-   * decoded instead. Must be a worklet.
+   * decoded instead. Ignored when `drawWhenPaused` is `true`, which redraws at
+   * every vsync. Must be a worklet.
    */
   redrawOnTime?: (previousTime: number, currentTime: number) => boolean;
   /**
@@ -88,8 +89,9 @@ type UseVideoCompositionPlayerOptions<T = undefined> = {
    * of both in the same one. On Android the import (YUV blit and its checked
    * GPU wait) and the draw (snapshot and its checked GPU wait) cost about as
    * much as each other, and together they overrun a 90–120 Hz frame budget on
-   * the UI thread. The picture then trails the clock by one vsync.
-   * The first frame of a session is still drawn as soon as it is imported.
+   * the UI thread. The option applies on iOS too. The picture then trails the
+   * clock by one vsync. The first frame of a session is still drawn as soon
+   * as it is imported.
    * @default false
    */
   splitImportAndDraw?: boolean;
@@ -317,6 +319,8 @@ export const useVideoComposition = <T = undefined>({
   const lastImportedVersion = useSharedValue(-1);
   // Whether the previous vsync imported and left its draw to this one.
   const drawDeferred = useSharedValue(false);
+  // Whether the previous vsync brought new frames.
+  const importedLastVsync = useSharedValue(false);
   // A new drawFrame can draw something else from the same time and frames.
   useEffect(() => {
     lastDrawnTime.value = -1;
@@ -382,14 +386,22 @@ export const useVideoComposition = <T = undefined>({
       sourcesGeneration.value = generation;
       const currentTime = framesExtractor.currentTime;
       const framesVersion = framesExtractor.framesVersion;
-      if (splitImportAndDraw && framesVersion !== lastImportedVersion.value) {
+      const imported =
+        splitImportAndDraw && framesVersion !== lastImportedVersion.value;
+      const importedBefore = importedLastVsync.value;
+      // Version 0: no frame decoded yet.
+      importedLastVsync.value = imported && framesVersion > 0;
+      if (imported) {
         // ownVideoFrames above has just imported the new frames: leave the
-        // draw to the next vsync, which finds them in the import cache. Never
-        // twice running: a frame that lands on the vsync owed a draw would
-        // push it again, and the frame before it would never be shown.
+        // draw to the next vsync, which finds them in the import cache. Not
+        // when the previous vsync brought new frames too: frames arriving at
+        // every vsync (60 fps, or a display gone down to 30 Hz) leave no vsync
+        // free for the draw, and the next frame would replace this one before
+        // it is shown. Never twice running either, for the same reason.
         lastImportedVersion.value = framesVersion;
         if (
           !drawDeferred.value &&
+          !importedBefore &&
           currentFrame.value !== null &&
           frameGeneration.value === generation
         ) {

package/src/yuvBlit.ts

@@ -30,6 +30,8 @@ type YuvBlit = {
   sampler: GPUSampler;
   module: GPUShaderModule;
   pipelines: Map<GPUTextureFormat, GPURenderPipeline>;
+  /** The whole frame, as is: the uniforms of blitNv12Frame, written once. */
+  wholeFrame?: GPUBuffer;
 };
 
 // An ImageReader holds four buffers (GLFrameExtractor.READER_IMAGES); the
@@ -329,3 +331,67 @@ export const blitYuvFrame = (
     entry.wrapper.endAccess()?.fence.destroy();
   }
 };
+
+/**
+ * iOS: draws a whole NV12 decoder buffer into `target`, converting YUV to RGB
+ * with the buffer's YCbCr matrix and range, in its decoded orientation (the
+ * frame's rotation applies when it is drawn, as for BGRA frames). Submits the
+ * draw. AVAssetReader hands out buffers from a pool rather than a fixed few,
+ * so each frame is imported for this draw alone, unlike Android's: the import
+ * ends right after the submit (Dawn keeps the surface for the in-flight
+ * draw), while the caller holds the frame's lease until the draw completed.
+ */
+export const blitNv12Frame = (
+  device: GPUDevice,
+  source: NativeVideoFrame,
+  target: GPUTexture,
+): void => {
+  "worklet";
+  const blit = getYuvBlit(device);
+  const pipeline = getPipeline(device, blit, target.format);
+  if (!blit.wholeFrame) {
+    blit.wholeFrame = device.createBuffer({
+      label: "video NV12 blit uniforms",
+      size: 16,
+      usage: videoUniformBufferUsage,
+    });
+    device.queue.writeBuffer(blit.wholeFrame, 0, new Float32Array([1, 1, 0, 0]));
+  }
+  const external = device.importExternalTexture({
+    label: "video decoder buffer",
+    source,
+    sourceColorSpace,
+  });
+  try {
+    const encoder = device.createCommandEncoder({ label: "video NV12 blit" });
+    const pass = encoder.beginRenderPass({
+      label: "video NV12 blit",
+      colorAttachments: [
+        {
+          view: target.createView(),
+          loadOp: "clear",
+          clearValue: { r: 0, g: 0, b: 0, a: 1 },
+          storeOp: "store",
+        },
+      ],
+    });
+    pass.setPipeline(pipeline);
+    pass.setBindGroup(
+      0,
+      device.createBindGroup({
+        label: "video NV12 blit",
+        layout: blit.layout,
+        entries: [
+          { binding: 0, resource: external },
+          { binding: 1, resource: blit.sampler },
+          { binding: 2, resource: { buffer: blit.wholeFrame } },
+        ],
+      }),
+    );
+    pass.draw(3);
+    pass.end();
+    device.queue.submit([encoder.finish()]);
+  } finally {
+    (external as GPUExternalTexture & { destroy?: () => void }).destroy?.();
+  }
+};