react-native-skia-webgpu-video 0.1.0-beta.2 → 0.1.0-beta.3

package/src/types.ts

@@ -149,6 +149,26 @@ export type VideoPlayer = {
   on(name: "error", listener: (error: any) => void): () => void;
 };
 
+/**
+ * The pixel format decoded frames are handed over in, before they are drawn.
+ *
+ * - `yuv`: the decoder's own YUV 4:2:0 buffers, converted to RGB on the GPU
+ *   when a frame is drawn. A frame takes 1.5 bytes a pixel rather than 4
+ *   (12.4 MB rather than 33.2 MB in 4K), so the decoder holds more frames
+ *   ahead for the same memory. On Android only 8-bit SDR tracks decoded at
+ *   their own size; other tracks stay `rgba`.
+ * - `rgba`: frames converted to RGB as they are decoded (iOS: by VideoToolbox,
+ *   Android: by an OpenGL pass), which a GPU export samples in place.
+ * - `auto`: `yuv`, on both platforms.
+ *
+ * Camera footage (Rec.709, or HLG tone mapped to Rec.709) renders the same
+ * either way: both convert with the frame's YCbCr matrix and range. On iOS,
+ * `rgba` also converts a file tagged with other primaries or another SDR
+ * transfer function (Display P3, sRGB) to Rec.709; `yuv` takes its samples as
+ * Rec.709, as Android does in both formats.
+ */
+export type VideoFramePixelFormat = "auto" | "yuv" | "rgba";
+
 /**
  * Represents a video composition.
  */
@@ -171,6 +191,13 @@ export type VideoComposition = {
    * until its decoder has opened. The export always waits for it.
    */
   lazyDecoders?: boolean;
+  /**
+   * The pixel format the video items' frames are decoded into, for the
+   * preview and the export, see {@link VideoFramePixelFormat}.
+   *
+   * @default 'auto'
+   */
+  framePixelFormat?: VideoFramePixelFormat;
 };
 
 type VideoCompositionItemBase = {

package/src/videoCompositionPlayer.ts

@@ -319,6 +319,8 @@ export const useVideoComposition = <T = undefined>({
   const lastImportedVersion = useSharedValue(-1);
   // Whether the previous vsync imported and left its draw to this one.
   const drawDeferred = useSharedValue(false);
+  // Whether the previous vsync brought new frames.
+  const importedLastVsync = useSharedValue(false);
   // A new drawFrame can draw something else from the same time and frames.
   useEffect(() => {
     lastDrawnTime.value = -1;
@@ -384,14 +386,22 @@ export const useVideoComposition = <T = undefined>({
       sourcesGeneration.value = generation;
       const currentTime = framesExtractor.currentTime;
       const framesVersion = framesExtractor.framesVersion;
-      if (splitImportAndDraw && framesVersion !== lastImportedVersion.value) {
+      const imported =
+        splitImportAndDraw && framesVersion !== lastImportedVersion.value;
+      const importedBefore = importedLastVsync.value;
+      // Version 0: no frame decoded yet.
+      importedLastVsync.value = imported && framesVersion > 0;
+      if (imported) {
         // ownVideoFrames above has just imported the new frames: leave the
-        // draw to the next vsync, which finds them in the import cache. Never
-        // twice running: a frame that lands on the vsync owed a draw would
-        // push it again, and the frame before it would never be shown.
+        // draw to the next vsync, which finds them in the import cache. Not
+        // when the previous vsync brought new frames too: frames arriving at
+        // every vsync (60 fps, or a display gone down to 30 Hz) leave no vsync
+        // free for the draw, and the next frame would replace this one before
+        // it is shown. Never twice running either, for the same reason.
         lastImportedVersion.value = framesVersion;
         if (
           !drawDeferred.value &&
+          !importedBefore &&
           currentFrame.value !== null &&
           frameGeneration.value === generation
         ) {

package/src/yuvBlit.ts

@@ -30,6 +30,8 @@ type YuvBlit = {
   sampler: GPUSampler;
   module: GPUShaderModule;
   pipelines: Map<GPUTextureFormat, GPURenderPipeline>;
+  /** The whole frame, as is: the uniforms of blitNv12Frame, written once. */
+  wholeFrame?: GPUBuffer;
 };
 
 // An ImageReader holds four buffers (GLFrameExtractor.READER_IMAGES); the
@@ -329,3 +331,67 @@ export const blitYuvFrame = (
     entry.wrapper.endAccess()?.fence.destroy();
   }
 };
+
+/**
+ * iOS: draws a whole NV12 decoder buffer into `target`, converting YUV to RGB
+ * with the buffer's YCbCr matrix and range, in its decoded orientation (the
+ * frame's rotation applies when it is drawn, as for BGRA frames). Submits the
+ * draw. AVAssetReader hands out buffers from a pool rather than a fixed few,
+ * so each frame is imported for this draw alone, unlike Android's: the import
+ * ends right after the submit (Dawn keeps the surface for the in-flight
+ * draw), while the caller holds the frame's lease until the draw completed.
+ */
+export const blitNv12Frame = (
+  device: GPUDevice,
+  source: NativeVideoFrame,
+  target: GPUTexture,
+): void => {
+  "worklet";
+  const blit = getYuvBlit(device);
+  const pipeline = getPipeline(device, blit, target.format);
+  if (!blit.wholeFrame) {
+    blit.wholeFrame = device.createBuffer({
+      label: "video NV12 blit uniforms",
+      size: 16,
+      usage: videoUniformBufferUsage,
+    });
+    device.queue.writeBuffer(blit.wholeFrame, 0, new Float32Array([1, 1, 0, 0]));
+  }
+  const external = device.importExternalTexture({
+    label: "video decoder buffer",
+    source,
+    sourceColorSpace,
+  });
+  try {
+    const encoder = device.createCommandEncoder({ label: "video NV12 blit" });
+    const pass = encoder.beginRenderPass({
+      label: "video NV12 blit",
+      colorAttachments: [
+        {
+          view: target.createView(),
+          loadOp: "clear",
+          clearValue: { r: 0, g: 0, b: 0, a: 1 },
+          storeOp: "store",
+        },
+      ],
+    });
+    pass.setPipeline(pipeline);
+    pass.setBindGroup(
+      0,
+      device.createBindGroup({
+        label: "video NV12 blit",
+        layout: blit.layout,
+        entries: [
+          { binding: 0, resource: external },
+          { binding: 1, resource: blit.sampler },
+          { binding: 2, resource: { buffer: blit.wholeFrame } },
+        ],
+      }),
+    );
+    pass.draw(3);
+    pass.end();
+    device.queue.submit([encoder.finish()]);
+  } finally {
+    (external as GPUExternalTexture & { destroy?: () => void }).destroy?.();
+  }
+};