react-native-skia-webgpu-video 0.1.0-beta.3 → 0.1.0

package/src/gpuDevice.ts

@@ -8,7 +8,7 @@ import {
   importDevice,
   installWebGPU,
 } from "react-native-webgpu";
-import type { NativeVideoFrame } from "react-native-webgpu";
+import type { NativeVideoFrame, RNCanvasContext } from "react-native-webgpu";
 import RNSkiaVideoModule from "./RNSkiaVideoModule";
 
 // Evaluate on the RN runtime. NativeObject serializers carry these objects to
@@ -61,11 +61,36 @@ export const createNativeVideoGpuFrame = (
   return nativeWebGPU.createVideoFrameFromNativeBuffer(pointer);
 };
 
+/** Android: a canvas context presenting to a native window that is not a
+ * view, such as an encoder's input Surface. */
+export const createNativeWindowGpuContext = (
+  window: bigint,
+  width: number,
+  height: number,
+): RNCanvasContext => {
+  "worklet";
+  if (
+    typeof nativeWebGPU?.MakeWebGPUCanvasContextFromNativeWindow !== "function"
+  )
+    throw new Error(
+      "React Native WebGPU cannot present to a native window; 0.19.0 or later is required",
+    );
+  return nativeWebGPU.MakeWebGPUCanvasContextFromNativeWindow(
+    window,
+    width,
+    height,
+  );
+};
+
 export const nativeVideoTextureUsage =
   GPUTextureUsage.RENDER_ATTACHMENT |
   GPUTextureUsage.TEXTURE_BINDING |
   GPUTextureUsage.COPY_SRC;
 
+// An encoder's input window only receives copies of the frames Skia drew.
+export const encoderWindowTextureUsage =
+  GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.COPY_DST;
+
 // CPU RGBA uploads additionally need COPY_DST. Keep the native-buffer blit's
 // descriptor unchanged: that path renders into its destination attachment.
 export const rgbaVideoTextureUsage =

package/src/memory.ts

@@ -37,6 +37,12 @@ export const frameByteSize = (width: number, height: number): number => {
   return bytes;
 };
 
+/** The budget's counters, from any runtime; null without them. */
+export const videoMemoryBudget = (): VideoMemoryStats | null => {
+  "worklet";
+  return RNSkiaVideoModule.getMemoryBudgetStats?.() ?? null;
+};
+
 export const reserveVideoMemory = (bytes: number, label: string): number => {
   "worklet";
   return RNSkiaVideoModule.reserveMemory?.(bytes, label) ?? 0;

package/src/previewLifecycle.ts

@@ -5,6 +5,8 @@ import type { VideoFrame } from "./types";
 
 type PreviewResources = {
   surface?: SkSurface | null;
+  /** The preview's second buffer (see useVideoCompositionPlayer). */
+  spareSurface?: SkSurface | null;
   canvas?: SkCanvas | null;
   images?: Array<SkImage | null>;
   frames?: VideoFrame[];
@@ -33,10 +35,11 @@ export const disposePreviewResources = (resources: PreviewResources) => {
       frame.dispose?.();
       resources.frames.shift();
     }
-    resources.canvas?.dispose();
     resources.canvas = null;
     resources.surface?.dispose();
     resources.surface = null;
+    resources.spareSurface?.dispose();
+    resources.spareSurface = null;
     releaseVideoMemory(resources.reservation ?? 0);
     resources.reservation = 0;
   } catch (error) {

package/src/types.ts

@@ -157,15 +157,16 @@ export type VideoPlayer = {
  *   (12.4 MB rather than 33.2 MB in 4K), so the decoder holds more frames
  *   ahead for the same memory. On Android only 8-bit SDR tracks decoded at
  *   their own size; other tracks stay `rgba`.
- * - `rgba`: frames converted to RGB as they are decoded (iOS: by VideoToolbox,
- *   Android: by an OpenGL pass), which a GPU export samples in place.
+ * - `rgba` (deprecated): frames converted to RGB as they are decoded (iOS: by
+ *   VideoToolbox, Android: by an OpenGL pass). It no longer has an advantage:
+ *   exports run as fast in `yuv`, which renders the same colors, and a 4K60
+ *   preview keeps fewer frames on screen in `rgba`, whose frames take 4 bytes
+ *   a pixel. It still works; use `auto`.
  * - `auto`: `yuv`, on both platforms.
  *
- * Camera footage (Rec.709, or HLG tone mapped to Rec.709) renders the same
- * either way: both convert with the frame's YCbCr matrix and range. On iOS,
- * `rgba` also converts a file tagged with other primaries or another SDR
- * transfer function (Display P3, sRGB) to Rec.709; `yuv` takes its samples as
- * Rec.709, as Android does in both formats.
+ * Both convert with the frame's YCbCr matrix and range. On iOS, both convert
+ * HDR (HLG, PQ), BT.2020, Display P3 and sRGB files to Rec.709 through
+ * AVFoundation. Android takes SDR samples as Rec.709 in both formats.
  */
 export type VideoFramePixelFormat = "auto" | "yuv" | "rgba";
 
@@ -421,14 +422,19 @@ export type VideoEncoder = {
    */
   encodeFrame(texture: unknown, time: number): void;
   /**
-   * Lends the next pool buffer to draw into on the GPU: a CVPixelBuffer on
-   * iOS, an RGBA AHardwareBuffer on Android. Pass it back as a native-buffer
-   * frame (on Android with the sync-fds ending its GPU access, in `fences`),
-   * or release it. One at a time.
+   * iOS: lends the next pool buffer (a CVPixelBuffer) to draw into on the
+   * GPU. Pass it back as a native-buffer frame, or release it. One at a time.
    */
   acquireFrameBuffer?(): bigint;
-  /** Returns a lent buffer that will not be encoded. */
+  /** iOS: returns a lent buffer that will not be encoded. */
   releaseFrameBuffer?(): void;
+  /**
+   * Android: the codec's input Surface, as an ANativeWindow pointer for a
+   * WebGPU canvas context to present to. Each frame is then encoded as
+   * `{ kind: "window", present }`: the encoder sets the frame's time on the
+   * window and calls `present`, which presents the context.
+   */
+  acquireInputWindow?(): bigint;
   /*
    * Finish writing the video to the output file.
    */

package/src/videoCompositionPlayer.ts

@@ -19,10 +19,18 @@ import RNSkiaVideoModule from "./RNSkiaVideoModule";
 import useEventListener from "./utils/useEventListener";
 import { PixelRatio } from "react-native";
 import { scheduleOnRN } from "react-native-worklets";
-import { ownVideoFrames, releaseVideoFrameImage } from "./frameInterop";
+import {
+  ownVideoFrames,
+  releaseVideoFrameImage,
+  videoFrameImportWaits,
+} from "./frameInterop";
 import { frameByteSize, reserveVideoMemory } from "./memory";
 import { disposePreviewResources } from "./previewLifecycle";
 
+// Draws in a row the preview submits without waiting for the GPU: one frame
+// of a 24 fps video spans five vsyncs at 120 Hz, redrawn as time goes by.
+const maxUncheckedDraws = 6;
+
 type UseVideoCompositionPlayerOptions<T = undefined> = {
   /**
    * The video composition to play.
@@ -87,11 +95,10 @@ type UseVideoCompositionPlayerOptions<T = undefined> = {
   /**
    * Import a newly decoded frame in one vsync and draw it in the next, instead
    * of both in the same one. On Android the import (YUV blit and its checked
-   * GPU wait) and the draw (snapshot and its checked GPU wait) cost about as
-   * much as each other, and together they overrun a 90–120 Hz frame budget on
-   * the UI thread. The option applies on iOS too. The picture then trails the
-   * clock by one vsync. The first frame of a session is still drawn as soon
-   * as it is imported.
+   * GPU wait) and the draw cost about as much as each other, and together they
+   * overrun a 90–120 Hz frame budget on the UI thread. The option applies on
+   * iOS too. The picture then trails the clock by one vsync. The first frame
+   * of a session is still drawn as soon as it is imported.
    * @default false
    */
   splitImportAndDraw?: boolean;
@@ -276,6 +283,15 @@ export const useVideoComposition = <T = undefined>({
 
   const surfaceSharedValue = useSharedValue<SkSurface | null>(null);
   const canvasSharedValue = useSharedValue<SkCanvas | null>(null);
+  // The preview draws into two surfaces in turn, and shows each through an
+  // image sharing its texture (asImage): no copy per draw, as a snapshot
+  // made. The picture on screen stays whole while the next is drawn into the
+  // other surface.
+  const spareSurfaceSharedValue = useSharedValue<SkSurface | null>(null);
+  const spareCanvasSharedValue = useSharedValue<SkCanvas | null>(null);
+  const bufferImages = useSharedValue<Array<SkImage | null>>([]);
+  // The buffer on screen: 0, 1 (the spare), or -1 before the first draw.
+  const frontBuffer = useSharedValue(-1);
   const surfaceReservation = useSharedValue(0);
   const surfaceWidth = useSharedValue(0);
   const surfaceHeight = useSharedValue(0);
@@ -283,8 +299,10 @@ export const useVideoComposition = <T = undefined>({
     "worklet";
     const resources = {
       surface: surfaceSharedValue.value,
+      spareSurface: spareSurfaceSharedValue.value,
       canvas: canvasSharedValue.value,
-      images: [currentFrame.value, pendingSnapshot.value],
+      // currentFrame is one of the buffers' images.
+      images: [...bufferImages.value, pendingSnapshot.value],
       frames: Object.values(previousSources.value),
       reservation: surfaceReservation.value,
       afterDraw: pendingAfterDraw.value?.run,
@@ -300,6 +318,10 @@ export const useVideoComposition = <T = undefined>({
     sourcesGeneration.value = 0;
     surfaceSharedValue.value = null;
     canvasSharedValue.value = null;
+    spareSurfaceSharedValue.value = null;
+    spareCanvasSharedValue.value = null;
+    bufferImages.value = [];
+    frontBuffer.value = -1;
     surfaceReservation.value = 0;
     surfaceWidth.value = 0;
     surfaceHeight.value = 0;
@@ -321,6 +343,10 @@ export const useVideoComposition = <T = undefined>({
   const drawDeferred = useSharedValue(false);
   // Whether the previous vsync brought new frames.
   const importedLastVsync = useSharedValue(false);
+  // Draws submitted without waiting for the GPU since the last wait, the
+  // checked wait that ends every import of a new frame included.
+  const uncheckedDraws = useSharedValue(0);
+  const importWaits = useSharedValue(0);
   // A new drawFrame can draw something else from the same time and frames.
   useEffect(() => {
     lastDrawnTime.value = -1;
@@ -386,6 +412,13 @@ export const useVideoComposition = <T = undefined>({
       sourcesGeneration.value = generation;
       const currentTime = framesExtractor.currentTime;
       const framesVersion = framesExtractor.framesVersion;
+      // An import (above, or of another player on this runtime) ends with a
+      // checked wait for all the GPU work submitted before it.
+      const waits = videoFrameImportWaits();
+      if (waits !== importWaits.value) {
+        importWaits.value = waits;
+        uncheckedDraws.value = 0;
+      }
       const imported =
         splitImportAndDraw && framesVersion !== lastImportedVersion.value;
       const importedBefore = importedLastVsync.value;
@@ -460,8 +493,9 @@ export const useVideoComposition = <T = undefined>({
         // picture by the ratio between them, so the surface is re-keyed on its
         // dimensions the way exportVideoComposition already re-keys its own.
         const retired = {
-          images: [currentFrame.value, pendingSnapshot.value],
+          images: [...bufferImages.value, pendingSnapshot.value],
           surface,
+          spareSurface: spareSurfaceSharedValue.value,
           canvas: canvasSharedValue.value,
           reservation: surfaceReservation.value,
           afterDraw: pendingAfterDraw.value?.run,
@@ -471,10 +505,14 @@ export const useVideoComposition = <T = undefined>({
         pendingAfterDraw.value = null;
         canvasSharedValue.value = null;
         surfaceSharedValue.value = null;
+        spareSurfaceSharedValue.value = null;
+        spareCanvasSharedValue.value = null;
+        bufferImages.value = [];
+        frontBuffer.value = -1;
         surfaceReservation.value = 0;
         const retiredFailure = disposePreviewResources(retired);
         if (retiredFailure.failed) throw retiredFailure.error;
-        // Surface + immutable snapshot/COW backing while replacing the preview.
+        // The two surfaces the preview draws into in turn.
         const reservation = reserveVideoMemory(
           frameByteSize(pixelWidth, pixelHeight) * 2,
           "composition preview",
@@ -485,6 +523,13 @@ export const useVideoComposition = <T = undefined>({
           surfaceSharedValue.value = surface;
           if (!surface) throw new Error("Cannot allocate preview surface");
           canvasSharedValue.value = getVideoCanvas(surface);
+          const spare = Skia.Surface.MakeOffscreen(pixelWidth, pixelHeight);
+          spareSurfaceSharedValue.value = spare;
+          if (!spare) throw new Error("Cannot allocate preview surface");
+          spareCanvasSharedValue.value = getVideoCanvas(spare);
+          const first = surface.asImage();
+          bufferImages.value = [first];
+          bufferImages.value = [first, spare.asImage()];
           surfaceWidth.value = pixelWidth;
           surfaceHeight.value = pixelHeight;
         } catch (error) {
@@ -497,8 +542,13 @@ export const useVideoComposition = <T = undefined>({
         console.warn("Failed to create surface");
         return;
       }
+      // Draw into the buffer that is not on screen.
+      const back = frontBuffer.value === 0 ? 1 : 0;
+      if (back === 1) surface = spareSurfaceSharedValue.value!;
+      const backSurface = surface;
 
-      const canvas = canvasSharedValue.value!;
+      const canvas =
+        back === 1 ? spareCanvasSharedValue.value! : canvasSharedValue.value!;
       // Cleared as the export clears it: a drawFrame that leaves part of the
       // canvas alone showed the previous frames there, and not in the export.
       canvas.drawColor(Skia.Color("#00000000"), BlendMode.Clear);
@@ -523,21 +573,24 @@ export const useVideoComposition = <T = undefined>({
           width: pixelWidth,
           height: pixelHeight,
         });
-        const previous = currentFrame.value;
-        const next = surface.makeImageSnapshot(
-          undefined,
-          previous ?? undefined,
-        );
-        if (next !== previous) pendingSnapshot.value = next;
-        surface.flush(true);
+        const next = bufferImages.value[back]!;
+        // Waiting for the GPU here held the UI thread for the work of every
+        // Skia view on the shared context: a vsync in three with 4K frames on
+        // iOS with Skia 3.9.1, every other one before. The draw samples only
+        // images Skia owns, which the GPU work keeps alive, so it waits only
+        // when afterDrawFrame releases a context of the caller's, and after a
+        // few draws without an import, so that the GPU queue stays bounded.
+        const wait =
+          afterDrawFrame !== undefined ||
+          uncheckedDraws.value >= maxUncheckedDraws;
+        backSurface.flush(wait);
+        uncheckedDraws.value = wait ? 0 : uncheckedDraws.value + 1;
         drawn.submitted = true;
         frameGeneration.value = generation;
-        if (next === previous) currentFrame.modify(undefined, true);
-        else {
-          currentFrame.value = next;
-          previous?.dispose();
-        }
-        pendingSnapshot.value = null;
+        // The image shares the surface's texture: it shows this draw once
+        // the flush above has submitted it.
+        currentFrame.value = next;
+        frontBuffer.value = back;
       } catch (error) {
         drawn.failed = true;
         drawn.error = error;

package/src/videoFrameImage.ts

@@ -41,7 +41,7 @@ export type VideoFrameFit = "fill" | "cover" | "contain";
  *
  * @param frame the frame, drawn in the tick it was handed out in
  * @param output an image returned for a previous frame, recycled rather than
- * allocating a new wrapper. Accepted for compatibility; owned snapshots are used in v3
+ * allocating a new wrapper. Accepted for compatibility; the image wraps a texture the library owns
  * @returns the image and its picture, or null when the frame has no texture
  */
 export const makeVideoFrameImage = (

package/src/yuvBlit.ts

@@ -68,40 +68,52 @@ struct VsOut { @builtin(position) position: vec4f, @location(0) uv: vec2f };
 // primaries to sRGB, and every export brightens the midtones again.
 const sourceColorSpace = { primaries: "srgb", transfer: "srgb" } as const;
 
-/** The corner of a decoder buffer's crop rectangle once the buffer is turned
- * clockwise for display: the blit takes its source origin in that turned
- * space. Throws unless the rectangle lies in the buffer and turns into the
- * frame's display size. */
-export const yuvFrameOrigin = (
+/** A decoder buffer's crop rectangle once the buffer is turned clockwise for
+ * display: the blit samples that upright region, and draws it at the size of
+ * its target, which a capped or explicit frame size makes smaller. Throws
+ * unless the rectangle lies in the buffer. */
+export const yuvFrameRegion = (
   region: { x: number; y: number; width: number; height: number },
   rotation: number,
   buffer: { width: number; height: number },
-  frame: { width: number; height: number },
-): { x: number; y: number } => {
+): { x: number; y: number; width: number; height: number } => {
   "worklet";
   const turned = rotation === 90 || rotation === 270;
   if (
     (rotation !== 0 && !turned && rotation !== 180) ||
     region.x < 0 ||
     region.y < 0 ||
+    region.width <= 0 ||
+    region.height <= 0 ||
     region.x + region.width > buffer.width ||
-    region.y + region.height > buffer.height ||
-    (turned ? region.height : region.width) !== frame.width ||
-    (turned ? region.width : region.height) !== frame.height
+    region.y + region.height > buffer.height
   ) {
     throw new Error("Native video buffer region or YUV layout mismatch");
   }
-  const { width, height } = buffer;
+  const width = turned ? region.height : region.width;
+  const height = turned ? region.width : region.height;
   if (rotation === 90)
-    return { x: height - region.y - region.height, y: region.x };
+    return {
+      x: buffer.height - region.y - region.height,
+      y: region.x,
+      width,
+      height,
+    };
   if (rotation === 180)
     return {
-      x: width - region.x - region.width,
-      y: height - region.y - region.height,
+      x: buffer.width - region.x - region.width,
+      y: buffer.height - region.y - region.height,
+      width,
+      height,
     };
   if (rotation === 270)
-    return { x: region.y, y: width - region.x - region.width };
-  return { x: region.x, y: region.y };
+    return {
+      x: region.y,
+      y: buffer.width - region.x - region.width,
+      width,
+      height,
+    };
+  return { x: region.x, y: region.y, width, height };
 };
 
 const getYuvBlit = (device: GPUDevice): YuvBlit => {
@@ -251,10 +263,11 @@ const importYuvBuffer = (
 };
 
 /**
- * Draws the upright `width`×`height` region at `origin` of a decoder buffer
- * into `target`, converting YUV to RGB; the buffer is imported on its first
- * frame. Submits the draw: the caller waits for it before releasing the
- * frame's lease or `source`, as after copyExternalImageToTexture.
+ * Draws the upright `region` of a decoder buffer (see yuvFrameRegion) into
+ * the whole of `target`, converting YUV to RGB and scaling it to the target's
+ * size; the buffer is imported on its first frame. Submits the draw: the
+ * caller waits for it before releasing the frame's lease or `source`, as
+ * after copyExternalImageToTexture.
  */
 export const blitYuvFrame = (
   device: GPUDevice,
@@ -262,9 +275,7 @@ export const blitYuvFrame = (
   producer: number,
   source: NativeVideoFrame,
   rotation: number,
-  origin: { x: number; y: number },
-  width: number,
-  height: number,
+  region: { x: number; y: number; width: number; height: number },
   target: GPUTexture,
 ): void => {
   "worklet";
@@ -280,7 +291,7 @@ export const blitYuvFrame = (
   );
   if (index < 0) {
     // The least recently drawn buffer goes first. Its last frame completed:
-    // every import ends with a checked flush before the next one starts.
+    // fewer imports than this are ever on the GPU (see frameInterop.ts).
     if (entries.length >= maxImportsPerProducer) {
       releaseYuvImport(entries[0]!);
       entries.shift();
@@ -297,10 +308,10 @@ export const blitYuvFrame = (
   const frameWidth = turned ? source.height : source.width;
   const frameHeight = turned ? source.width : source.height;
   const mapping = [
-    width / frameWidth,
-    height / frameHeight,
-    origin.x / frameWidth,
-    origin.y / frameHeight,
+    region.width / frameWidth,
+    region.height / frameHeight,
+    region.x / frameWidth,
+    region.y / frameHeight,
   ];
   if (mapping.some((value, i) => value !== entry.mapping[i])) {
     device.queue.writeBuffer(entry.uniforms, 0, new Float32Array(mapping));
@@ -326,8 +337,8 @@ export const blitYuvFrame = (
     pass.end();
     device.queue.submit([encoder.finish()]);
   } finally {
-    // The caller's checked flush covers the draw; the release fence is not
-    // needed to hand the buffer back.
+    // The caller hands the decoder buffer back once the GPU is done with the
+    // draw; the release fence is not needed for that.
     entry.wrapper.endAccess()?.fence.destroy();
   }
 };
@@ -355,7 +366,11 @@ export const blitNv12Frame = (
       size: 16,
       usage: videoUniformBufferUsage,
     });
-    device.queue.writeBuffer(blit.wholeFrame, 0, new Float32Array([1, 1, 0, 0]));
+    device.queue.writeBuffer(
+      blit.wholeFrame,
+      0,
+      new Float32Array([1, 1, 0, 0]),
+    );
   }
   const external = device.importExternalTexture({
     label: "video decoder buffer",