react-native-skia-webgpu-video 0.1.0-beta.4 → 0.1.0

package/src/videoCompositionPlayer.ts

@@ -19,10 +19,18 @@ import RNSkiaVideoModule from "./RNSkiaVideoModule";
 import useEventListener from "./utils/useEventListener";
 import { PixelRatio } from "react-native";
 import { scheduleOnRN } from "react-native-worklets";
-import { ownVideoFrames, releaseVideoFrameImage } from "./frameInterop";
+import {
+  ownVideoFrames,
+  releaseVideoFrameImage,
+  videoFrameImportWaits,
+} from "./frameInterop";
 import { frameByteSize, reserveVideoMemory } from "./memory";
 import { disposePreviewResources } from "./previewLifecycle";
 
+// Draws in a row the preview submits without waiting for the GPU: one frame
+// of a 24 fps video spans five vsyncs at 120 Hz, redrawn as time goes by.
+const maxUncheckedDraws = 6;
+
 type UseVideoCompositionPlayerOptions<T = undefined> = {
   /**
    * The video composition to play.
@@ -87,11 +95,10 @@ type UseVideoCompositionPlayerOptions<T = undefined> = {
   /**
    * Import a newly decoded frame in one vsync and draw it in the next, instead
    * of both in the same one. On Android the import (YUV blit and its checked
-   * GPU wait) and the draw (snapshot and its checked GPU wait) cost about as
-   * much as each other, and together they overrun a 90–120 Hz frame budget on
-   * the UI thread. The option applies on iOS too. The picture then trails the
-   * clock by one vsync. The first frame of a session is still drawn as soon
-   * as it is imported.
+   * GPU wait) and the draw cost about as much as each other, and together they
+   * overrun a 90–120 Hz frame budget on the UI thread. The option applies on
+   * iOS too. The picture then trails the clock by one vsync. The first frame
+   * of a session is still drawn as soon as it is imported.
    * @default false
    */
   splitImportAndDraw?: boolean;
@@ -276,6 +283,15 @@ export const useVideoComposition = <T = undefined>({
 
   const surfaceSharedValue = useSharedValue<SkSurface | null>(null);
   const canvasSharedValue = useSharedValue<SkCanvas | null>(null);
+  // The preview draws into two surfaces in turn, and shows each through an
+  // image sharing its texture (asImage): no copy per draw, as a snapshot
+  // made. The picture on screen stays whole while the next is drawn into the
+  // other surface.
+  const spareSurfaceSharedValue = useSharedValue<SkSurface | null>(null);
+  const spareCanvasSharedValue = useSharedValue<SkCanvas | null>(null);
+  const bufferImages = useSharedValue<Array<SkImage | null>>([]);
+  // The buffer on screen: 0, 1 (the spare), or -1 before the first draw.
+  const frontBuffer = useSharedValue(-1);
   const surfaceReservation = useSharedValue(0);
   const surfaceWidth = useSharedValue(0);
   const surfaceHeight = useSharedValue(0);
@@ -283,8 +299,10 @@ export const useVideoComposition = <T = undefined>({
     "worklet";
     const resources = {
       surface: surfaceSharedValue.value,
+      spareSurface: spareSurfaceSharedValue.value,
       canvas: canvasSharedValue.value,
-      images: [currentFrame.value, pendingSnapshot.value],
+      // currentFrame is one of the buffers' images.
+      images: [...bufferImages.value, pendingSnapshot.value],
       frames: Object.values(previousSources.value),
       reservation: surfaceReservation.value,
       afterDraw: pendingAfterDraw.value?.run,
@@ -300,6 +318,10 @@ export const useVideoComposition = <T = undefined>({
     sourcesGeneration.value = 0;
     surfaceSharedValue.value = null;
     canvasSharedValue.value = null;
+    spareSurfaceSharedValue.value = null;
+    spareCanvasSharedValue.value = null;
+    bufferImages.value = [];
+    frontBuffer.value = -1;
     surfaceReservation.value = 0;
     surfaceWidth.value = 0;
     surfaceHeight.value = 0;
@@ -321,6 +343,10 @@ export const useVideoComposition = <T = undefined>({
   const drawDeferred = useSharedValue(false);
   // Whether the previous vsync brought new frames.
   const importedLastVsync = useSharedValue(false);
+  // Draws submitted without waiting for the GPU since the last wait, the
+  // checked wait that ends every import of a new frame included.
+  const uncheckedDraws = useSharedValue(0);
+  const importWaits = useSharedValue(0);
   // A new drawFrame can draw something else from the same time and frames.
   useEffect(() => {
     lastDrawnTime.value = -1;
@@ -386,6 +412,13 @@ export const useVideoComposition = <T = undefined>({
       sourcesGeneration.value = generation;
       const currentTime = framesExtractor.currentTime;
       const framesVersion = framesExtractor.framesVersion;
+      // An import (above, or of another player on this runtime) ends with a
+      // checked wait for all the GPU work submitted before it.
+      const waits = videoFrameImportWaits();
+      if (waits !== importWaits.value) {
+        importWaits.value = waits;
+        uncheckedDraws.value = 0;
+      }
       const imported =
         splitImportAndDraw && framesVersion !== lastImportedVersion.value;
       const importedBefore = importedLastVsync.value;
@@ -460,8 +493,9 @@ export const useVideoComposition = <T = undefined>({
         // picture by the ratio between them, so the surface is re-keyed on its
         // dimensions the way exportVideoComposition already re-keys its own.
         const retired = {
-          images: [currentFrame.value, pendingSnapshot.value],
+          images: [...bufferImages.value, pendingSnapshot.value],
           surface,
+          spareSurface: spareSurfaceSharedValue.value,
           canvas: canvasSharedValue.value,
           reservation: surfaceReservation.value,
           afterDraw: pendingAfterDraw.value?.run,
@@ -471,10 +505,14 @@ export const useVideoComposition = <T = undefined>({
         pendingAfterDraw.value = null;
         canvasSharedValue.value = null;
         surfaceSharedValue.value = null;
+        spareSurfaceSharedValue.value = null;
+        spareCanvasSharedValue.value = null;
+        bufferImages.value = [];
+        frontBuffer.value = -1;
         surfaceReservation.value = 0;
         const retiredFailure = disposePreviewResources(retired);
         if (retiredFailure.failed) throw retiredFailure.error;
-        // Surface + immutable snapshot/COW backing while replacing the preview.
+        // The two surfaces the preview draws into in turn.
         const reservation = reserveVideoMemory(
           frameByteSize(pixelWidth, pixelHeight) * 2,
           "composition preview",
@@ -485,6 +523,13 @@ export const useVideoComposition = <T = undefined>({
           surfaceSharedValue.value = surface;
           if (!surface) throw new Error("Cannot allocate preview surface");
           canvasSharedValue.value = getVideoCanvas(surface);
+          const spare = Skia.Surface.MakeOffscreen(pixelWidth, pixelHeight);
+          spareSurfaceSharedValue.value = spare;
+          if (!spare) throw new Error("Cannot allocate preview surface");
+          spareCanvasSharedValue.value = getVideoCanvas(spare);
+          const first = surface.asImage();
+          bufferImages.value = [first];
+          bufferImages.value = [first, spare.asImage()];
           surfaceWidth.value = pixelWidth;
           surfaceHeight.value = pixelHeight;
         } catch (error) {
@@ -497,8 +542,13 @@ export const useVideoComposition = <T = undefined>({
         console.warn("Failed to create surface");
         return;
       }
+      // Draw into the buffer that is not on screen.
+      const back = frontBuffer.value === 0 ? 1 : 0;
+      if (back === 1) surface = spareSurfaceSharedValue.value!;
+      const backSurface = surface;
 
-      const canvas = canvasSharedValue.value!;
+      const canvas =
+        back === 1 ? spareCanvasSharedValue.value! : canvasSharedValue.value!;
       // Cleared as the export clears it: a drawFrame that leaves part of the
       // canvas alone showed the previous frames there, and not in the export.
       canvas.drawColor(Skia.Color("#00000000"), BlendMode.Clear);
@@ -523,21 +573,24 @@ export const useVideoComposition = <T = undefined>({
           width: pixelWidth,
           height: pixelHeight,
         });
-        const previous = currentFrame.value;
-        const next = surface.makeImageSnapshot(
-          undefined,
-          previous ?? undefined,
-        );
-        if (next !== previous) pendingSnapshot.value = next;
-        surface.flush(true);
+        const next = bufferImages.value[back]!;
+        // Waiting for the GPU here held the UI thread for the work of every
+        // Skia view on the shared context: a vsync in three with 4K frames on
+        // iOS with Skia 3.9.1, every other one before. The draw samples only
+        // images Skia owns, which the GPU work keeps alive, so it waits only
+        // when afterDrawFrame releases a context of the caller's, and after a
+        // few draws without an import, so that the GPU queue stays bounded.
+        const wait =
+          afterDrawFrame !== undefined ||
+          uncheckedDraws.value >= maxUncheckedDraws;
+        backSurface.flush(wait);
+        uncheckedDraws.value = wait ? 0 : uncheckedDraws.value + 1;
         drawn.submitted = true;
         frameGeneration.value = generation;
-        if (next === previous) currentFrame.modify(undefined, true);
-        else {
-          currentFrame.value = next;
-          previous?.dispose();
-        }
-        pendingSnapshot.value = null;
+        // The image shares the surface's texture: it shows this draw once
+        // the flush above has submitted it.
+        currentFrame.value = next;
+        frontBuffer.value = back;
       } catch (error) {
         drawn.failed = true;
         drawn.error = error;

package/src/videoFrameImage.ts

@@ -41,7 +41,7 @@ export type VideoFrameFit = "fill" | "cover" | "contain";
  *
  * @param frame the frame, drawn in the tick it was handed out in
  * @param output an image returned for a previous frame, recycled rather than
- * allocating a new wrapper. Accepted for compatibility; owned snapshots are used in v3
+ * allocating a new wrapper. Accepted for compatibility; the image wraps a texture the library owns
  * @returns the image and its picture, or null when the frame has no texture
  */
 export const makeVideoFrameImage = (

package/src/yuvBlit.ts

@@ -291,7 +291,7 @@ export const blitYuvFrame = (
   );
   if (index < 0) {
     // The least recently drawn buffer goes first. Its last frame completed:
-    // every import ends with a checked flush before the next one starts.
+    // fewer imports than this are ever on the GPU (see frameInterop.ts).
     if (entries.length >= maxImportsPerProducer) {
       releaseYuvImport(entries[0]!);
       entries.shift();
@@ -337,8 +337,8 @@ export const blitYuvFrame = (
     pass.end();
     device.queue.submit([encoder.finish()]);
   } finally {
-    // The caller's checked flush covers the draw; the release fence is not
-    // needed to hand the buffer back.
+    // The caller hands the decoder buffer back once the GPU is done with the
+    // draw; the release fence is not needed for that.
     entry.wrapper.endAccess()?.fence.destroy();
   }
 };