react-native-skia-webgpu-video 0.1.0-beta.3 → 0.1.0

package/src/exportVideoComposition.ts

@@ -11,7 +11,7 @@ import {
   type WorkletRuntime,
 } from "react-native-worklets";
 import { AlphaType, BlendMode, ColorType, Skia } from "react-native-skia";
-import type { ImageInfo, SkCanvas, SkSurface } from "react-native-skia";
+import type { SkCanvas, SkSurface } from "react-native-skia";
 import { Platform } from "react-native";
 import type {
   ExportOptions,
@@ -35,10 +35,16 @@ import {
 import { enqueueExport, exportAbortError } from "./exportQueue";
 import {
   acquireEncoderTarget,
+  acquireWindowTarget,
   closeEncoderTarget,
+  closeEncoderWindow,
+  presentWindowTarget,
   probeEncoderTarget,
+  probeEncoderWindow,
+  releaseEncoderWindow,
   supportsEncoderTargets,
   type EncoderTarget,
+  type EncoderWindow,
 } from "./exportTarget";
 import { yieldToRuntime } from "./runtimeYield";
 import {
@@ -82,7 +88,8 @@ export type ExportProfile = {
   targetMs: number;
   /** drawFrame, before/afterDrawFrame and progress scheduling. */
   drawMs: number;
-  /** The checked flush(true): waiting for the GPU to complete the frame. */
+  /** The checked flush(true): waiting for the GPU to complete the frame,
+   * which Android's window export does once it decoded the next frame. */
   gpuMs: number;
   /** Readback when used, submission to the encoder including its
    * backpressure, and the final file write. */
@@ -114,33 +121,24 @@ export type VideoExportArguments<T = undefined> = ExportOptions & {
   onProfile?: (profile: ExportProfile) => void;
 };
 
-// Native Skia 3.4.4 accepts destination/stride in readPixels; its
-// published Canvas type currently omits these arguments. Use only a private,
-// full-length, zero-offset Uint8Array (the native implementation ignores offsets).
-type ReadbackCanvas = {
-  readPixels: (
-    x: number,
-    y: number,
-    info: ImageInfo,
-    destination: Uint8Array,
-    bytesPerRow: number,
-  ) => Uint8Array | Float32Array | null;
-};
-
 type ExportState = {
   encoder: VideoEncoder | null;
   extractor: VideoCompositionFramesExtractorSync | null;
   surface: SkSurface | null;
   canvas: SkCanvas | null;
-  readback: Uint8Array | null;
   /** Draw into a lent encoder buffer instead of reading pixels back. */
   gpu: boolean;
   target: EncoderTarget | null;
+  /** Android GPU export: the encoder's input window, drawn into directly. */
+  window: EncoderWindow | null;
   /** iOS GPU export: decoded frames sampled in place, by producer. */
   imports: ExportFrameImports | null;
   frames: Record<string, VideoFrame>;
   processor: VideoFrameProcessor | null;
   pendingAfterDraw: (() => void) | null;
+  /** Android window: the last frame was submitted without its checked GPU
+   * wait, which comes before anything it sampled is released. */
+  unchecked?: boolean;
   reservations: number[];
   index: number;
   setupError?: unknown;
@@ -290,6 +288,16 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
         if (clock.last >= 0) profile[stage] += now - clock.last;
         clock.last = now;
       };
+      /** Android window: the checked GPU wait the last frame left for later,
+       * so that the GPU converted and drew it while the CPU presented it and
+       * decoded the next one. An empty recording: Graphite still waits for
+       * every command submitted so far. */
+      const checkWindowFrame = () => {
+        "worklet";
+        if (!state.unchecked) return;
+        state.window!.slots[state.window!.next]!.surface!.flush(true);
+        state.unchecked = false;
+      };
       const finish = async (
         error: unknown,
         aborted = false,
@@ -313,6 +321,7 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
         };
         // Submit any recording left by a drawing exception before an
         // adapter is allowed to destroy textures referenced by that canvas.
+        clean(checkWindowFrame);
         clean(() => state.surface?.flush(true));
         clean(() => state.target?.surface?.flush(true));
         if (cleanup.safeToRelease && state.pendingAfterDraw) {
@@ -329,6 +338,10 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
           }
         }
         if (clean(() => state.extractor?.dispose())) state.extractor = null;
+        // Android: the window's swapchain lets go of the codec's buffers
+        // before the encoder goes. Its own textures are freed further down.
+        const window = state.window;
+        if (window) clean(() => closeEncoderWindow(window));
         if (clean(() => state.encoder?.dispose())) state.encoder = null;
         if (cleanup.safeToRelease) {
           clean(() => state.surface?.flush(true));
@@ -341,10 +354,20 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
               state.imports = null;
             clean(() => clearVideoFrameImages(true));
           }
-          if (cleanup.safeToRelease && clean(() => state.canvas?.dispose()))
-            state.canvas = null;
-          if (cleanup.safeToRelease && clean(() => state.surface?.dispose()))
+          // Disposing the surface detaches its canvas.
+          if (cleanup.safeToRelease && clean(() => state.surface?.dispose())) {
             state.surface = null;
+            state.canvas = null;
+          }
+          // The target, if any, is one of the window's slots.
+          if (
+            cleanup.safeToRelease &&
+            window &&
+            clean(() => releaseEncoderWindow(window))
+          ) {
+            state.window = null;
+            state.target = null;
+          }
           const target = state.target;
           if (cleanup.safeToRelease && target) {
             if (clean(() => closeEncoderTarget(target))) state.target = null;
@@ -364,7 +387,6 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
         }
         if (cleanup.safeToRelease) {
           state.frames = {};
-          state.readback = null;
           delete storage.__rnskwgpuExport;
         } else {
           // One retained session at most; subsequent exports fail before
@@ -394,6 +416,8 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
           const owned = runWithVideoAutoreleasePool(() => {
             const decoded = state.extractor!.decodeCompositionFrames(time);
             lap("decodeMs");
+            checkWindowFrame();
+            lap("gpuMs");
             const imports = state.imports;
             // In place, a producer's frame is released by its next import.
             const imported = imports
@@ -424,14 +448,20 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
             await finish(null, true);
             return false;
           }
+          // CPU readback path only: the frame's pixels, still on their way.
+          const readback: {
+            pixels: ReturnType<SkCanvas["readPixels"]> | null;
+          } = { pixels: null };
           const draw = () => {
             "worklet";
             if (state.gpu)
-              state.target = acquireEncoderTarget(
-                state.encoder!,
-                options.width,
-                options.height,
-              );
+              state.target = state.window
+                ? acquireWindowTarget(state.window)
+                : acquireEncoderTarget(
+                    state.encoder!,
+                    options.width,
+                    options.height,
+                  );
             lap("targetMs");
             const target = state.target;
             const surface = target ? target.surface! : state.surface!;
@@ -454,61 +484,36 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
                 height: options.height,
               });
               lap("drawMs");
-              surface.flush(true);
+              // Android window, no context to release: the next frame waits
+              // for this one (see checkWindowFrame).
+              const unchecked =
+                !!state.window &&
+                !!target &&
+                !afterDrawFrame &&
+                !state.processor;
+              surface.flush(!unchecked);
+              state.unchecked = unchecked;
               drawn.drained = true;
               lap("gpuMs");
-              if (target) {
-                closeEncoderTarget(target, true);
+              if (target && state.window) {
+                state.target = null;
+                presentWindowTarget(state.encoder!, state.window, target, time);
+              } else if (target) {
+                closeEncoderTarget(target);
                 state.target = null;
                 state.encoder!.encodeFrame(
-                  {
-                    kind: "native-buffer",
-                    nativeBuffer: target.pointer,
-                    fences: target.fences,
-                  },
+                  { kind: "native-buffer", nativeBuffer: target.pointer },
                   time,
                 );
               } else {
-                const readbackCanvas = canvas as unknown as ReadbackCanvas;
-                const pixels = readbackCanvas.readPixels(
-                  0,
-                  0,
-                  {
-                    width: options.width,
-                    height: options.height,
-                    colorType: bgra ? ColorType.BGRA_8888 : ColorType.RGBA_8888,
-                    alphaType: AlphaType.Premul,
-                  },
-                  state.readback!,
-                  options.width * 4,
-                );
-                if (
-                  !(pixels instanceof Uint8Array) ||
-                  pixels.byteLength !== bytes ||
-                  pixels.byteOffset !== 0 ||
-                  pixels.buffer !== state.readback!.buffer
-                ) {
-                  throw new Error("Video export pixel readback failed");
-                }
-                // Fallback: consumed synchronously by the bounded native pool.
-                const storage =
-                  pixels.byteOffset === 0 &&
-                  pixels.buffer.byteLength === pixels.byteLength
-                    ? pixels.buffer
-                    : pixels.buffer.slice(
-                        pixels.byteOffset,
-                        pixels.byteOffset + pixels.byteLength,
-                      );
-                state.encoder!.encodeFrame(
-                  {
-                    kind: bgra ? "bgra" : "rgba",
-                    data: storage,
-                    width: options.width,
-                    height: options.height,
-                    bytesPerRow: options.width * 4,
-                  },
-                  time,
-                );
+                // Skia 3.9 reads back asynchronously, into a new array: the
+                // frame is encoded once it lands, after this scope.
+                readback.pixels = canvas.readPixels(0, 0, {
+                  width: options.width,
+                  height: options.height,
+                  colorType: bgra ? ColorType.BGRA_8888 : ColorType.RGBA_8888,
+                  alphaType: AlphaType.Premul,
+                });
               }
               lap("encodeMs");
             } catch (error) {
@@ -545,6 +550,35 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
             if (drawn.failed) throw drawn.failure;
           };
           runWithVideoAutoreleasePool(draw);
+          if (readback.pixels) {
+            const pixels = await readback.pixels;
+            lap("gpuMs");
+            // isView, not instanceof: true of a typed array of any realm.
+            if (!ArrayBuffer.isView(pixels) || pixels.byteLength !== bytes)
+              throw new Error("Video export pixel readback failed");
+            // Fallback: consumed synchronously by the bounded native pool.
+            const storage =
+              pixels.byteOffset === 0 &&
+              pixels.buffer.byteLength === pixels.byteLength
+                ? pixels.buffer
+                : pixels.buffer.slice(
+                    pixels.byteOffset,
+                    pixels.byteOffset + pixels.byteLength,
+                  );
+            runWithVideoAutoreleasePool(() =>
+              state.encoder!.encodeFrame(
+                {
+                  kind: bgra ? "bgra" : "rgba",
+                  data: storage,
+                  width: options.width,
+                  height: options.height,
+                  bytesPerRow: options.width * 4,
+                },
+                time,
+              ),
+            );
+            lap("encodeMs");
+          }
           state.index++;
           if (onProgress)
             scheduleOnRN(onProgress, {
@@ -612,9 +646,9 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
           extractor: null,
           surface: null,
           canvas: null,
-          readback: null,
           gpu: false,
           target: null,
+          window: null,
           imports: null,
           frames: {},
           processor: null,
@@ -641,10 +675,9 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
                 state.reservations.push(
                   reserveVideoMemory(
                     bytes * 2,
-                    "export reusable readback and transient raster",
+                    "export readback and transient raster",
                   ),
                 );
-                state.readback = new Uint8Array(bytes);
                 state.hasRecorder = true;
                 state.surface = Skia.Surface.MakeOffscreen(
                   options.width,
@@ -666,16 +699,28 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
               );
               state.encoder.prepare();
               if (gpuExport) {
-                // Skia draws straight into the encoder's buffers; the probe
-                // keeps devices without IOSurface interop on the readback path.
+                // Skia draws straight into the encoder's buffers: lent pool
+                // buffers on iOS, the input window on Android. The probe keeps
+                // devices where that fails on the readback path.
                 state.hasRecorder = true;
-                const failure = supportsEncoderTargets(state.encoder)
-                  ? probeEncoderTarget(
-                      state.encoder,
-                      options.width,
-                      options.height,
-                    )
-                  : new Error("Encoder cannot lend frame buffers");
+                let failure: unknown;
+                if (bgra) {
+                  failure = supportsEncoderTargets(state.encoder)
+                    ? probeEncoderTarget(
+                        state.encoder,
+                        options.width,
+                        options.height,
+                      )
+                    : new Error("Encoder cannot lend frame buffers");
+                } else {
+                  const probe = probeEncoderWindow(
+                    state.encoder,
+                    options.width,
+                    options.height,
+                  );
+                  state.window = probe.window;
+                  failure = probe.failure;
+                }
                 state.gpu = failure === null;
                 // Decoded frames are sampled in place too: on iOS through the
                 // same IOSurface import, on Android once converted from YUV.