react-native-skia-webgpu-video 0.1.0-beta.2 → 0.1.0-beta.4

package/src/canvas.ts

@@ -12,11 +12,7 @@ export const getVideoCanvas = (surface: SkSurface): SkCanvas => {
     );
   }
   checked.enableCheckedSubmissions();
-  const canvas = surface.getCanvas();
-  if (typeof canvas.dispose !== "function") {
-    throw new Error(
-      "Skia canvas lifecycle patch is missing from the native binary. Apply the patch and rebuild the native app.",
-    );
-  }
-  return canvas;
+  // Owned by the surface: its dispose() detaches the canvas, and drawing on
+  // it throws from then on.
+  return surface.getCanvas();
 };

package/src/exportFrames.ts

@@ -18,13 +18,18 @@ import {
   reserveVideoMemory,
 } from "./memory";
 import type { VideoFrame } from "./types";
-import { blitYuvFrame, releaseYuvImports, yuvFrameOrigin } from "./yuvBlit";
+import {
+  blitNv12Frame,
+  blitYuvFrame,
+  releaseYuvImports,
+  yuvFrameRegion,
+} from "./yuvBlit";
 import type { YuvImport } from "./yuvBlit";
 
-/** A decoded frame sampled in place, without an owned snapshot: on iOS its
- * IOSurface, on Android its producer's RGBA texture, which the decoder
- * buffer was just converted into. The decoder lease stays open while the
- * image is in use. */
+/** A decoded frame sampled in place, without an owned snapshot: a BGRA
+ * frame's IOSurface on iOS, or for a YUV frame (Android, iOS NV12) its
+ * producer's RGBA texture, which the decoder buffer was just converted into.
+ * The decoder lease stays open while the image is in use. */
 export type ExportFrameImport = {
   id: number;
   frame: VideoFrame;
@@ -37,7 +42,7 @@ export type ExportFrameImport = {
   aliases?: WeakSet<VideoFrame>;
 };
 
-/** Android: the texture a producer's decoded frames are converted into. */
+/** The texture a producer's YUV frames are converted into. */
 type ProducerTexture = { texture: GPUTexture; reservation: number };
 
 /** Per export session, keyed by producer. `enabled` turns off after a first
@@ -48,7 +53,7 @@ export type ExportFrameImports = {
   producers: Set<number>;
   enabled: boolean;
   imported: boolean;
-  /** Android: one RGBA texture per producer, rewritten by each frame. */
+  /** One RGBA texture per producer of YUV frames, rewritten by each frame. */
   textures: Map<number, ProducerTexture>;
   /** Android: decoder buffers imported once (see yuvBlit.ts). */
   yuv: Map<number, YuvImport[]>;
@@ -65,6 +70,8 @@ type NativeBufferTexture = {
   nativeBuffer: bigint;
   region?: { x: number; y: number; width: number; height: number };
   rotation?: number;
+  /** iOS: the decoder's NV12 buffer (`framePixelFormat: 'yuv'`), or BGRA. */
+  format?: "nv12" | "bgra8";
 };
 
 // Captured on the RN runtime. Android samples its ImageReader YUV frames
@@ -117,8 +124,8 @@ export const releaseExportFrame = (entry: ExportFrameImport): void => {
   entry.frame.dispose?.();
 };
 
-/** Android: releases a producer's texture and decoder buffer imports. Call
- * only once its last frame's GPU work has completed. */
+/** Releases a producer's texture and decoder buffer imports. Call only once
+ * its last frame's GPU work has completed. */
 const releaseProducerTexture = (
   imports: ExportFrameImports,
   producer: number,
@@ -133,10 +140,11 @@ const releaseProducerTexture = (
   }
 };
 
-/** Android: converts a decoded ImageReader frame into its producer's RGBA
- * texture, wrapped as a Skia image. The conversion is submitted ahead of the
- * frame's drawing, on the same queue; the export's checked flush waits for
- * both before the lease or the texture is reused. */
+/** Converts a decoded YUV frame (an Android ImageReader buffer, or an iOS
+ * NV12 one) into its producer's RGBA texture, wrapped as a Skia image. The
+ * conversion is submitted ahead of the frame's drawing, on the same queue;
+ * the export's checked flush waits for both before the lease or the texture
+ * is reused. */
 const importYuvExportFrame = (
   imports: ExportFrameImports,
   frame: VideoFrame,
@@ -158,12 +166,17 @@ const importYuvExportFrame = (
   try {
     entry.nativeFrame = createNativeVideoGpuFrame(source.nativeBuffer);
     const rotation = source.rotation ?? 0;
-    const origin = yuvFrameOrigin(
-      source.region!,
-      rotation,
-      entry.nativeFrame,
-      frame,
-    );
+    // iOS NV12: the whole buffer, in decoded orientation.
+    const nv12 = source.format === "nv12";
+    if (
+      nv12 &&
+      (entry.nativeFrame.width !== frame.width ||
+        entry.nativeFrame.height !== frame.height)
+    )
+      throw new Error("Native video buffer dimensions mismatch");
+    const region = nv12
+      ? null
+      : yuvFrameRegion(source.region!, rotation, entry.nativeFrame);
     const device = getVideoGpuDevice();
     target = imports.textures.get(producer);
     if (
@@ -199,17 +212,17 @@ const importYuvExportFrame = (
     }
     // Can throw after submitting: drain below before giving the lease back.
     submitted = true;
-    blitYuvFrame(
-      device,
-      imports.yuv,
-      producer,
-      entry.nativeFrame,
-      rotation,
-      origin,
-      frame.width,
-      frame.height,
-      target.texture,
-    );
+    if (nv12) blitNv12Frame(device, entry.nativeFrame, target.texture);
+    else
+      blitYuvFrame(
+        device,
+        imports.yuv,
+        producer,
+        entry.nativeFrame,
+        rotation,
+        region!,
+        target.texture,
+      );
     entry.image = Skia.Image.MakeImageFromGPUTexture(target.texture);
     if (!entry.image) throw new Error("Cannot wrap the decoded frame");
     return entry;
@@ -233,15 +246,18 @@ const importYuvExportFrame = (
 };
 
 /** Wraps a decoded BGRA CVPixelBuffer's IOSurface as a Skia image, read-only.
- * The decoder already finished writing it; no producer fence is needed. On
- * Android, converts the decoded YUV frame instead (see importYuvExportFrame). */
+ * The decoder already finished writing it; no producer fence is needed. A
+ * YUV frame (Android, iOS NV12) is converted instead (see importYuvExportFrame). */
 export const importExportFrame = (
   frame: VideoFrame,
   imports?: ExportFrameImports,
 ): ExportFrameImport => {
   "worklet";
-  if (convertsYuv) {
-    if (!imports) throw new Error("Android in-place frames need their imports");
+  if (
+    convertsYuv ||
+    (frame.texture as NativeBufferTexture).format === "nv12"
+  ) {
+    if (!imports) throw new Error("YUV in-place frames need their imports");
     return importYuvExportFrame(imports, frame);
   }
   const entry: ExportFrameImport = {
@@ -311,8 +327,9 @@ export const releaseExportFrames = (imports: ExportFrameImports): void => {
 
 /** The export counterpart of ownVideoFrames on iOS: decoded frames are drawn
  * in place, and a producer's previous frame is released once its next one is
- * imported, since every exported frame ends with a checked GPU wait. Other
- * frames, or every frame after a first failed import, use `fallback`. */
+ * imported, since the export waits for the GPU to complete a frame before it
+ * imports the next one. Other frames, or every frame after a first failed
+ * import, use `fallback`. */
 export const ownExportFrames = (
   imports: ExportFrameImports,
   decoded: Record<string, VideoFrame>,

package/src/exportTarget.ts

@@ -1,21 +1,33 @@
 /// <reference types="@webgpu/types" />
-import { Platform } from "react-native";
 import { BlendMode, Skia } from "react-native-skia";
 import type { SkCanvas, SkSurface } from "react-native-skia";
 import type {
   GPUSharedTextureMemory,
   GPUSharedTextureMemoryDescriptor,
   NativeVideoFrame,
+  RNCanvasContext,
 } from "react-native-webgpu";
 import { getVideoCanvas } from "./canvas";
-import { createNativeVideoGpuFrame, getVideoGpuDevice } from "./gpuDevice";
+import {
+  createNativeVideoGpuFrame,
+  createNativeWindowGpuContext,
+  encoderWindowTextureUsage,
+  getVideoGpuDevice,
+  nativeVideoTextureUsage,
+} from "./gpuDevice";
+import {
+  frameByteSize,
+  releaseVideoMemory,
+  reserveVideoMemory,
+} from "./memory";
 import type { VideoEncoder } from "./types";
 
-/** An encoder pool buffer that Skia draws into through WebGPU. */
+/** A texture Skia draws an exported frame into through WebGPU: a lent
+ * encoder pool buffer on iOS, one of the encoder window's own textures on
+ * Android. */
 export type EncoderTarget = {
-  pointer: bigint;
-  /** Android: sync-fds ending the GPU access, handed over to the encoder. */
-  fences: bigint[];
+  /** iOS: the lent CVPixelBuffer. */
+  pointer: bigint | null;
   frame: NativeVideoFrame | null;
   memory: GPUSharedTextureMemory | null;
   texture: GPUTexture | null;
@@ -24,9 +36,25 @@ export type EncoderTarget = {
   canvas: SkCanvas | null;
 };
 
-// Captured on the RN runtime. iOS lends CVPixelBuffers, imported through
-// their IOSurface; Android lends the AHardwareBuffer itself.
-const lendsPixelBuffers = Platform.OS === "ios";
+/** Android: a canvas context presenting to the codec's input Surface. Skia
+ * draws each frame into one of two textures of our own, which a GPU copy then
+ * puts into the window's texture. The window's textures are the codec's
+ * input buffers: the copy waits on the GPU for the codec to hand one back,
+ * while the export goes on with the next frame, rather than the frame's
+ * checked wait. */
+export type EncoderWindow = {
+  context: RNCanvasContext;
+  device: GPUDevice;
+  configured: boolean;
+  /** Drawn into in turn, each copied into the window after its frame. */
+  slots: EncoderTarget[];
+  next: number;
+  reservations: number[];
+  /** The window texture the probe acquired, the first frame's copy target. */
+  pending: GPUTexture | null;
+};
+
+const windowSlots = 2;
 
 type SharedMemoryDevice = GPUDevice & {
   importSharedTextureMemory?: (
@@ -43,38 +71,19 @@ export const supportsEncoderTargets = (encoder: VideoEncoder): boolean => {
 };
 
 // Declared before its callers: worklets capture their closure on creation.
-/** Ends GPU access; the lent buffer stays with the caller, to encode or
- * release. Call only after a checked flush(true) of its surface. With
- * `encoding`, Android keeps the access's sync-fds in `target.fences` for the
- * encoder, which owns and closes them: ending an AHB access submits a
- * release that flush(true) has not waited for. */
-export const closeEncoderTarget = (
-  target: EncoderTarget,
-  encoding = false,
-): void => {
+/** iOS: ends GPU access; the lent buffer stays with the caller, to encode or
+ * release. Call only after a checked flush(true) of its surface. */
+export const closeEncoderTarget = (target: EncoderTarget): void => {
   "worklet";
-  target.canvas?.dispose();
+  // Disposing the surface detaches its canvas.
   target.canvas = null;
   target.surface?.dispose();
   target.surface = null;
   if (target.accessing && target.memory && target.texture) {
     const ended = target.memory.endAccess(target.texture);
     target.accessing = false;
-    for (const state of ended?.fences ?? []) {
-      try {
-        if (encoding && !lendsPixelBuffers) {
-          const exported = state.fence.export();
-          // Unreachable when negotiated: AHB interop requires the device's
-          // only shared-fence feature to be sync-fd (see gpuDevice.ts).
-          if (exported.type !== "sync-fd")
-            throw new Error("GPU export needs sync-fd fences");
-          target.fences.push(exported.handle);
-        }
-      } finally {
-        // iOS: already waited for. The wrapper is released either way.
-        state.fence.destroy();
-      }
-    }
+    // Already waited for. The wrappers are released either way.
+    for (const state of ended?.fences ?? []) state.fence.destroy();
   }
   target.accessing = false;
   target.texture?.destroy();
@@ -88,7 +97,7 @@ export const closeEncoderTarget = (
   target.frame = null;
 };
 
-/** Lends the next pool buffer and wraps it as a Skia surface. */
+/** iOS: lends the next pool buffer and wraps it as a Skia surface. */
 export const acquireEncoderTarget = (
   encoder: VideoEncoder,
   width: number,
@@ -97,7 +106,6 @@ export const acquireEncoderTarget = (
   "worklet";
   const target: EncoderTarget = {
     pointer: encoder.acquireFrameBuffer!(),
-    fences: [],
     frame: null,
     memory: null,
     texture: null,
@@ -106,33 +114,23 @@ export const acquireEncoderTarget = (
     canvas: null,
   };
   try {
-    if (lendsPixelBuffers) {
-      target.frame = createNativeVideoGpuFrame(target.pointer);
-      if (
-        target.frame.width !== width ||
-        target.frame.height !== height ||
-        target.frame.pixelFormat !== "bgra8"
-      ) {
-        throw new Error("Encoder buffer dimensions or BGRA layout mismatch");
-      }
+    target.frame = createNativeVideoGpuFrame(target.pointer!);
+    if (
+      target.frame.width !== width ||
+      target.frame.height !== height ||
+      target.frame.pixelFormat !== "bgra8"
+    ) {
+      throw new Error("Encoder buffer dimensions or BGRA layout mismatch");
     }
     const device = getVideoGpuDevice() as SharedMemoryDevice;
     if (typeof device.importSharedTextureMemory !== "function") {
       throw new Error("WebGPU shared texture memory is unavailable");
     }
     target.memory = device.importSharedTextureMemory({
-      handle: target.frame ? target.frame.handle : target.pointer,
+      handle: target.frame.handle,
       label: "video export frame",
     });
     target.texture = target.memory.createTexture();
-    if (
-      !lendsPixelBuffers &&
-      (target.texture.width !== width ||
-        target.texture.height !== height ||
-        target.texture.format !== "rgba8unorm")
-    ) {
-      throw new Error("Encoder buffer dimensions or RGBA layout mismatch");
-    }
     // Uninitialized: every frame starts with a full clear.
     target.memory.beginAccess(target.texture, false);
     target.accessing = true;
@@ -151,8 +149,9 @@ export const acquireEncoderTarget = (
   }
 };
 
-/** Clears one lent buffer on the GPU, then returns it unencoded. Returns the
- * failure, or null when this device can export through encoder targets. */
+/** iOS: clears one lent buffer on the GPU, then returns it unencoded.
+ * Returns the failure, or null when this device can export through encoder
+ * targets. */
 export const probeEncoderTarget = (
   encoder: VideoEncoder,
   width: number,
@@ -177,3 +176,165 @@ export const probeEncoderTarget = (
   encoder.releaseFrameBuffer!();
   return failure;
 };
+
+/** Android: the window's next texture, checked against the export size. */
+const windowTexture = (
+  window: EncoderWindow,
+  width: number,
+  height: number,
+): GPUTexture => {
+  "worklet";
+  const texture = window.pending ?? window.context.getCurrentTexture();
+  window.pending = null;
+  if (
+    texture.width !== width ||
+    texture.height !== height ||
+    texture.format !== "rgba8unorm"
+  ) {
+    throw new Error("Encoder window dimensions or RGBA layout mismatch");
+  }
+  return texture;
+};
+
+/** Android: the slot the next frame draws into. Its previous copy completed:
+ * the checked wait the export makes before each frame's import covers the
+ * queue before it. */
+export const acquireWindowTarget = (window: EncoderWindow): EncoderTarget => {
+  "worklet";
+  return window.slots[window.next]!;
+};
+
+/** Android: copies the slot Skia drew into to the window's next texture and
+ * hands it to the encoder, with its time. Call once the slot's surface was
+ * flushed: the copy follows the drawing on the same queue. */
+export const presentWindowTarget = (
+  encoder: VideoEncoder,
+  window: EncoderWindow,
+  target: EncoderTarget,
+  time: number,
+): void => {
+  "worklet";
+  const source = target.texture!;
+  const destination = windowTexture(window, source.width, source.height);
+  const commands = window.device.createCommandEncoder({
+    label: "video export window copy",
+  });
+  commands.copyTextureToTexture(
+    { texture: source },
+    { texture: destination },
+    { width: source.width, height: source.height },
+  );
+  window.device.queue.submit([commands.finish()]);
+  window.next = (window.next + 1) % window.slots.length;
+  // The encoder sets the frame's time on the window, then presents.
+  encoder.encodeFrame(
+    { kind: "window", present: () => window.context.present() },
+    time,
+  );
+};
+
+/** Android: the swapchain gives the codec its buffers back at once, rather
+ * than once the context is collected. Call before disposing the encoder. */
+export const closeEncoderWindow = (window: EncoderWindow): void => {
+  "worklet";
+  window.pending = null;
+  if (!window.configured) return;
+  window.context.unconfigure();
+  window.configured = false;
+};
+
+/** Android: frees the window's own textures. Call once their last recording
+ * was submitted: a copy still in flight keeps its source alive. */
+export const releaseEncoderWindow = (window: EncoderWindow): void => {
+  "worklet";
+  while (window.slots.length) {
+    const slot = window.slots[window.slots.length - 1]!;
+    // Disposing the surface detaches its canvas.
+    slot.canvas = null;
+    slot.surface?.dispose();
+    slot.surface = null;
+    slot.texture?.destroy();
+    slot.texture = null;
+    window.slots.pop();
+  }
+  while (window.reservations.length) {
+    releaseVideoMemory(window.reservations[window.reservations.length - 1]!);
+    window.reservations.pop();
+  }
+};
+
+/** Android: opens a canvas context on the encoder's input window with its
+ * slots, acquires the window's first texture and draws a clear into the
+ * first slot: nothing reaches the encoder. Returns the window, or the
+ * failure, after which the window is closed and the export can fall back to
+ * readback. */
+export const probeEncoderWindow = (
+  encoder: VideoEncoder,
+  width: number,
+  height: number,
+): { window: EncoderWindow | null; failure: unknown } => {
+  "worklet";
+  let window: EncoderWindow | null = null;
+  let failure: unknown = null;
+  try {
+    if (typeof encoder.acquireInputWindow !== "function")
+      throw new Error("Encoder cannot present through its input window");
+    const device = getVideoGpuDevice();
+    window = {
+      context: createNativeWindowGpuContext(
+        encoder.acquireInputWindow(),
+        width,
+        height,
+      ),
+      device,
+      configured: false,
+      slots: [],
+      next: 0,
+      reservations: [],
+      pending: null,
+    };
+    // Presented in FIFO order, opaque: every frame reaches the codec.
+    window.context.configure({
+      device,
+      format: "rgba8unorm",
+      usage: encoderWindowTextureUsage,
+    });
+    window.configured = true;
+    for (let i = 0; i < windowSlots; i++) {
+      window.reservations.push(
+        reserveVideoMemory(frameByteSize(width, height), "export window slot"),
+      );
+      const slot: EncoderTarget = {
+        pointer: null,
+        frame: null,
+        memory: null,
+        texture: device.createTexture({
+          label: "video export window slot",
+          size: { width, height },
+          format: "rgba8unorm",
+          usage: nativeVideoTextureUsage,
+        }),
+        accessing: false,
+        surface: null,
+        canvas: null,
+      };
+      window.slots.push(slot);
+      slot.surface = Skia.Surface.MakeFromGPUTexture(slot.texture!);
+      if (!slot.surface) throw new Error("Cannot wrap the export frame");
+      slot.canvas = getVideoCanvas(slot.surface);
+    }
+    window.pending = windowTexture(window, width, height);
+    const target = acquireWindowTarget(window);
+    target.canvas!.drawColor(Skia.Color("#00000000"), BlendMode.Clear);
+    target.surface!.flush(true);
+  } catch (error) {
+    failure = error ?? new Error("GPU export probe failed");
+  }
+  if (failure === null) return { window, failure };
+  if (window) {
+    // Not caught: a window still drawn by WebGPU cannot fall back to EGL.
+    closeEncoderWindow(window);
+    releaseEncoderWindow(window);
+  }
+  return { window: null, failure };
+};