react-native-skia-webgpu-video 0.1.0-beta.3 → 0.1.0

package/src/exportTarget.ts

@@ -1,21 +1,33 @@
 /// <reference types="@webgpu/types" />
-import { Platform } from "react-native";
 import { BlendMode, Skia } from "react-native-skia";
 import type { SkCanvas, SkSurface } from "react-native-skia";
 import type {
   GPUSharedTextureMemory,
   GPUSharedTextureMemoryDescriptor,
   NativeVideoFrame,
+  RNCanvasContext,
 } from "react-native-webgpu";
 import { getVideoCanvas } from "./canvas";
-import { createNativeVideoGpuFrame, getVideoGpuDevice } from "./gpuDevice";
+import {
+  createNativeVideoGpuFrame,
+  createNativeWindowGpuContext,
+  encoderWindowTextureUsage,
+  getVideoGpuDevice,
+  nativeVideoTextureUsage,
+} from "./gpuDevice";
+import {
+  frameByteSize,
+  releaseVideoMemory,
+  reserveVideoMemory,
+} from "./memory";
 import type { VideoEncoder } from "./types";
 
-/** An encoder pool buffer that Skia draws into through WebGPU. */
+/** A texture Skia draws an exported frame into through WebGPU: a lent
+ * encoder pool buffer on iOS, one of the encoder window's own textures on
+ * Android. */
 export type EncoderTarget = {
-  pointer: bigint;
-  /** Android: sync-fds ending the GPU access, handed over to the encoder. */
-  fences: bigint[];
+  /** iOS: the lent CVPixelBuffer. */
+  pointer: bigint | null;
   frame: NativeVideoFrame | null;
   memory: GPUSharedTextureMemory | null;
   texture: GPUTexture | null;
@@ -24,9 +36,25 @@ export type EncoderTarget = {
   canvas: SkCanvas | null;
 };
 
-// Captured on the RN runtime. iOS lends CVPixelBuffers, imported through
-// their IOSurface; Android lends the AHardwareBuffer itself.
-const lendsPixelBuffers = Platform.OS === "ios";
+/** Android: a canvas context presenting to the codec's input Surface. Skia
+ * draws each frame into one of two textures of our own, which a GPU copy then
+ * puts into the window's texture. The window's textures are the codec's
+ * input buffers: the copy waits on the GPU for the codec to hand one back,
+ * while the export goes on with the next frame, rather than the frame's
+ * checked wait. */
+export type EncoderWindow = {
+  context: RNCanvasContext;
+  device: GPUDevice;
+  configured: boolean;
+  /** Drawn into in turn, each copied into the window after its frame. */
+  slots: EncoderTarget[];
+  next: number;
+  reservations: number[];
+  /** The window texture the probe acquired, the first frame's copy target. */
+  pending: GPUTexture | null;
+};
+
+const windowSlots = 2;
 
 type SharedMemoryDevice = GPUDevice & {
   importSharedTextureMemory?: (
@@ -43,38 +71,19 @@ export const supportsEncoderTargets = (encoder: VideoEncoder): boolean => {
 };
 
 // Declared before its callers: worklets capture their closure on creation.
-/** Ends GPU access; the lent buffer stays with the caller, to encode or
- * release. Call only after a checked flush(true) of its surface. With
- * `encoding`, Android keeps the access's sync-fds in `target.fences` for the
- * encoder, which owns and closes them: ending an AHB access submits a
- * release that flush(true) has not waited for. */
-export const closeEncoderTarget = (
-  target: EncoderTarget,
-  encoding = false,
-): void => {
+/** iOS: ends GPU access; the lent buffer stays with the caller, to encode or
+ * release. Call only after a checked flush(true) of its surface. */
+export const closeEncoderTarget = (target: EncoderTarget): void => {
   "worklet";
-  target.canvas?.dispose();
+  // Disposing the surface detaches its canvas.
   target.canvas = null;
   target.surface?.dispose();
   target.surface = null;
   if (target.accessing && target.memory && target.texture) {
     const ended = target.memory.endAccess(target.texture);
     target.accessing = false;
-    for (const state of ended?.fences ?? []) {
-      try {
-        if (encoding && !lendsPixelBuffers) {
-          const exported = state.fence.export();
-          // Unreachable when negotiated: AHB interop requires the device's
-          // only shared-fence feature to be sync-fd (see gpuDevice.ts).
-          if (exported.type !== "sync-fd")
-            throw new Error("GPU export needs sync-fd fences");
-          target.fences.push(exported.handle);
-        }
-      } finally {
-        // iOS: already waited for. The wrapper is released either way.
-        state.fence.destroy();
-      }
-    }
+    // Already waited for. The wrappers are released either way.
+    for (const state of ended?.fences ?? []) state.fence.destroy();
   }
   target.accessing = false;
   target.texture?.destroy();
@@ -88,7 +97,7 @@ export const closeEncoderTarget = (
   target.frame = null;
 };
 
-/** Lends the next pool buffer and wraps it as a Skia surface. */
+/** iOS: lends the next pool buffer and wraps it as a Skia surface. */
 export const acquireEncoderTarget = (
   encoder: VideoEncoder,
   width: number,
@@ -97,7 +106,6 @@ export const acquireEncoderTarget = (
   "worklet";
   const target: EncoderTarget = {
     pointer: encoder.acquireFrameBuffer!(),
-    fences: [],
     frame: null,
     memory: null,
     texture: null,
@@ -106,33 +114,23 @@ export const acquireEncoderTarget = (
     canvas: null,
   };
   try {
-    if (lendsPixelBuffers) {
-      target.frame = createNativeVideoGpuFrame(target.pointer);
-      if (
-        target.frame.width !== width ||
-        target.frame.height !== height ||
-        target.frame.pixelFormat !== "bgra8"
-      ) {
-        throw new Error("Encoder buffer dimensions or BGRA layout mismatch");
-      }
+    target.frame = createNativeVideoGpuFrame(target.pointer!);
+    if (
+      target.frame.width !== width ||
+      target.frame.height !== height ||
+      target.frame.pixelFormat !== "bgra8"
+    ) {
+      throw new Error("Encoder buffer dimensions or BGRA layout mismatch");
     }
     const device = getVideoGpuDevice() as SharedMemoryDevice;
     if (typeof device.importSharedTextureMemory !== "function") {
       throw new Error("WebGPU shared texture memory is unavailable");
     }
     target.memory = device.importSharedTextureMemory({
-      handle: target.frame ? target.frame.handle : target.pointer,
+      handle: target.frame.handle,
       label: "video export frame",
     });
     target.texture = target.memory.createTexture();
-    if (
-      !lendsPixelBuffers &&
-      (target.texture.width !== width ||
-        target.texture.height !== height ||
-        target.texture.format !== "rgba8unorm")
-    ) {
-      throw new Error("Encoder buffer dimensions or RGBA layout mismatch");
-    }
     // Uninitialized: every frame starts with a full clear.
     target.memory.beginAccess(target.texture, false);
     target.accessing = true;
@@ -151,8 +149,9 @@ export const acquireEncoderTarget = (
   }
 };
 
-/** Clears one lent buffer on the GPU, then returns it unencoded. Returns the
- * failure, or null when this device can export through encoder targets. */
+/** iOS: clears one lent buffer on the GPU, then returns it unencoded.
+ * Returns the failure, or null when this device can export through encoder
+ * targets. */
 export const probeEncoderTarget = (
   encoder: VideoEncoder,
   width: number,
@@ -177,3 +176,165 @@ export const probeEncoderTarget = (
   encoder.releaseFrameBuffer!();
   return failure;
 };
+
+/** Android: the window's next texture, checked against the export size. */
+const windowTexture = (
+  window: EncoderWindow,
+  width: number,
+  height: number,
+): GPUTexture => {
+  "worklet";
+  const texture = window.pending ?? window.context.getCurrentTexture();
+  window.pending = null;
+  if (
+    texture.width !== width ||
+    texture.height !== height ||
+    texture.format !== "rgba8unorm"
+  ) {
+    throw new Error("Encoder window dimensions or RGBA layout mismatch");
+  }
+  return texture;
+};
+
+/** Android: the slot the next frame draws into. Its previous copy completed:
+ * the checked wait the export makes before each frame's import covers the
+ * queue before it. */
+export const acquireWindowTarget = (window: EncoderWindow): EncoderTarget => {
+  "worklet";
+  return window.slots[window.next]!;
+};
+
+/** Android: copies the slot Skia drew into to the window's next texture and
+ * hands it to the encoder, with its time. Call once the slot's surface was
+ * flushed: the copy follows the drawing on the same queue. */
+export const presentWindowTarget = (
+  encoder: VideoEncoder,
+  window: EncoderWindow,
+  target: EncoderTarget,
+  time: number,
+): void => {
+  "worklet";
+  const source = target.texture!;
+  const destination = windowTexture(window, source.width, source.height);
+  const commands = window.device.createCommandEncoder({
+    label: "video export window copy",
+  });
+  commands.copyTextureToTexture(
+    { texture: source },
+    { texture: destination },
+    { width: source.width, height: source.height },
+  );
+  window.device.queue.submit([commands.finish()]);
+  window.next = (window.next + 1) % window.slots.length;
+  // The encoder sets the frame's time on the window, then presents.
+  encoder.encodeFrame(
+    { kind: "window", present: () => window.context.present() },
+    time,
+  );
+};
+
+/** Android: the swapchain gives the codec its buffers back at once, rather
+ * than once the context is collected. Call before disposing the encoder. */
+export const closeEncoderWindow = (window: EncoderWindow): void => {
+  "worklet";
+  window.pending = null;
+  if (!window.configured) return;
+  window.context.unconfigure();
+  window.configured = false;
+};
+
+/** Android: frees the window's own textures. Call once their last recording
+ * was submitted: a copy still in flight keeps its source alive. */
+export const releaseEncoderWindow = (window: EncoderWindow): void => {
+  "worklet";
+  while (window.slots.length) {
+    const slot = window.slots[window.slots.length - 1]!;
+    // Disposing the surface detaches its canvas.
+    slot.canvas = null;
+    slot.surface?.dispose();
+    slot.surface = null;
+    slot.texture?.destroy();
+    slot.texture = null;
+    window.slots.pop();
+  }
+  while (window.reservations.length) {
+    releaseVideoMemory(window.reservations[window.reservations.length - 1]!);
+    window.reservations.pop();
+  }
+};
+
+/** Android: opens a canvas context on the encoder's input window with its
+ * slots, acquires the window's first texture and draws a clear into the
+ * first slot: nothing reaches the encoder. Returns the window, or the
+ * failure, after which the window is closed and the export can fall back to
+ * readback. */
+export const probeEncoderWindow = (
+  encoder: VideoEncoder,
+  width: number,
+  height: number,
+): { window: EncoderWindow | null; failure: unknown } => {
+  "worklet";
+  let window: EncoderWindow | null = null;
+  let failure: unknown = null;
+  try {
+    if (typeof encoder.acquireInputWindow !== "function")
+      throw new Error("Encoder cannot present through its input window");
+    const device = getVideoGpuDevice();
+    window = {
+      context: createNativeWindowGpuContext(
+        encoder.acquireInputWindow(),
+        width,
+        height,
+      ),
+      device,
+      configured: false,
+      slots: [],
+      next: 0,
+      reservations: [],
+      pending: null,
+    };
+    // Presented in FIFO order, opaque: every frame reaches the codec.
+    window.context.configure({
+      device,
+      format: "rgba8unorm",
+      usage: encoderWindowTextureUsage,
+    });
+    window.configured = true;
+    for (let i = 0; i < windowSlots; i++) {
+      window.reservations.push(
+        reserveVideoMemory(frameByteSize(width, height), "export window slot"),
+      );
+      const slot: EncoderTarget = {
+        pointer: null,
+        frame: null,
+        memory: null,
+        texture: device.createTexture({
+          label: "video export window slot",
+          size: { width, height },
+          format: "rgba8unorm",
+          usage: nativeVideoTextureUsage,
+        }),
+        accessing: false,
+        surface: null,
+        canvas: null,
+      };
+      window.slots.push(slot);
+      slot.surface = Skia.Surface.MakeFromGPUTexture(slot.texture!);
+      if (!slot.surface) throw new Error("Cannot wrap the export frame");
+      slot.canvas = getVideoCanvas(slot.surface);
+    }
+    window.pending = windowTexture(window, width, height);
+    const target = acquireWindowTarget(window);
+    target.canvas!.drawColor(Skia.Color("#00000000"), BlendMode.Clear);
+    target.surface!.flush(true);
+  } catch (error) {
+    failure = error ?? new Error("GPU export probe failed");
+  }
+  if (failure === null) return { window, failure };
+  if (window) {
+    // Not caught: a window still drawn by WebGPU cannot fall back to EGL.
+    closeEncoderWindow(window);
+    releaseEncoderWindow(window);
+  }
+  return { window: null, failure };
+};