react-native-webgpu 0.15.2 → 0.16.1

package/src/Canvas.tsx

@@ -2,64 +2,38 @@ import React, { useEffect, useImperativeHandle, useRef, useState } from "react";
 import type { ViewProps } from "react-native";
 import { Platform, View } from "react-native";
 
+import type { CanvasRef, RNCanvasContext } from "./types";
 import WebGPUNativeView from "./WebGPUViewNativeComponent";
 
+export type { CanvasRef, NativeCanvas, RNCanvasContext } from "./types";
+
 let CONTEXT_COUNTER = 1;
 function generateContextId() {
   return CONTEXT_COUNTER++;
 }
 
-type SurfacePointer = bigint;
-
-export interface NativeCanvas {
-  surface: SurfacePointer;
-  width: number;
-  height: number;
-  clientWidth: number;
-  clientHeight: number;
-  // No-op DOM-compatibility stubs so web renderers (Three.js,
-  // react-three-fiber) can treat the canvas like an HTMLCanvasElement.
-  addEventListener(type: string, listener: EventListener): void;
-  removeEventListener(type: string, listener: EventListener): void;
-  dispatchEvent(event: Event): void;
-  setPointerCapture(pointerId: number): void;
-  releasePointerCapture(pointerId: number): void;
-}
-
-export type RNCanvasContext = GPUCanvasContext & {
-  /**
-   * Present the current frame.
-   *
-   * Call this after `queue.submit()` on every runtime: the main JS runtime, the
-   * Reanimated UI runtime, and dedicated worklet runtimes (e.g.
-   * `createWorkletRuntime` / `runOnRuntime`, or a Vision Camera frame
-   * processor). It runs synchronously on the calling thread, so the frame is
-   * presented from whichever thread did the rendering.
-   */
-  present: () => void;
-};
-
-export interface CanvasRef {
-  getContextId: () => number;
-  getContext(contextName: "webgpu"): RNCanvasContext | null;
-  getNativeSurface: () => NativeCanvas;
-}
-
 export type AndroidSurfaceType =
   "SurfaceView" | "TextureView" | "HardwareBufferView";
 
+/**
+ * Android-only rendering options. They select the backing view, which is
+ * chosen when the canvas mounts and kept: a later change is ignored, with a
+ * warning in development. Remount the canvas, with a `key`, to change it.
+ */
 export interface AndroidCanvasProps {
   /**
-   * Backing view. Defaults to `SurfaceView` when the canvas is opaque and to
-   * `TextureView` otherwise; both composite correctly in React Native
+   * Backing view. Defaults to `SurfaceView` when the canvas mounts opaque and
+   * to `TextureView` otherwise; both composite correctly in React Native
    * stacking order without further flags. `HardwareBufferView` (Android 10+,
    * a plain View that draws each frame's AHardwareBuffer inline with no extra
    * copy) is opt-in and experimental; on older devices it uses `TextureView`.
+   * Read once, at mount.
    */
   surfaceType?: AndroidSurfaceType;
   /**
    * SurfaceView only: composite above every React Native view in the window,
-   * ignoring `zIndex`. Ignored for TextureView. Defaults to false.
+   * ignoring `zIndex`. Ignored for TextureView. Defaults to false. Applied in
+   * place on Android 11 and later; read once, at mount, before that.
    */
   zOrderOnTop?: boolean;
 }
@@ -69,9 +43,16 @@ export interface CanvasProps extends ViewProps {
    * Defaults to true. Set to false to alpha-composite the canvas over the
    * views behind it (pair it with `alphaMode: "premultiplied"` and an alpha-0
    * clear color). Android and web only; on iOS `alphaMode` alone controls it.
+   * On Android the backing view is chosen from it when the canvas mounts; a
+   * later change updates that view in place.
    */
   opaque?: boolean;
-  /** Android-only rendering options. Ignored on iOS and web. */
+  /**
+   * Android-only rendering options. They select the backing view, which is
+   * chosen when the canvas mounts and kept: a later change is ignored, with a
+   * warning in development. Remount the canvas, with a `key`, to change it.
+   * Ignored on iOS and web.
+   */
   android?: AndroidCanvasProps;
   ref?: React.Ref<CanvasRef>;
 }
@@ -87,6 +68,58 @@ const resolveSurfaceType = (
     ? surfaceType
     : "auto";
 
+// The props the Android backing view is chosen from when the canvas mounts.
+interface BackingViewProps {
+  opaque: boolean;
+  android?: AndroidCanvasProps;
+}
+
+// The backing view the props select, see WebGPUView.resolveKind().
+const resolveKind = ({ opaque, android }: BackingViewProps) => {
+  const surfaceType = resolveSurfaceType(android?.surfaceType);
+  if (surfaceType !== "auto") {
+    return surfaceType;
+  }
+  return opaque ? "SurfaceView" : "TextureView";
+};
+
+const warnIgnored = (prop: string) =>
+  console.warn(
+    `Ignoring a change of ${prop} on a mounted canvas: the backing view is ` +
+      "chosen when the canvas mounts. Remount the canvas (give it a new key) " +
+      "to change it.",
+  );
+
+// Warns, in development on Android, about a change of these props that the
+// native view ignores: the backing view is chosen from them when the canvas
+// mounts (see WebGPUView.java, whose setters these rules mirror). The native
+// side only says so in logcat, which is not where a JS developer looks.
+const useBackingViewWarning = (props: BackingViewProps) => {
+  const mount = useRef(props);
+  const prev = useRef(props);
+  useEffect(() => {
+    if (__DEV__ && Platform.OS === "android") {
+      const { current: before } = prev;
+      if (
+        resolveSurfaceType(before.android?.surfaceType) !==
+        resolveSurfaceType(props.android?.surfaceType)
+      ) {
+        warnIgnored("android.surfaceType");
+      }
+      // Only a SurfaceView has a z-order of its own, which Android 11 and
+      // later changes in place.
+      if (
+        !!before.android?.zOrderOnTop !== !!props.android?.zOrderOnTop &&
+        resolveKind(mount.current) === "SurfaceView" &&
+        Number(Platform.Version) < 30
+      ) {
+        warnIgnored("android.zOrderOnTop");
+      }
+    }
+    prev.current = props;
+  });
+};
+
 // A view measured relative to itself gets its layout size, without the
 // transforms getBoundingClientRect() includes; measureLayout() calls back
 // synchronously on the new architecture. On the web react-native-web measures
@@ -113,6 +146,7 @@ export const Canvas = ({
 }: CanvasProps) => {
   const viewRef = useRef<View>(null);
   const [contextId, _] = useState(() => generateContextId());
+  useBackingViewWarning({ opaque, android });
   // Retire the native registry entry for this contextId on unmount. When a
   // native surface is still attached, this is a no-op and the native view's
   // own teardown retires the entry instead — which keeps StrictMode's

package/src/external/reanimated/registerWebGPUForReanimated.ts

@@ -10,6 +10,10 @@ interface WebGPUWorkletHelpers {
   // WebGPU prototype are rejected, and no JS property is ever read.
   isNativeObject: (value: unknown) => boolean;
   box: (value: object) => BoxedWebGPUObject;
+  // Installs the constructor of every WebGPU class on the calling runtime,
+  // skipping the ones already there. installWebGPU() calls it so `instanceof`
+  // works on worklet runtimes.
+  installConstructors: () => void;
 }
 
 interface BoxedWebGPUObject {

package/src/index.tsx

@@ -9,6 +9,10 @@
 import "./main/installNative";
 
 import type {
+  GPUNativeBufferExternalTexture,
+  GPUNativeBufferExternalTextureDescriptor,
+  GPUNativeBufferTexture,
+  GPUNativeBufferTextureDescriptor,
   GPUDrawElementImageDestination,
   GPUDrawElementImageSource,
   GPUSharedFence,
@@ -44,6 +48,13 @@ export type {
   GPUCopyExternalNativeFrameSourceInfo,
   GPUDawnTogglesDescriptor,
   GPUNativeExternalTextureDescriptor,
+  NativeVideoFrameColorSpace,
+  GPUExtent2D,
+  GPUNativeBufferExternalTextureDescriptor,
+  GPUNativeBufferTextureDescriptor,
+  GPUNativeBufferWrapper,
+  GPUNativeBufferTexture,
+  GPUNativeBufferExternalTexture,
   GPUDrawElementImageDestination,
   GPUDrawElementImageSource,
   GPUDrawElementImageSourceView,
@@ -120,6 +131,19 @@ declare global {
       descriptor: GPUSharedTextureMemoryDescriptor,
     ): GPUSharedTextureMemory;
     importSharedFence(descriptor: GPUSharedFenceDescriptor): GPUSharedFence;
+    // Non-spec RN extensions, the counterparts of createTexture /
+    // createExternalTexture in GPUAndroidHardwareBufferUtil of Dawn's Android
+    // layer: import a native buffer once, as a regular texture or as an
+    // external texture, and bracket each use with beginAccess() / endAccess()
+    // on the wrapper instead of importing the buffer again per frame.
+    createNativeBufferTexture(
+      source: NativeVideoFrame,
+      descriptor?: GPUNativeBufferTextureDescriptor,
+    ): GPUNativeBufferTexture;
+    createNativeBufferExternalTexture(
+      source: NativeVideoFrame,
+      descriptor?: GPUNativeBufferExternalTextureDescriptor,
+    ): GPUNativeBufferExternalTexture;
   }
 
   // Non-spec RN extension modeled on the html-in-canvas proposal's

package/src/install.ts

@@ -29,14 +29,25 @@ const constants = {
 const gpu: GPU | undefined =
   typeof RNWebGPU === "undefined" ? undefined : RNWebGPU.gpu;
 
+// The native half of the worklet serializer (see registerWebGPUForReanimated),
+// read from the main runtime like `gpu` above and captured into the worklet
+// closure: worklets hands a HostObject to every runtime by reference, so it is
+// callable on the target runtime without installing anything there. Its
+// installConstructors() installs the WebGPU class constructors on the runtime
+// that calls it. Undefined on web and node, where the native module is not
+// installed.
+const helpers = globalThis.__webgpuWorkletHelpers;
+
 /**
  * Install WebGPU on the runtime that calls it.
  *
  * The native module sets up WebGPU on the main JS runtime, but worklet
  * runtimes (Reanimated UI, dedicated worklet runtimes, Vision Camera frame
- * processors) start without it: `navigator.gpu` and the flag constants
+ * processors) start without it: `navigator.gpu`, the flag constants
  * (`GPUBufferUsage`, `GPUTextureUsage`, `GPUShaderStage`, `GPUColorWrite`,
- * `GPUMapMode`) are all `undefined` there.
+ * `GPUMapMode`) and the class constructors (`GPUBuffer`, `GPUTexture`,
+ * `GPURenderBundleEncoder`, ...) are all `undefined` there. Without the
+ * constructors, `instanceof` against a WebGPU class throws a ReferenceError.
  *
  * Call `installWebGPU()` once at the top of a worklet to make them available:
  *
@@ -65,12 +76,13 @@ const gpu: GPU | undefined =
  *   Observe those on a device created on the main JS thread.
  *
  * Everything is captured into the worklet by closure: the constants like a
- * shader string would be, and the GPU object through the Worklets custom
+ * shader string would be, the GPU object through the Worklets custom
  * serializer, which installs the native prototypes on the target runtime when
- * it crosses. Promises returned by `requestAdapter`/`requestDevice` settle on
- * the calling runtime (each runtime gets its own async pump). Calling it on a
- * runtime that already has the globals (e.g. the main JS runtime) is a safe
- * no-op.
+ * it crosses, and the native helper that installs the class constructors,
+ * which worklets hands to every runtime by reference. Promises returned by
+ * `requestAdapter`/`requestDevice` settle on the calling runtime (each
+ * runtime gets its own async pump). Calling it on a runtime that already has
+ * the globals (e.g. the main JS runtime) is a safe no-op.
  */
 export const installWebGPU = () => {
   "worklet";
@@ -88,4 +100,11 @@ export const installWebGPU = () => {
       nav.gpu = gpu;
     }
   }
+  // The class constructors, so `instanceof` works against any WebGPU class on
+  // this runtime (three.js checks `pass instanceof GPURenderBundleEncoder` on
+  // every render, for instance). Unboxing an object installs only its own
+  // class; this installs the rest and skips the ones already there.
+  if (helpers !== undefined) {
+    helpers.installConstructors();
+  }
 };

package/src/types.ts

@@ -21,7 +21,7 @@ export type RNCanvasContext = GPUCanvasContext & {
    *
    * Call this after `queue.submit()` on every runtime: the main JS runtime, the
    * Reanimated UI runtime, and dedicated worklet runtimes (e.g.
-   * `createWorkletRuntime` / `runOnRuntime`, or a Vision Camera frame
+   * `createWorkletRuntime` / `scheduleOnRuntime`, or a Vision Camera frame
    * processor). It runs synchronously on the calling thread, so the frame is
    * presented from whichever thread did the rendering.
    */
@@ -52,6 +52,13 @@ export type NativeVideoPixelFormat = "bgra8" | "nv12";
 //   - pixelFormat describes the surface layout: 'bgra8' for a sampled
 //     GPUTexture; 'nv12' (biplanar Y + CbCr) for the importExternalTexture
 //     path.
+//   - acquireFenceFd is the sync fd the producer of the frame (the decoder,
+//     the camera) signals once it has written the buffer, as a BigInt, on
+//     Android; null when the frame has none (Apple platforms, a released
+//     frame). importExternalTexture() waits for it on its own; a hardware
+//     buffer wrapper takes it through importSharedFence({ type: "sync-fd",
+//     handle }) in beginAccess(). The frame owns the descriptor: import it
+//     before release(), and never close it.
 //   - release() drops the underlying backing object (a CVPixelBuffer on Apple).
 //     The frame is also released when the JS wrapper is garbage-collected; call
 //     release() eagerly when you know you're done.
@@ -60,9 +67,31 @@ export interface NativeVideoFrame {
   readonly width: number;
   readonly height: number;
   readonly pixelFormat: NativeVideoPixelFormat;
+  readonly acquireFenceFd: bigint | null;
   release(): void;
 }
 
+// The color space of a YUV frame, the way its producer reports it: the YCbCr
+// matrix and range that encode the samples, and the primaries and transfer
+// function of the RGB they encode. Each member is optional; what is left out
+// comes from the frame, then from the platform's best guess for the buffer.
+export interface NativeVideoFrameColorSpace {
+  matrix?: "identity" | "rec601" | "rec709" | "rec2020";
+  range?: "identity" | "narrow" | "full";
+  primaries?: "srgb" | "rec709" | "rec601" | "rec2020" | "display-p3";
+  transfer?:
+    | "identity"
+    | "srgb"
+    | "display-p3"
+    | "smpte-170m"
+    | "hlg"
+    | "pq"
+    | "bt-1886";
+}
+
+// A 2D size, as { width, height } or [width, height].
+export type GPUExtent2D = { width: number; height: number } | [number, number];
+
 // GPUExternalTextureDescriptor with a NativeVideoFrame as the source: what
 // GPUDevice.importExternalTexture() takes on React Native, where there is no
 // HTMLVideoElement or WebCodecs VideoFrame. The other members (label,
@@ -72,6 +101,74 @@ export interface GPUNativeExternalTextureDescriptor extends Omit<
   "source"
 > {
   source: NativeVideoFrame;
+  // Overrides the members of the frame's own color space, for a buffer that
+  // carries none (one wrapped from another library) or reports it wrong.
+  sourceColorSpace?: NativeVideoFrameColorSpace;
+  // The part of the frame the external texture shows, the whole frame by
+  // default (Dawn's cropOrigin / cropSize).
+  cropOrigin?: GPUOrigin2D;
+  cropSize?: GPUExtent2D;
+}
+
+// The descriptor of GPUDevice.createNativeBufferExternalTexture(): the same
+// options as importExternalTexture(), without the source (the frame is the
+// first argument).
+export type GPUNativeBufferExternalTextureDescriptor = Omit<
+  GPUNativeExternalTextureDescriptor,
+  "source"
+>;
+
+// The options of GPUDevice.createNativeBufferTexture(). `usage` defaults to
+// every usage the buffer allows, as GPUSharedTextureMemory.createTexture()
+// without a descriptor.
+export interface GPUNativeBufferTextureDescriptor {
+  usage?: GPUTextureUsageFlags;
+  label?: string;
+}
+
+// A native buffer imported once, with the access window of each use explicit:
+// the counterpart of GPUHardwareBufferWrapper in Dawn's Android layer
+// (androidx.webgpu). A decoder or a camera renders into the same few buffers
+// in turn; keep one wrapper per buffer (keyed by NativeVideoFrame.handle), and
+// bracket each frame with beginAccess() / endAccess() instead of importing it
+// again. The wrapper keeps the frame it was created from alive until
+// destroy().
+export interface GPUNativeBufferWrapper {
+  label: string;
+  // Begins an access to the buffer: the GPU waits for each fence to reach its
+  // signaledValue first (the producer's fence for the frame, e.g. the frame's
+  // acquireFenceFd imported with importSharedFence, 1n for a sync fd). Throws,
+  // after a validation error, on a destroyed wrapper or fence, and if Dawn
+  // rejects the access (one is already open).
+  beginAccess(fences?: GPUSharedFenceState[]): void;
+  // Ends the access, after the queue.submit() that used the buffer, and
+  // returns the one fence that signals once that GPU work is done (on Android,
+  // the sync fds of the access merged into one), or null when there is none.
+  // Hand it to the producer before it reuses the buffer, or destroy it.
+  endAccess(): GPUSharedFenceState | null;
+  // Releases the import (and the frame) now rather than when the object is
+  // garbage collected. An access still open is a validation error; it is
+  // ended, and its fence is lost. A later use is a validation error and a
+  // throw.
+  destroy(): void;
+}
+
+// A color buffer (an RGBA / R8 / ... AHardwareBuffer, a BGRA IOSurface)
+// imported once as a regular texture. Bind `texture` only between
+// beginAccess() and endAccess().
+export interface GPUNativeBufferTexture extends GPUNativeBufferWrapper {
+  readonly __brand: "GPUNativeBufferTexture";
+  readonly texture: GPUTexture;
+}
+
+// A frame buffer (the YUV AHardwareBuffer of a decoder or a camera, an NV12 or
+// BGRA IOSurface) imported once as an external texture, color conversion
+// included. Bind `externalTexture` only between beginAccess() and endAccess();
+// destroying it on its own (externalTexture.destroy()) only releases the
+// external texture, the wrapper keeps the import.
+export interface GPUNativeBufferExternalTexture extends GPUNativeBufferWrapper {
+  readonly __brand: "GPUNativeBufferExternalTexture";
+  readonly externalTexture: GPUExternalTexture;
 }
 
 // A decoded video, modeled on HTMLMediaElement: play(), pause(), paused,