react-native-webgpu 0.15.2 → 0.16.0

package/src/Canvas.tsx

@@ -2,49 +2,16 @@ import React, { useEffect, useImperativeHandle, useRef, useState } from "react";
 import type { ViewProps } from "react-native";
 import { Platform, View } from "react-native";
 
+import type { CanvasRef, RNCanvasContext } from "./types";
 import WebGPUNativeView from "./WebGPUViewNativeComponent";
 
+export type { CanvasRef, NativeCanvas, RNCanvasContext } from "./types";
+
 let CONTEXT_COUNTER = 1;
 function generateContextId() {
   return CONTEXT_COUNTER++;
 }
 
-type SurfacePointer = bigint;
-
-export interface NativeCanvas {
-  surface: SurfacePointer;
-  width: number;
-  height: number;
-  clientWidth: number;
-  clientHeight: number;
-  // No-op DOM-compatibility stubs so web renderers (Three.js,
-  // react-three-fiber) can treat the canvas like an HTMLCanvasElement.
-  addEventListener(type: string, listener: EventListener): void;
-  removeEventListener(type: string, listener: EventListener): void;
-  dispatchEvent(event: Event): void;
-  setPointerCapture(pointerId: number): void;
-  releasePointerCapture(pointerId: number): void;
-}
-
-export type RNCanvasContext = GPUCanvasContext & {
-  /**
-   * Present the current frame.
-   *
-   * Call this after `queue.submit()` on every runtime: the main JS runtime, the
-   * Reanimated UI runtime, and dedicated worklet runtimes (e.g.
-   * `createWorkletRuntime` / `runOnRuntime`, or a Vision Camera frame
-   * processor). It runs synchronously on the calling thread, so the frame is
-   * presented from whichever thread did the rendering.
-   */
-  present: () => void;
-};
-
-export interface CanvasRef {
-  getContextId: () => number;
-  getContext(contextName: "webgpu"): RNCanvasContext | null;
-  getNativeSurface: () => NativeCanvas;
-}
-
 export type AndroidSurfaceType =
   "SurfaceView" | "TextureView" | "HardwareBufferView";
 

package/src/index.tsx

@@ -9,6 +9,10 @@
 import "./main/installNative";
 
 import type {
+  GPUNativeBufferExternalTexture,
+  GPUNativeBufferExternalTextureDescriptor,
+  GPUNativeBufferTexture,
+  GPUNativeBufferTextureDescriptor,
   GPUDrawElementImageDestination,
   GPUDrawElementImageSource,
   GPUSharedFence,
@@ -44,6 +48,13 @@ export type {
   GPUCopyExternalNativeFrameSourceInfo,
   GPUDawnTogglesDescriptor,
   GPUNativeExternalTextureDescriptor,
+  NativeVideoFrameColorSpace,
+  GPUExtent2D,
+  GPUNativeBufferExternalTextureDescriptor,
+  GPUNativeBufferTextureDescriptor,
+  GPUNativeBufferWrapper,
+  GPUNativeBufferTexture,
+  GPUNativeBufferExternalTexture,
   GPUDrawElementImageDestination,
   GPUDrawElementImageSource,
   GPUDrawElementImageSourceView,
@@ -120,6 +131,19 @@ declare global {
       descriptor: GPUSharedTextureMemoryDescriptor,
     ): GPUSharedTextureMemory;
     importSharedFence(descriptor: GPUSharedFenceDescriptor): GPUSharedFence;
+    // Non-spec RN extensions, the counterparts of createTexture /
+    // createExternalTexture in GPUAndroidHardwareBufferUtil of Dawn's Android
+    // layer: import a native buffer once, as a regular texture or as an
+    // external texture, and bracket each use with beginAccess() / endAccess()
+    // on the wrapper instead of importing the buffer again per frame.
+    createNativeBufferTexture(
+      source: NativeVideoFrame,
+      descriptor?: GPUNativeBufferTextureDescriptor,
+    ): GPUNativeBufferTexture;
+    createNativeBufferExternalTexture(
+      source: NativeVideoFrame,
+      descriptor?: GPUNativeBufferExternalTextureDescriptor,
+    ): GPUNativeBufferExternalTexture;
   }
 
   // Non-spec RN extension modeled on the html-in-canvas proposal's

package/src/types.ts

@@ -52,6 +52,13 @@ export type NativeVideoPixelFormat = "bgra8" | "nv12";
 //   - pixelFormat describes the surface layout: 'bgra8' for a sampled
 //     GPUTexture; 'nv12' (biplanar Y + CbCr) for the importExternalTexture
 //     path.
+//   - acquireFenceFd is the sync fd the producer of the frame (the decoder,
+//     the camera) signals once it has written the buffer, as a BigInt, on
+//     Android; null when the frame has none (Apple platforms, a released
+//     frame). importExternalTexture() waits for it on its own; a hardware
+//     buffer wrapper takes it through importSharedFence({ type: "sync-fd",
+//     handle }) in beginAccess(). The frame owns the descriptor: import it
+//     before release(), and never close it.
 //   - release() drops the underlying backing object (a CVPixelBuffer on Apple).
 //     The frame is also released when the JS wrapper is garbage-collected; call
 //     release() eagerly when you know you're done.
@@ -60,9 +67,31 @@ export interface NativeVideoFrame {
   readonly width: number;
   readonly height: number;
   readonly pixelFormat: NativeVideoPixelFormat;
+  readonly acquireFenceFd: bigint | null;
   release(): void;
 }
 
+// The color space of a YUV frame, the way its producer reports it: the YCbCr
+// matrix and range that encode the samples, and the primaries and transfer
+// function of the RGB they encode. Each member is optional; what is left out
+// comes from the frame, then from the platform's best guess for the buffer.
+export interface NativeVideoFrameColorSpace {
+  matrix?: "identity" | "rec601" | "rec709" | "rec2020";
+  range?: "identity" | "narrow" | "full";
+  primaries?: "srgb" | "rec709" | "rec601" | "rec2020" | "display-p3";
+  transfer?:
+    | "identity"
+    | "srgb"
+    | "display-p3"
+    | "smpte-170m"
+    | "hlg"
+    | "pq"
+    | "bt-1886";
+}
+
+// A 2D size, as { width, height } or [width, height].
+export type GPUExtent2D = { width: number; height: number } | [number, number];
+
 // GPUExternalTextureDescriptor with a NativeVideoFrame as the source: what
 // GPUDevice.importExternalTexture() takes on React Native, where there is no
 // HTMLVideoElement or WebCodecs VideoFrame. The other members (label,
@@ -72,6 +101,74 @@ export interface GPUNativeExternalTextureDescriptor extends Omit<
   "source"
 > {
   source: NativeVideoFrame;
+  // Overrides the members of the frame's own color space, for a buffer that
+  // carries none (one wrapped from another library) or reports it wrong.
+  sourceColorSpace?: NativeVideoFrameColorSpace;
+  // The part of the frame the external texture shows, the whole frame by
+  // default (Dawn's cropOrigin / cropSize).
+  cropOrigin?: GPUOrigin2D;
+  cropSize?: GPUExtent2D;
+}
+
+// The descriptor of GPUDevice.createNativeBufferExternalTexture(): the same
+// options as importExternalTexture(), without the source (the frame is the
+// first argument).
+export type GPUNativeBufferExternalTextureDescriptor = Omit<
+  GPUNativeExternalTextureDescriptor,
+  "source"
+>;
+
+// The options of GPUDevice.createNativeBufferTexture(). `usage` defaults to
+// every usage the buffer allows, as GPUSharedTextureMemory.createTexture()
+// without a descriptor.
+export interface GPUNativeBufferTextureDescriptor {
+  usage?: GPUTextureUsageFlags;
+  label?: string;
+}
+
+// A native buffer imported once, with the access window of each use explicit:
+// the counterpart of GPUHardwareBufferWrapper in Dawn's Android layer
+// (androidx.webgpu). A decoder or a camera renders into the same few buffers
+// in turn; keep one wrapper per buffer (keyed by NativeVideoFrame.handle), and
+// bracket each frame with beginAccess() / endAccess() instead of importing it
+// again. The wrapper keeps the frame it was created from alive until
+// destroy().
+export interface GPUNativeBufferWrapper {
+  label: string;
+  // Begins an access to the buffer: the GPU waits for each fence to reach its
+  // signaledValue first (the producer's fence for the frame, e.g. the frame's
+  // acquireFenceFd imported with importSharedFence, 1n for a sync fd). Throws,
+  // after a validation error, on a destroyed wrapper or fence, and if Dawn
+  // rejects the access (one is already open).
+  beginAccess(fences?: GPUSharedFenceState[]): void;
+  // Ends the access, after the queue.submit() that used the buffer, and
+  // returns the one fence that signals once that GPU work is done (on Android,
+  // the sync fds of the access merged into one), or null when there is none.
+  // Hand it to the producer before it reuses the buffer, or destroy it.
+  endAccess(): GPUSharedFenceState | null;
+  // Releases the import (and the frame) now rather than when the object is
+  // garbage collected. An access still open is a validation error; it is
+  // ended, and its fence is lost. A later use is a validation error and a
+  // throw.
+  destroy(): void;
+}
+
+// A color buffer (an RGBA / R8 / ... AHardwareBuffer, a BGRA IOSurface)
+// imported once as a regular texture. Bind `texture` only between
+// beginAccess() and endAccess().
+export interface GPUNativeBufferTexture extends GPUNativeBufferWrapper {
+  readonly __brand: "GPUNativeBufferTexture";
+  readonly texture: GPUTexture;
+}
+
+// A frame buffer (the YUV AHardwareBuffer of a decoder or a camera, an NV12 or
+// BGRA IOSurface) imported once as an external texture, color conversion
+// included. Bind `externalTexture` only between beginAccess() and endAccess();
+// destroying it on its own (externalTexture.destroy()) only releases the
+// external texture, the wrapper keeps the import.
+export interface GPUNativeBufferExternalTexture extends GPUNativeBufferWrapper {
+  readonly __brand: "GPUNativeBufferExternalTexture";
+  readonly externalTexture: GPUExternalTexture;
 }
 
 // A decoded video, modeled on HTMLMediaElement: play(), pause(), paused,