react-native-webgpu 0.10.5 → 0.11.1

package/src/importDevice.ts

@@ -26,10 +26,10 @@ export const importDevice = (pointer: bigint): GPUDevice => {
  * ownership of one reference: the returned texture releases it when destroyed.
  *
  * Pair with producers that hand out a +1 reference, such as
- * `Skia.Image.MakeNativeTextureFromImage()`:
+ * `Skia.Image.MakeGPUTextureFromImage()`:
  *
  * ```ts
- * const texture = adoptTexture(Skia.Image.MakeNativeTextureFromImage(image));
+ * const texture = adoptTexture(Skia.Image.MakeGPUTextureFromImage(image));
  * ```
  */
 export const adoptTexture = (pointer: bigint): GPUTexture => {

package/src/index.tsx

@@ -1,10 +1,14 @@
 /// <reference types="@webgpu/types" />
 import type {
+  GPUDrawElementImageDestination,
+  GPUDrawElementImageSource,
   GPUSharedFence,
   GPUSharedFenceDescriptor,
   GPUDawnTogglesDescriptor,
+  GPUNativeExternalTextureDescriptor,
   GPUSharedTextureMemory,
   GPUSharedTextureMemoryDescriptor,
+  GPUCopyExternalNativeFrameSourceInfo,
   NativeCanvas,
   RNCanvasContext,
   VideoPlayer,
@@ -28,7 +32,12 @@ export type {
   GPUSharedTextureMemoryEndAccessState,
   NativeVideoPixelFormat,
   CreateVideoPlayerOptions,
+  GPUCopyExternalNativeFrameSourceInfo,
   GPUDawnTogglesDescriptor,
+  GPUNativeExternalTextureDescriptor,
+  GPUDrawElementImageDestination,
+  GPUDrawElementImageSource,
+  GPUDrawElementImageSourceView,
 } from "./types";
 
 declare global {
@@ -64,10 +73,11 @@ declare global {
     importDevice: (pointer: bigint) => GPUDevice;
     // Wrap an externally created WGPUTexture pointer into a GPUTexture,
     // taking ownership of one reference (pair with producers that return a
-    // +1 pointer, e.g. Skia.Image.MakeNativeTextureFromImage()).
+    // +1 pointer, e.g. Skia.Image.MakeGPUTextureFromImage()).
     adoptTexture: (pointer: bigint) => GPUTexture;
+    // Prefer the createVideoPlayer() export, which takes an options object.
     createVideoPlayer: (
-      path: string,
+      source: string,
       pixelFormat?: NativeVideoPixelFormat,
     ) => VideoPlayer;
     writeTestVideoFile: () => string;
@@ -75,18 +85,62 @@ declare global {
 
   interface GPUTexture {
     // Non-spec RN extension: raw WGPUTexture handle as a BigInt for
-    // pointer-based interop (e.g. Skia.Image.MakeImageFromNativeTexture).
+    // pointer-based interop (Skia.Image.MakeImageFromGPUTexture reads it).
     // Borrowed: keep this GPUTexture alive while the pointer is in use.
     readonly nativePointer: bigint;
   }
 
   interface GPUDevice {
+    // Non-spec RN extension: the source of an external texture is a
+    // NativeVideoFrame (from a VideoPlayer, createVideoFrameFromNativeBuffer
+    // or createTestVideoFrame) instead of the web's HTMLVideoElement /
+    // VideoFrame. Overloads the spec signature, which stays available for
+    // the web implementation.
+    importExternalTexture(
+      descriptor: GPUNativeExternalTextureDescriptor,
+    ): GPUExternalTexture;
     importSharedTextureMemory(
       descriptor: GPUSharedTextureMemoryDescriptor,
     ): GPUSharedTextureMemory;
     importSharedFence(descriptor: GPUSharedFenceDescriptor): GPUSharedFence;
   }
 
+  // Non-spec RN extension modeled on the html-in-canvas proposal's
+  // GPUQueue.drawElementImageToTexture: rasterizes a native React Native view
+  // (the "element") and writes the pixels into `destination.texture`, which
+  // must be rgba8unorm or bgra8unorm (or their -srgb variants) with
+  // GPUTextureUsage.COPY_DST. Unlike the web API it returns a Promise: the
+  // view is rasterized on the platform UI thread, and the texture write is
+  // issued right before the promise resolves, so submit work that samples the
+  // texture after awaiting it. Snapshot the view with `collapsable={false}`
+  // so React Native does not optimize the host view away.
+  //
+  // What is captured is what the platform renders for the view hierarchy
+  // (iOS: UIView.drawViewHierarchyInRect; Android API 29+: HardwareRenderer
+  // over the views' display lists, older: View.draw into a software canvas).
+  // On Android, SurfaceView content (a WebGPU Canvas with the default
+  // "SurfaceView" surfaceType, video players) is not part of the hierarchy and
+  // comes out blank; use `android={{ surfaceType: "TextureView" }}` for a
+  // canvas that must be captured.
+  interface GPUQueue {
+    drawElementImageToTexture(
+      source: GPUDrawElementImageSource,
+      destination: GPUDrawElementImageDestination,
+    ): Promise<void>;
+    // Non-spec extension: a NativeVideoFrame (from createVideoPlayer,
+    // createVideoFrameFromNativeBuffer or createTestVideoFrame) as the copy
+    // source. The frame is rendered into the destination on the GPU through
+    // an external texture, which converts YUV to RGB and applies `rotation`
+    // and `mirrored`; the destination needs GPUTextureUsage.RENDER_ATTACHMENT
+    // and an rgba8unorm, bgra8unorm (or -srgb), rgba16float or rgb10a2unorm
+    // format. The frame can be released as soon as the call returns.
+    copyExternalImageToTexture(
+      source: GPUCopyExternalNativeFrameSourceInfo,
+      destination: GPUCopyExternalImageDestInfo,
+      copySize: GPUExtent3DStrict,
+    ): undefined;
+  }
+
   // Non-standard, Dawn-only. Lets callers set Dawn device-stage toggles at
   // device creation: adapter.requestDevice({ dawnToggles: { ... } }).
   interface GPUDeviceDescriptor {

package/src/main/index.tsx

@@ -1,6 +1,54 @@
+import { findNodeHandle } from "react-native";
+
 import { registerWebGPUForReanimated } from "../external";
 import { provideGPUForInstall } from "../install";
 import WebGPUModule from "../NativeWebGPUModule";
+import type { GPUDrawElementImageSourceView } from "../types";
+
+// Resolve the `source` of drawElementImageToTexture to a React tag on the
+// main JS runtime, where findNodeHandle is available: it handles class
+// components and legacy host instances that the native fallback (a number or
+// an instance carrying `__nativeTag`) does not.
+const resolveElementSource = (source: GPUDrawElementImageSourceView) => {
+  if (typeof source === "number") {
+    return source;
+  }
+  const instance =
+    source !== null && typeof source === "object" && "current" in source
+      ? source.current
+      : source;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const tag = instance == null ? null : findNodeHandle(instance as any);
+  if (tag === null || tag === undefined) {
+    throw new Error(
+      "[WebGPU] drawElementImageToTexture: `source` does not resolve to a " +
+        "mounted native view (pass a host component ref rendered with " +
+        "collapsable={false})",
+    );
+  }
+  return tag;
+};
+
+const installDrawElementImageToTexture = () => {
+  if (typeof GPUQueue === "undefined") {
+    return;
+  }
+  const native = GPUQueue.prototype.drawElementImageToTexture;
+  if (typeof native !== "function") {
+    return;
+  }
+  GPUQueue.prototype.drawElementImageToTexture = function (
+    this: GPUQueue,
+    source,
+    destination,
+  ) {
+    return native.call(
+      this,
+      { ...source, source: resolveElementSource(source.source) },
+      destination,
+    );
+  };
+};
 
 export * from "../Canvas";
 export * from "../Offscreen";
@@ -8,6 +56,7 @@ export * from "../WebGPUViewNativeComponent";
 export * from "../hooks";
 export * from "../GPUDeviceProvider";
 export * from "../importDevice";
+export * from "../video";
 export * from "../formats";
 
 export { default as WebGPUModule } from "../NativeWebGPUModule";
@@ -37,6 +86,7 @@ if (typeof RNWebGPU !== "undefined") {
   }
   global.createImageBitmap =
     global.createImageBitmap ?? RNWebGPU.createImageBitmap.bind(RNWebGPU);
+  installDrawElementImageToTexture();
   // Let installWebGPU() put navigator.gpu on other runtimes (see install.ts).
   provideGPUForInstall(RNWebGPU.gpu);
 } else {

package/src/main/index.web.tsx

@@ -6,5 +6,6 @@ export * from "../WebGPUViewNativeComponent";
 export * from "../hooks";
 export * from "../GPUDeviceProvider";
 export * from "../formats";
+export * from "../video";
 
 // We don't need to set all global properties on web, webgpu is already available globally

package/src/types.ts

@@ -63,24 +63,106 @@ export interface NativeVideoFrame {
   release(): void;
 }
 
-// A handle to a decoded video stream. Poll copyLatestFrame() each render tick
-// to obtain the most recently decoded frame as an IOSurface/AHardwareBuffer
-// (returns null between frames so callers can skip the import work).
+// GPUExternalTextureDescriptor with a NativeVideoFrame as the source: what
+// GPUDevice.importExternalTexture() takes on React Native, where there is no
+// HTMLVideoElement or WebCodecs VideoFrame. The other members (label,
+// colorSpace, and the rotation / mirrored extensions) are the same.
+export interface GPUNativeExternalTextureDescriptor extends Omit<
+  GPUExternalTextureDescriptor,
+  "source"
+> {
+  source: NativeVideoFrame;
+}
+
+// A decoded video, modeled on HTMLMediaElement: play(), pause(), paused,
+// currentTime (in seconds, assign it to seek), duration, volume and loop work
+// as on the web. Frames are pulled instead of displayed: poll
+// copyLatestFrame() on every render tick to obtain the most recently decoded
+// frame as an IOSurface/AHardwareBuffer-backed NativeVideoFrame (null between
+// frames, so callers can skip the import work).
 export interface VideoPlayer {
   copyLatestFrame(): NativeVideoFrame | null;
   play(): void;
   pause(): void;
+  // Stops playback and releases the decoder. Frames already copied stay
+  // valid until they are released.
   release(): void;
+  readonly paused: boolean;
+  // The playback position in seconds. Assign it to seek, also while paused:
+  // the next copyLatestFrame() then returns the frame at the new position.
+  currentTime: number;
+  // The length in seconds, 0 until the metadata has loaded.
+  readonly duration: number;
+  // 0 to 1.
+  volume: number;
+  // Restart from the beginning at the end of the stream. Unlike on the web,
+  // it defaults to true.
+  loop: boolean;
+  // The coded size of the frames in pixels, 0 until the metadata has loaded.
+  readonly videoWidth: number;
+  readonly videoHeight: number;
+  // The clockwise rotation to apply when displaying the frames: a video
+  // recorded in portrait stores landscape frames with a rotation. Pass it as
+  // the `rotation` of importExternalTexture().
+  readonly rotation: 0 | 90 | 180 | 270;
+  // The nominal frame rate of the video track, 0 if unknown.
+  readonly frameRate: number;
+}
+
+// The source info of GPUQueue.copyExternalImageToTexture() for a native
+// frame (a React Native extension of GPUCopyExternalImageSourceInfo). The
+// frame is rendered into the destination through an external texture, which
+// converts YUV to RGB and applies `rotation` and `mirrored` (the extension of
+// importExternalTexture), so the destination needs
+// GPUTextureUsage.RENDER_ATTACHMENT. `origin` and the copy size are in the
+// upright frame, after rotation.
+export interface GPUCopyExternalNativeFrameSourceInfo {
+  source: NativeVideoFrame;
+  origin?: GPUOrigin2D;
+  flipY?: boolean;
+  rotation?: 0 | 90 | 180 | 270;
+  mirrored?: boolean;
 }
 
 export interface CreateVideoPlayerOptions {
-  // 'bgra8' (default): emit a single-plane BGRA surface, suitable for
-  // SharedTextureMemory and a regular sampled GPUTexture.
-  // 'nv12': emit biplanar Y + CbCr surfaces, suitable for
-  // GPUDevice.importExternalTexture.
+  // The layout of the frames on Apple platforms:
+  // 'bgra8' (default): single-plane BGRA surfaces, for importSharedTextureMemory
+  // and a regular sampled GPUTexture.
+  // 'nv12': biplanar Y + CbCr surfaces, for GPUDevice.importExternalTexture.
+  // On Android the decoder always writes its native YUV layout (reported as
+  // 'nv12'), which only importExternalTexture samples; the option is ignored.
   pixelFormat?: NativeVideoPixelFormat;
 }
 
+// A React Native view to draw with GPUQueue.drawElementImageToTexture: a host
+// component ref (`useRef` object), the host instance itself (`ref.current`),
+// or the numeric tag returned by findNodeHandle. On worklet runtimes (no
+// findNodeHandle) pass the instance or the tag.
+export type GPUDrawElementImageSourceView =
+  number | { current: unknown } | React.Component | object;
+
+// The source argument of drawElementImageToTexture, modeled on the
+// html-in-canvas proposal's GPUDrawElementImageSource. The optional rectangle
+// selects the part of the view to draw, in the view's own coordinate space
+// and in density-independent points (React Native layout units); it defaults
+// to the whole view.
+export interface GPUDrawElementImageSource {
+  source: GPUDrawElementImageSourceView;
+  sourceX?: number;
+  sourceY?: number;
+  sourceWidth?: number;
+  sourceHeight?: number;
+}
+
+// The destination argument of drawElementImageToTexture:
+// GPUCopyExternalImageDestInfo plus the pixel size of the region the view is
+// drawn into. `size` defaults to the source rectangle's natural pixel size
+// (points times the device pixel ratio); pass it explicitly to scale the view
+// to the destination region.
+export interface GPUDrawElementImageDestination extends GPUCopyExternalImageDestInfo {
+  size?: GPUExtent3DStrict;
+}
+
 export interface GPUSharedTextureMemoryDescriptor {
   // Raw native handle (IOSurfaceRef on Apple, AHardwareBuffer* on Android),
   // encoded as a BigInt. The caller is responsible for keeping the underlying

package/src/video.ts

@@ -0,0 +1,69 @@
+import type {
+  CreateVideoPlayerOptions,
+  NativeVideoFrame,
+  VideoPlayer,
+} from "./types";
+
+const native = (name: string) => {
+  if (typeof RNWebGPU === "undefined") {
+    throw new Error(
+      `react-native-webgpu is not installed natively; ${name} is unavailable`,
+    );
+  }
+  return RNWebGPU;
+};
+
+/**
+ * Decodes a video into native frames the GPU samples without any copy.
+ *
+ * `source` is a URL (`http`, `https`, `file`) or a file path. Playback is
+ * controlled like an HTMLMediaElement (`play()`, `pause()`, `currentTime`,
+ * `loop`, `volume`); frames are pulled with `copyLatestFrame()` on every
+ * render tick and sampled through `importExternalTexture()` or
+ * `importSharedTextureMemory()`:
+ *
+ * ```ts
+ * const player = createVideoPlayer(url);
+ * player.play();
+ * const render = () => {
+ *   const frame = player.copyLatestFrame();
+ *   if (frame) {
+ *     // import, draw, then release once the frame is no longer sampled
+ *   }
+ *   requestAnimationFrame(render);
+ * };
+ * ```
+ *
+ * On Apple platforms the frames are BGRA by default, or NV12 with
+ * `{ pixelFormat: "nv12" }`. On Android they are always in the decoder's
+ * native YUV layout, for `importExternalTexture()`.
+ */
+export const createVideoPlayer = (
+  source: string,
+  options?: CreateVideoPlayerOptions,
+): VideoPlayer =>
+  native("createVideoPlayer").createVideoPlayer(source, options?.pixelFormat);
+
+/**
+ * Wraps a native buffer produced by another library (a `CVPixelBufferRef` on
+ * Apple platforms, an `AHardwareBuffer*` on Android, for instance
+ * VisionCamera's `frame.getNativeBuffer().pointer`) into a `NativeVideoFrame`
+ * that keeps it alive until released.
+ */
+export const createVideoFrameFromNativeBuffer = (
+  pointer: bigint,
+): NativeVideoFrame =>
+  native("createVideoFrameFromNativeBuffer").createVideoFrameFromNativeBuffer(
+    pointer,
+  );
+
+/**
+ * Allocates a native frame of the given size filled with a test pattern (a
+ * red/green gradient with diagonal stripes in the blue channel), for examples
+ * and tests that need a frame without a camera or a video.
+ */
+export const createTestVideoFrame = (
+  width: number,
+  height: number,
+): NativeVideoFrame =>
+  native("createTestVideoFrame").createTestVideoFrame(width, height);