react-native-webgpu 0.11.0 → 0.11.1

package/lib/typescript/src/video.d.ts

@@ -0,0 +1,41 @@
+import type { CreateVideoPlayerOptions, NativeVideoFrame, VideoPlayer } from "./types";
+/**
+ * Decodes a video into native frames the GPU samples without any copy.
+ *
+ * `source` is a URL (`http`, `https`, `file`) or a file path. Playback is
+ * controlled like an HTMLMediaElement (`play()`, `pause()`, `currentTime`,
+ * `loop`, `volume`); frames are pulled with `copyLatestFrame()` on every
+ * render tick and sampled through `importExternalTexture()` or
+ * `importSharedTextureMemory()`:
+ *
+ * ```ts
+ * const player = createVideoPlayer(url);
+ * player.play();
+ * const render = () => {
+ *   const frame = player.copyLatestFrame();
+ *   if (frame) {
+ *     // import, draw, then release once the frame is no longer sampled
+ *   }
+ *   requestAnimationFrame(render);
+ * };
+ * ```
+ *
+ * On Apple platforms the frames are BGRA by default, or NV12 with
+ * `{ pixelFormat: "nv12" }`. On Android they are always in the decoder's
+ * native YUV layout, for `importExternalTexture()`.
+ */
+export declare const createVideoPlayer: (source: string, options?: CreateVideoPlayerOptions) => VideoPlayer;
+/**
+ * Wraps a native buffer produced by another library (a `CVPixelBufferRef` on
+ * Apple platforms, an `AHardwareBuffer*` on Android, for instance
+ * VisionCamera's `frame.getNativeBuffer().pointer`) into a `NativeVideoFrame`
+ * that keeps it alive until released.
+ */
+export declare const createVideoFrameFromNativeBuffer: (pointer: bigint) => NativeVideoFrame;
+/**
+ * Allocates a native frame of the given size filled with a test pattern (a
+ * red/green gradient with diagonal stripes in the blue channel), for examples
+ * and tests that need a frame without a camera or a video.
+ */
+export declare const createTestVideoFrame: (width: number, height: number) => NativeVideoFrame;
+//# sourceMappingURL=video.d.ts.map

package/lib/typescript/src/video.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"video.d.ts","sourceRoot":"","sources":["../../../src/video.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,wBAAwB,EACxB,gBAAgB,EAChB,WAAW,EACZ,MAAM,SAAS,CAAC;AAWjB;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,iBAAiB,GAC5B,QAAQ,MAAM,EACd,UAAU,wBAAwB,KACjC,WAC0E,CAAC;AAE9E;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,GAC3C,SAAS,MAAM,KACd,gBAGA,CAAC;AAEJ;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,GAC/B,OAAO,MAAM,EACb,QAAQ,MAAM,KACb,gBACiE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-webgpu",
-  "version": "0.11.0",
+  "version": "0.11.1",
   "description": "React Native WebGPU",
   "main": "lib/commonjs/index",
   "module": "lib/module/index",

package/src/importDevice.ts

@@ -26,10 +26,10 @@ export const importDevice = (pointer: bigint): GPUDevice => {
  * ownership of one reference: the returned texture releases it when destroyed.
  *
  * Pair with producers that hand out a +1 reference, such as
- * `Skia.Image.MakeNativeTextureFromImage()`:
+ * `Skia.Image.MakeGPUTextureFromImage()`:
  *
  * ```ts
- * const texture = adoptTexture(Skia.Image.MakeNativeTextureFromImage(image));
+ * const texture = adoptTexture(Skia.Image.MakeGPUTextureFromImage(image));
  * ```
  */
 export const adoptTexture = (pointer: bigint): GPUTexture => {

package/src/index.tsx

@@ -5,8 +5,10 @@ import type {
   GPUSharedFence,
   GPUSharedFenceDescriptor,
   GPUDawnTogglesDescriptor,
+  GPUNativeExternalTextureDescriptor,
   GPUSharedTextureMemory,
   GPUSharedTextureMemoryDescriptor,
+  GPUCopyExternalNativeFrameSourceInfo,
   NativeCanvas,
   RNCanvasContext,
   VideoPlayer,
@@ -30,7 +32,9 @@ export type {
   GPUSharedTextureMemoryEndAccessState,
   NativeVideoPixelFormat,
   CreateVideoPlayerOptions,
+  GPUCopyExternalNativeFrameSourceInfo,
   GPUDawnTogglesDescriptor,
+  GPUNativeExternalTextureDescriptor,
   GPUDrawElementImageDestination,
   GPUDrawElementImageSource,
   GPUDrawElementImageSourceView,
@@ -69,10 +73,11 @@ declare global {
     importDevice: (pointer: bigint) => GPUDevice;
     // Wrap an externally created WGPUTexture pointer into a GPUTexture,
     // taking ownership of one reference (pair with producers that return a
-    // +1 pointer, e.g. Skia.Image.MakeNativeTextureFromImage()).
+    // +1 pointer, e.g. Skia.Image.MakeGPUTextureFromImage()).
     adoptTexture: (pointer: bigint) => GPUTexture;
+    // Prefer the createVideoPlayer() export, which takes an options object.
     createVideoPlayer: (
-      path: string,
+      source: string,
       pixelFormat?: NativeVideoPixelFormat,
     ) => VideoPlayer;
     writeTestVideoFile: () => string;
@@ -80,12 +85,20 @@ declare global {
 
   interface GPUTexture {
     // Non-spec RN extension: raw WGPUTexture handle as a BigInt for
-    // pointer-based interop (e.g. Skia.Image.MakeImageFromNativeTexture).
+    // pointer-based interop (Skia.Image.MakeImageFromGPUTexture reads it).
     // Borrowed: keep this GPUTexture alive while the pointer is in use.
     readonly nativePointer: bigint;
   }
 
   interface GPUDevice {
+    // Non-spec RN extension: the source of an external texture is a
+    // NativeVideoFrame (from a VideoPlayer, createVideoFrameFromNativeBuffer
+    // or createTestVideoFrame) instead of the web's HTMLVideoElement /
+    // VideoFrame. Overloads the spec signature, which stays available for
+    // the web implementation.
+    importExternalTexture(
+      descriptor: GPUNativeExternalTextureDescriptor,
+    ): GPUExternalTexture;
     importSharedTextureMemory(
       descriptor: GPUSharedTextureMemoryDescriptor,
     ): GPUSharedTextureMemory;
@@ -114,6 +127,18 @@ declare global {
       source: GPUDrawElementImageSource,
       destination: GPUDrawElementImageDestination,
     ): Promise<void>;
+    // Non-spec extension: a NativeVideoFrame (from createVideoPlayer,
+    // createVideoFrameFromNativeBuffer or createTestVideoFrame) as the copy
+    // source. The frame is rendered into the destination on the GPU through
+    // an external texture, which converts YUV to RGB and applies `rotation`
+    // and `mirrored`; the destination needs GPUTextureUsage.RENDER_ATTACHMENT
+    // and an rgba8unorm, bgra8unorm (or -srgb), rgba16float or rgb10a2unorm
+    // format. The frame can be released as soon as the call returns.
+    copyExternalImageToTexture(
+      source: GPUCopyExternalNativeFrameSourceInfo,
+      destination: GPUCopyExternalImageDestInfo,
+      copySize: GPUExtent3DStrict,
+    ): undefined;
   }
 
   // Non-standard, Dawn-only. Lets callers set Dawn device-stage toggles at

package/src/main/index.tsx

@@ -56,6 +56,7 @@ export * from "../WebGPUViewNativeComponent";
 export * from "../hooks";
 export * from "../GPUDeviceProvider";
 export * from "../importDevice";
+export * from "../video";
 export * from "../formats";
 
 export { default as WebGPUModule } from "../NativeWebGPUModule";

package/src/main/index.web.tsx

@@ -6,5 +6,6 @@ export * from "../WebGPUViewNativeComponent";
 export * from "../hooks";
 export * from "../GPUDeviceProvider";
 export * from "../formats";
+export * from "../video";
 
 // We don't need to set all global properties on web, webgpu is already available globally

package/src/types.ts

@@ -63,21 +63,74 @@ export interface NativeVideoFrame {
   release(): void;
 }
 
-// A handle to a decoded video stream. Poll copyLatestFrame() each render tick
-// to obtain the most recently decoded frame as an IOSurface/AHardwareBuffer
-// (returns null between frames so callers can skip the import work).
+// GPUExternalTextureDescriptor with a NativeVideoFrame as the source: what
+// GPUDevice.importExternalTexture() takes on React Native, where there is no
+// HTMLVideoElement or WebCodecs VideoFrame. The other members (label,
+// colorSpace, and the rotation / mirrored extensions) are the same.
+export interface GPUNativeExternalTextureDescriptor extends Omit<
+  GPUExternalTextureDescriptor,
+  "source"
+> {
+  source: NativeVideoFrame;
+}
+
+// A decoded video, modeled on HTMLMediaElement: play(), pause(), paused,
+// currentTime (in seconds, assign it to seek), duration, volume and loop work
+// as on the web. Frames are pulled instead of displayed: poll
+// copyLatestFrame() on every render tick to obtain the most recently decoded
+// frame as an IOSurface/AHardwareBuffer-backed NativeVideoFrame (null between
+// frames, so callers can skip the import work).
 export interface VideoPlayer {
   copyLatestFrame(): NativeVideoFrame | null;
   play(): void;
   pause(): void;
+  // Stops playback and releases the decoder. Frames already copied stay
+  // valid until they are released.
   release(): void;
+  readonly paused: boolean;
+  // The playback position in seconds. Assign it to seek, also while paused:
+  // the next copyLatestFrame() then returns the frame at the new position.
+  currentTime: number;
+  // The length in seconds, 0 until the metadata has loaded.
+  readonly duration: number;
+  // 0 to 1.
+  volume: number;
+  // Restart from the beginning at the end of the stream. Unlike on the web,
+  // it defaults to true.
+  loop: boolean;
+  // The coded size of the frames in pixels, 0 until the metadata has loaded.
+  readonly videoWidth: number;
+  readonly videoHeight: number;
+  // The clockwise rotation to apply when displaying the frames: a video
+  // recorded in portrait stores landscape frames with a rotation. Pass it as
+  // the `rotation` of importExternalTexture().
+  readonly rotation: 0 | 90 | 180 | 270;
+  // The nominal frame rate of the video track, 0 if unknown.
+  readonly frameRate: number;
+}
+
+// The source info of GPUQueue.copyExternalImageToTexture() for a native
+// frame (a React Native extension of GPUCopyExternalImageSourceInfo). The
+// frame is rendered into the destination through an external texture, which
+// converts YUV to RGB and applies `rotation` and `mirrored` (the extension of
+// importExternalTexture), so the destination needs
+// GPUTextureUsage.RENDER_ATTACHMENT. `origin` and the copy size are in the
+// upright frame, after rotation.
+export interface GPUCopyExternalNativeFrameSourceInfo {
+  source: NativeVideoFrame;
+  origin?: GPUOrigin2D;
+  flipY?: boolean;
+  rotation?: 0 | 90 | 180 | 270;
+  mirrored?: boolean;
 }
 
 export interface CreateVideoPlayerOptions {
-  // 'bgra8' (default): emit a single-plane BGRA surface, suitable for
-  // SharedTextureMemory and a regular sampled GPUTexture.
-  // 'nv12': emit biplanar Y + CbCr surfaces, suitable for
-  // GPUDevice.importExternalTexture.
+  // The layout of the frames on Apple platforms:
+  // 'bgra8' (default): single-plane BGRA surfaces, for importSharedTextureMemory
+  // and a regular sampled GPUTexture.
+  // 'nv12': biplanar Y + CbCr surfaces, for GPUDevice.importExternalTexture.
+  // On Android the decoder always writes its native YUV layout (reported as
+  // 'nv12'), which only importExternalTexture samples; the option is ignored.
   pixelFormat?: NativeVideoPixelFormat;
 }
 

package/src/video.ts

@@ -0,0 +1,69 @@
+import type {
+  CreateVideoPlayerOptions,
+  NativeVideoFrame,
+  VideoPlayer,
+} from "./types";
+
+const native = (name: string) => {
+  if (typeof RNWebGPU === "undefined") {
+    throw new Error(
+      `react-native-webgpu is not installed natively; ${name} is unavailable`,
+    );
+  }
+  return RNWebGPU;
+};
+
+/**
+ * Decodes a video into native frames the GPU samples without any copy.
+ *
+ * `source` is a URL (`http`, `https`, `file`) or a file path. Playback is
+ * controlled like an HTMLMediaElement (`play()`, `pause()`, `currentTime`,
+ * `loop`, `volume`); frames are pulled with `copyLatestFrame()` on every
+ * render tick and sampled through `importExternalTexture()` or
+ * `importSharedTextureMemory()`:
+ *
+ * ```ts
+ * const player = createVideoPlayer(url);
+ * player.play();
+ * const render = () => {
+ *   const frame = player.copyLatestFrame();
+ *   if (frame) {
+ *     // import, draw, then release once the frame is no longer sampled
+ *   }
+ *   requestAnimationFrame(render);
+ * };
+ * ```
+ *
+ * On Apple platforms the frames are BGRA by default, or NV12 with
+ * `{ pixelFormat: "nv12" }`. On Android they are always in the decoder's
+ * native YUV layout, for `importExternalTexture()`.
+ */
+export const createVideoPlayer = (
+  source: string,
+  options?: CreateVideoPlayerOptions,
+): VideoPlayer =>
+  native("createVideoPlayer").createVideoPlayer(source, options?.pixelFormat);
+
+/**
+ * Wraps a native buffer produced by another library (a `CVPixelBufferRef` on
+ * Apple platforms, an `AHardwareBuffer*` on Android, for instance
+ * VisionCamera's `frame.getNativeBuffer().pointer`) into a `NativeVideoFrame`
+ * that keeps it alive until released.
+ */
+export const createVideoFrameFromNativeBuffer = (
+  pointer: bigint,
+): NativeVideoFrame =>
+  native("createVideoFrameFromNativeBuffer").createVideoFrameFromNativeBuffer(
+    pointer,
+  );
+
+/**
+ * Allocates a native frame of the given size filled with a test pattern (a
+ * red/green gradient with diagonal stripes in the blue channel), for examples
+ * and tests that need a frame without a camera or a video.
+ */
+export const createTestVideoFrame = (
+  width: number,
+  height: number,
+): NativeVideoFrame =>
+  native("createTestVideoFrame").createTestVideoFrame(width, height);