react-native-webgpu 0.5.11 → 0.5.13

package/README.md

@@ -222,6 +222,90 @@ device.queue.copyExternalImageToTexture(
 );
 ```
 
+### Shared Texture Memory
+
+React Native WebGPU exposes Dawn's `SharedTextureMemory` so you can import a native pixel surface (an `IOSurface`-backed `CVPixelBuffer` on iOS, an `AHardwareBuffer` on Android) as a sampleable `GPUTexture` without copying pixels through the CPU. This is the path you want for camera frames, video frames, or anything coming out of a hardware producer.
+
+Like `importExternalTexture` on the web, this is **enabled by default**, there is nothing to request at device creation. The only thing to check is that the device supports it before importing. It always does on iOS/macOS; it can be missing on some Android drivers and emulators.
+
+```tsx
+import type { NativeVideoFrame } from "react-native-wgpu";
+
+const adapter = await navigator.gpu.requestAdapter();
+const device = await adapter!.requestDevice();
+
+// On by default when supported; this is the only check you need.
+if (!device.features.has("rnwebgpu/native-texture" as GPUFeatureName)) {
+  return; // rare: some Android drivers/emulators can't import native surfaces
+}
+
+// `frame` here is a NativeVideoFrame whose .handle is the native surface
+// (IOSurfaceRef / AHardwareBuffer*). NativeVideoFrames are produced by helpers
+// like RNWebGPU.createVideoPlayer or RNWebGPU.createTestVideoFrame, or by
+// any third-party module that hands you a compatible native pointer.
+const memory = device.importSharedTextureMemory({
+  handle: frame.handle,
+  label: "video-frame",
+});
+const texture = memory.createTexture();
+
+memory.beginAccess(texture, /* initialized */ true);
+// ... bind `texture` into a sampler and render normally ...
+memory.endAccess(texture);
+
+texture.destroy();
+frame.release();
+```
+
+`beginAccess`/`endAccess` bracket the GPU's read window on the shared surface. Pass `initialized: true` when the producer has already written meaningful pixels (the typical video/camera case) and `false` when the next pass will fully overwrite the texture.
+
+### Importing External Textures
+
+`GPUDevice.importExternalTexture` is the higher-level path for sampling a native surface. You hand it a `NativeVideoFrame` and get back a `GPUExternalTexture` that you bind as a `texture_external` and read with `textureSampleBaseClampToEdge`. It does two things for you on top of `SharedTextureMemory`:
+
+- **Color conversion.** Camera and video surfaces are usually biplanar YUV (NV12), not RGB. An external texture carries the YUV→RGB matrix and the source/destination color-space transfer functions, so on the supported paths the sampler returns ready-to-use RGB in hardware. With raw `SharedTextureMemory` you would sample the luma/chroma planes and do that conversion by hand in WGSL. This is the main reason to prefer it for camera and video frames.
+- **Lifecycle.** It owns the `SharedTextureMemory` + `createTexture` + `beginAccess`/`endAccess` sequence internally, so you just import the frame and `destroy()` the result.
+
+It builds on the same default-on capability as Shared Texture Memory above, so feature-detect the device the same way before importing.
+
+> **Android note:** the hardware YUV→RGB conversion is fully automatic on iOS (NV12 `IOSurface`). On Android, camera frames arrive as an _opaque_ YCbCr `AHardwareBuffer`, and Dawn's Vulkan path forces an identity (`RGB_IDENTITY`) sampler conversion, so the external sample comes back as raw `[Y, Cb, Cr]`. You still get the zero-copy import and the rotation/mirror transform, but you need to apply the YUV→RGB matrix yourself in the shader. See the `CAMERA_PRELUDE` in the [VisionCamera example](/apps/example/src/VisionCamera/shaders.ts) for a ready-made BT.709 decode.
+
+```tsx
+const adapter = await navigator.gpu.requestAdapter();
+const device = await adapter!.requestDevice();
+// Feature-detect as shown above before importing on unsupported hardware.
+
+const render = () => {
+  // A GPUExternalTexture expires once the queue work that used it is submitted,
+  // so re-import one every frame.
+  const externalTexture = device.importExternalTexture({
+    source: frame, // a NativeVideoFrame
+    label: "video-frame",
+  });
+
+  const bindGroup = device.createBindGroup({
+    layout: pipeline.getBindGroupLayout(0),
+    entries: [
+      { binding: 0, resource: externalTexture },
+      { binding: 1, resource: sampler },
+    ],
+  });
+
+  // ... encode a pass that samples `externalTexture`, then:
+  device.queue.submit([encoder.finish()]);
+
+  // Release the surface's access window right after the submit that sampled it.
+  externalTexture.destroy();
+  context.present();
+};
+```
+
+Camera frames arrive in the sensor's native orientation, so `importExternalTexture` also accepts non-spec `rotation` (`0` | `90` | `180` | `270`, in degrees) and `mirrored` (horizontal flip) options. Dawn bakes them into the sampling transform, so the shader sees an upright frame. They map directly onto VisionCamera's `frame.orientation` / `frame.isMirrored`.
+
+#### Calling `destroy()`
+
+A `GPUExternalTexture` keeps an open access window on the underlying native surface until the wrapper is destroyed. On the Web `importExternalTexture` is core and the lifetime is handled for you; here the window is tied to the JavaScript object's lifetime. Call `externalTexture.destroy()` right after the `queue.submit()` that sampled it (never before) to release the surface back to its producer immediately. `destroy()` is idempotent, and the surface is also released when the object is garbage-collected, but relying on GC can starve a producer's buffer pool (e.g. an `AVPlayer`'s recycled `IOSurface`s) and pile up GPU resources, so prefer the explicit call in a render loop.
+
 ### Reanimated Integration
 
 React Native WebGPU supports running WebGPU rendering on the UI thread using [React Native Reanimated](https://docs.swmansion.com/react-native-reanimated/) and [React Native Worklets](https://github.com/margelo/react-native-worklets).

package/android/CMakeLists.txt

@@ -37,6 +37,8 @@ add_library(${PACKAGE_NAME} SHARED
     ../cpp/rnwgpu/api/GPUCommandEncoder.cpp
     ../cpp/rnwgpu/api/GPUQuerySet.cpp
     ../cpp/rnwgpu/api/GPUTexture.cpp
+    ../cpp/rnwgpu/api/GPUSharedTextureMemory.cpp
+    ../cpp/rnwgpu/api/GPUExternalTexture.cpp
     ../cpp/rnwgpu/api/GPURenderBundleEncoder.cpp
     ../cpp/rnwgpu/api/GPURenderPassEncoder.cpp
     ../cpp/rnwgpu/api/GPURenderPipeline.cpp

package/android/cpp/AndroidPlatformContext.h

@@ -1,8 +1,10 @@
 #pragma once
 
 #include <android/bitmap.h>
+#include <android/hardware_buffer.h>
 #include <jni.h>
 
+#include <algorithm>
 #include <functional>
 #include <memory>
 #include <string>
@@ -202,6 +204,125 @@ public:
       }
     }).detach();
   }
+
+  VideoFrameHandle loadVideoFrame(const std::string & /*path*/) override {
+    // TODO: implement using MediaExtractor + MediaCodec to decode the first
+    // frame into an AHardwareBuffer-backed Image (Android API 26+).
+    throw std::runtime_error(
+        "loadVideoFrame is not yet implemented on Android. Pass an "
+        "AHardwareBuffer pointer obtained elsewhere (e.g. from "
+        "react-native-vision-camera) directly to "
+        "device.importSharedTextureMemory.");
+  }
+
+  VideoFrameHandle createTestVideoFrame(uint32_t width,
+                                        uint32_t height) override {
+    // Dawn's Android backend already requires API 26+, so AHardwareBuffer_*
+    // symbols are guaranteed to be available at link time on supported builds.
+    AHardwareBuffer_Desc desc = {};
+    desc.width = width;
+    desc.height = height;
+    desc.layers = 1;
+    desc.format = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
+    desc.usage = AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE |
+                 AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY |
+                 AHARDWAREBUFFER_USAGE_CPU_READ_RARELY;
+
+    AHardwareBuffer *buffer = nullptr;
+    int err = AHardwareBuffer_allocate(&desc, &buffer);
+    if (err != 0 || !buffer) {
+      throw std::runtime_error(
+          "createTestVideoFrame: AHardwareBuffer_allocate failed (" +
+          std::to_string(err) + ")");
+    }
+
+    AHardwareBuffer_Desc actualDesc = {};
+    AHardwareBuffer_describe(buffer, &actualDesc);
+    const uint32_t stridePixels = actualDesc.stride;
+
+    void *vaddr = nullptr;
+    int rc = AHardwareBuffer_lock(buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY,
+                                  -1, nullptr, &vaddr);
+    if (rc != 0 || !vaddr) {
+      AHardwareBuffer_release(buffer);
+      throw std::runtime_error(
+          "createTestVideoFrame: AHardwareBuffer_lock failed (" +
+          std::to_string(rc) + ")");
+    }
+
+    // Same procedural pattern as ApplePlatformContext::createTestVideoFrame so
+    // the shared snapshot matches on both platforms. Apple writes BGRA bytes
+    // (CVPixelBuffer is kCVPixelFormatType_32BGRA, sampled as bgra8unorm);
+    // here we write RGBA bytes (AHardwareBuffer is R8G8B8A8_UNORM, sampled as
+    // rgba8unorm). The fragment shader sees the same logical (r, g, b, a).
+    uint8_t *base = static_cast<uint8_t *>(vaddr);
+    const size_t rowBytes = stridePixels * 4;
+    for (uint32_t y = 0; y < height; ++y) {
+      uint8_t *row = base + y * rowBytes;
+      for (uint32_t x = 0; x < width; ++x) {
+        uint8_t r = static_cast<uint8_t>((x * 255) / std::max(width - 1, 1u));
+        uint8_t g = static_cast<uint8_t>((y * 255) / std::max(height - 1, 1u));
+        uint8_t b = static_cast<uint8_t>(((x + y) & 0x20) ? 220 : 30);
+        row[x * 4 + 0] = r;
+        row[x * 4 + 1] = g;
+        row[x * 4 + 2] = b;
+        row[x * 4 + 3] = 0xFF;
+      }
+    }
+    AHardwareBuffer_unlock(buffer, nullptr);
+
+    VideoFrameHandle handle;
+    handle.handle = static_cast<void *>(buffer);
+    handle.width = width;
+    handle.height = height;
+    handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); };
+    return handle;
+  }
+
+  std::unique_ptr<IVideoPlayer>
+  createVideoPlayer(const std::string & /*path*/,
+                    VideoPixelFormat /*format*/) override {
+    // TODO: implement using MediaCodec -> ImageReader (AHardwareBuffer mode).
+    throw std::runtime_error(
+        "createVideoPlayer is not yet implemented on Android.");
+  }
+
+  std::string writeTestVideoFile() override {
+    // TODO: implement using MediaCodec (H.264 encoder) or MediaMuxer.
+    throw std::runtime_error(
+        "writeTestVideoFile is not yet implemented on Android.");
+  }
+
+  VideoFrameHandle wrapNativeBuffer(void *pointer) override {
+    if (!pointer) {
+      throw std::runtime_error("wrapNativeBuffer: pointer is null");
+    }
+    auto *buffer = static_cast<AHardwareBuffer *>(pointer);
+
+    AHardwareBuffer_Desc desc = {};
+    AHardwareBuffer_describe(buffer, &desc);
+
+    AHardwareBuffer_acquire(buffer);
+
+    VideoFrameHandle handle;
+    handle.handle = static_cast<void *>(buffer);
+    handle.width = desc.width;
+    handle.height = desc.height;
+    // YUV / opaque formats route through Vulkan's SamplerYcbcrConversion via
+    // Dawn's OpaqueYCbCrAndroidForExternalTexture path. Single-plane RGBA AHBs
+    // take the plain BGRA8 path (sampled as a regular 2D texture).
+    switch (desc.format) {
+      case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420:
+      case AHARDWAREBUFFER_FORMAT_YCbCr_P010:
+        handle.pixelFormat = VideoPixelFormat::NV12;
+        break;
+      default:
+        handle.pixelFormat = VideoPixelFormat::BGRA8;
+        break;
+    }
+    handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); };
+    return handle;
+  }
 };
 
 } // namespace rnwgpu

package/apple/ApplePlatformContext.h

@@ -26,6 +26,19 @@ public:
   void createImageBitmapFromDataAsync(
       std::span<const uint8_t> data, std::function<void(ImageData)> onSuccess,
       std::function<void(std::string)> onError) override;
+
+  VideoFrameHandle loadVideoFrame(const std::string &path) override;
+
+  VideoFrameHandle createTestVideoFrame(uint32_t width,
+                                        uint32_t height) override;
+
+  std::unique_ptr<IVideoPlayer>
+  createVideoPlayer(const std::string &path,
+                    VideoPixelFormat format) override;
+
+  std::string writeTestVideoFile() override;
+
+  VideoFrameHandle wrapNativeBuffer(void *pointer) override;
 };
 
 } // namespace rnwgpu

package/apple/ApplePlatformContext.mm

@@ -2,10 +2,14 @@
 
 #include <TargetConditionals.h>
 
+#import <AVFoundation/AVFoundation.h>
+#import <CoreVideo/CoreVideo.h>
 #import <React/RCTBlobManager.h>
 #import <React/RCTBridge+Private.h>
 #import <ReactCommon/RCTTurboModule.h>
 
+#include "AppleVideoPlayer.h"
+
 #include "RNWebGPUManager.h"
 #include "WebGPUModule.h"
 
@@ -154,4 +158,145 @@ void ApplePlatformContext::createImageBitmapFromDataAsync(
   });
 }
 
+VideoFrameHandle
+ApplePlatformContext::loadVideoFrame(const std::string &path) {
+  NSString *nsPath = [NSString stringWithUTF8String:path.c_str()];
+  NSURL *url = [nsPath hasPrefix:@"file://"]
+                   ? [NSURL URLWithString:nsPath]
+                   : [NSURL fileURLWithPath:nsPath];
+  AVURLAsset *asset = [AVURLAsset assetWithURL:url];
+
+  NSArray<AVAssetTrack *> *videoTracks =
+      [asset tracksWithMediaType:AVMediaTypeVideo];
+  if (videoTracks.count == 0) {
+    throw std::runtime_error("loadVideoFrame: no video track in file");
+  }
+  AVAssetTrack *videoTrack = videoTracks.firstObject;
+
+  NSError *error = nil;
+  AVAssetReader *reader = [AVAssetReader assetReaderWithAsset:asset
+                                                        error:&error];
+  if (error || !reader) {
+    throw std::runtime_error(
+        std::string("loadVideoFrame: AVAssetReader init failed: ") +
+        [[error localizedDescription] UTF8String]);
+  }
+
+  NSDictionary *outputSettings = @{
+    (NSString *)kCVPixelBufferPixelFormatTypeKey :
+        @(kCVPixelFormatType_32BGRA),
+    (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{},
+    (NSString *)kCVPixelBufferMetalCompatibilityKey : @YES,
+  };
+  AVAssetReaderTrackOutput *output =
+      [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:videoTrack
+                                                 outputSettings:outputSettings];
+  output.alwaysCopiesSampleData = NO;
+  if (![reader canAddOutput:output]) {
+    throw std::runtime_error("loadVideoFrame: cannot add output");
+  }
+  [reader addOutput:output];
+
+  if (![reader startReading]) {
+    throw std::runtime_error(
+        std::string("loadVideoFrame: startReading failed: ") +
+        [[reader.error localizedDescription] UTF8String]);
+  }
+
+  CMSampleBufferRef sampleBuffer = [output copyNextSampleBuffer];
+  if (!sampleBuffer) {
+    throw std::runtime_error("loadVideoFrame: no sample buffer");
+  }
+
+  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+  if (!pixelBuffer) {
+    CFRelease(sampleBuffer);
+    throw std::runtime_error("loadVideoFrame: no pixel buffer");
+  }
+
+  IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer);
+  if (!ioSurface) {
+    CFRelease(sampleBuffer);
+    throw std::runtime_error(
+        "loadVideoFrame: pixel buffer is not IOSurface-backed");
+  }
+
+  // Retain the IOSurface so it survives past the sample buffer's lifetime.
+  CFRetain(ioSurface);
+
+  VideoFrameHandle handle;
+  handle.handle = (void *)ioSurface;
+  handle.width = static_cast<uint32_t>(CVPixelBufferGetWidth(pixelBuffer));
+  handle.height = static_cast<uint32_t>(CVPixelBufferGetHeight(pixelBuffer));
+  handle.deleter = [ioSurface]() { CFRelease(ioSurface); };
+
+  CFRelease(sampleBuffer);
+  [reader cancelReading];
+
+  return handle;
+}
+
+std::unique_ptr<IVideoPlayer>
+ApplePlatformContext::createVideoPlayer(const std::string &path,
+                                         VideoPixelFormat format) {
+  return createAppleVideoPlayer(path, format);
+}
+
+std::string ApplePlatformContext::writeTestVideoFile() {
+  return writeAppleTestVideoFile();
+}
+
+VideoFrameHandle ApplePlatformContext::wrapNativeBuffer(void *pointer) {
+  return wrapCVPixelBuffer(static_cast<CVPixelBufferRef>(pointer));
+}
+
+VideoFrameHandle
+ApplePlatformContext::createTestVideoFrame(uint32_t width, uint32_t height) {
+  NSDictionary *attrs = @{
+    (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{},
+    (NSString *)kCVPixelBufferMetalCompatibilityKey : @YES,
+  };
+  CVPixelBufferRef pixelBuffer = NULL;
+  CVReturn err = CVPixelBufferCreate(
+      kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA,
+      (__bridge CFDictionaryRef)attrs, &pixelBuffer);
+  if (err != kCVReturnSuccess || !pixelBuffer) {
+    throw std::runtime_error("createTestVideoFrame: CVPixelBufferCreate "
+                             "failed");
+  }
+
+  CVPixelBufferLockBaseAddress(pixelBuffer, 0);
+  uint8_t *base =
+      static_cast<uint8_t *>(CVPixelBufferGetBaseAddress(pixelBuffer));
+  size_t rowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer);
+  for (uint32_t y = 0; y < height; ++y) {
+    uint8_t *row = base + y * rowBytes;
+    for (uint32_t x = 0; x < width; ++x) {
+      // RGB gradient + diagonal stripes, in BGRA byte order.
+      uint8_t r = static_cast<uint8_t>((x * 255) / std::max(width - 1, 1u));
+      uint8_t g = static_cast<uint8_t>((y * 255) / std::max(height - 1, 1u));
+      uint8_t b = static_cast<uint8_t>(((x + y) & 0x20) ? 220 : 30);
+      row[x * 4 + 0] = b;
+      row[x * 4 + 1] = g;
+      row[x * 4 + 2] = r;
+      row[x * 4 + 3] = 0xFF;
+    }
+  }
+  CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
+
+  IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer);
+  if (!ioSurface) {
+    CFRelease(pixelBuffer);
+    throw std::runtime_error(
+        "createTestVideoFrame: pixel buffer is not IOSurface-backed");
+  }
+
+  VideoFrameHandle handle;
+  handle.handle = (void *)ioSurface;
+  handle.width = width;
+  handle.height = height;
+  handle.deleter = [pixelBuffer]() { CFRelease(pixelBuffer); };
+  return handle;
+}
+
 } // namespace rnwgpu

package/apple/AppleVideoPlayer.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "PlatformContext.h"
+
+#include <memory>
+#include <string>
+
+#ifdef __OBJC__
+#import <CoreVideo/CoreVideo.h>
+#endif
+
+namespace rnwgpu {
+
+// Factory: creates a new IVideoPlayer backed by AVPlayer +
+// AVPlayerItemVideoOutput. `format` selects the surface layout.
+std::unique_ptr<IVideoPlayer>
+createAppleVideoPlayer(const std::string &path, VideoPixelFormat format);
+
+// Generate a small procedurally-animated test video and write it to a
+// temporary file. Returns the absolute path. Used by the SharedTextureMemory
+// example so it doesn't need a bundled .mp4.
+std::string writeAppleTestVideoFile();
+
+#ifdef __OBJC__
+// Build a VideoFrameHandle from an existing CVPixelBuffer. CFRetains the
+// pixel buffer so the caller can release their reference immediately. Reads
+// IOSurface, dimensions, pixel format, and YUV matrix off the buffer.
+VideoFrameHandle wrapCVPixelBuffer(CVPixelBufferRef pixelBuffer);
+#endif
+
+} // namespace rnwgpu