react-native-webgpu 0.11.0 → 0.11.2

package/cpp/rnwgpu/PlatformContext.h

@@ -59,7 +59,8 @@ struct VideoFrameHandle {
 };
 
 // Platform-implemented video source that hands out fresh IOSurface /
-// AHardwareBuffer-backed frames as a video plays.
+// AHardwareBuffer-backed frames as a video plays, with the playback controls
+// of an HTMLMediaElement (times in seconds).
 class IVideoPlayer {
 public:
   virtual ~IVideoPlayer() = default;
@@ -71,6 +72,26 @@ public:
 
   virtual void play() = 0;
   virtual void pause() = 0;
+  virtual bool paused() = 0;
+
+  // Playback position and length, in seconds. The duration is 0 until the
+  // metadata of the media has loaded.
+  virtual double currentTime() = 0;
+  virtual void seek(double seconds) = 0;
+  virtual double duration() = 0;
+
+  virtual double volume() = 0;
+  virtual void setVolume(double volume) = 0;
+  virtual bool loop() = 0;
+  virtual void setLoop(bool loop) = 0;
+
+  // The geometry of the frames, 0 until the metadata has loaded: their coded
+  // size, the clockwise rotation (0, 90, 180 or 270) to apply when displaying
+  // them, and the nominal frame rate of the video track.
+  virtual uint32_t videoWidth() = 0;
+  virtual uint32_t videoHeight() = 0;
+  virtual int rotation() = 0;
+  virtual double frameRate() = 0;
 };
 
 // A request to rasterize a native React Native view (looked up by its React
@@ -117,10 +138,9 @@ public:
   // pixel, `format` is rgba8unorm or bgra8unorm depending on what the platform
   // rasterizer produces natively, and `premultiplied` reports its alpha
   // representation.
-  virtual void
-  snapshotView(const ViewSnapshotRequest &request,
-               std::function<void(ImageData)> onSuccess,
-               std::function<void(std::string)> onError) = 0;
+  virtual void snapshotView(const ViewSnapshotRequest &request,
+                            std::function<void(ImageData)> onSuccess,
+                            std::function<void(std::string)> onError) = 0;
 
   virtual wgpu::Surface makeSurface(wgpu::Instance instance, void *surface,
                                     int width, int height) = 0;

package/cpp/rnwgpu/api/GPUAdapter.cpp

@@ -25,18 +25,24 @@ namespace rnwgpu {
 async::AsyncTaskHandle GPUAdapter::requestDevice(
     jsi::Runtime &runtime,
     std::optional<std::shared_ptr<GPUDeviceDescriptor>> descriptor) {
-  // Enable the react-native-wgpu "native-texture" umbrella by default, mirroring
-  // the web where importExternalTexture is core and needs no feature request.
-  // We append the umbrella's backing Dawn features to requiredFeatures so the
-  // capability is on without the caller listing it. Two rules keep this safe:
+  // Enable the react-native-wgpu "native-texture" umbrella by default,
+  // mirroring the web where importExternalTexture is core and needs no feature
+  // request. We append the umbrella's backing Dawn features to requiredFeatures
+  // so the capability is on without the caller listing it. Two rules keep this
+  // safe:
   //   - All-or-nothing: only inject when the adapter supports *every* backing
   //     feature (same semantics as maybeSynthesizeRnNativeTextureFeature). On a
-  //     web/fallback adapter the backing set is empty or unsupported, so this is
-  //     a no-op and device creation is unaffected.
+  //     web/fallback adapter the backing set is empty or unsupported, so this
+  //     is a no-op and device creation is unaffected.
   //   - Requesting a feature the adapter doesn't support makes RequestDevice
   //     fail, hence the support check below.
   // Callers can still pass "rnwebgpu/native-texture" explicitly; the dedupe
   // keeps that idempotent.
+  // The optional features (rnNativeTextureOptionalFeatures) ride along one by
+  // one, each only when the adapter supports it: on Android the YUV frames of
+  // the decoder and of the camera cannot be imported without
+  // OpaqueYCbCrAndroidForExternalTexture, while RGBA buffers need nothing
+  // beyond the backing set.
   {
     auto backing = rnNativeTextureBackingFeatures();
     if (!backing.empty()) {
@@ -44,9 +50,10 @@ async::AsyncTaskHandle GPUAdapter::requestDevice(
       _instance.GetFeatures(&supported);
       std::unordered_set<wgpu::FeatureName> supportedSet(
           supported.features, supported.features + supported.featureCount);
-      bool allSupported = std::all_of(
-          backing.begin(), backing.end(),
-          [&](wgpu::FeatureName f) { return supportedSet.count(f) > 0; });
+      bool allSupported =
+          std::all_of(backing.begin(), backing.end(), [&](wgpu::FeatureName f) {
+            return supportedSet.count(f) > 0;
+          });
       if (allSupported) {
         if (!descriptor.has_value()) {
           descriptor = std::make_shared<GPUDeviceDescriptor>();
@@ -62,6 +69,13 @@ async::AsyncTaskHandle GPUAdapter::requestDevice(
             features.push_back(f);
           }
         }
+        for (auto f : rnNativeTextureOptionalFeatures()) {
+          if (supportedSet.count(f) > 0 &&
+              std::find(features.begin(), features.end(), f) ==
+                  features.end()) {
+            features.push_back(f);
+          }
+        }
       }
     }
   }
@@ -101,8 +115,7 @@ async::AsyncTaskHandle GPUAdapter::requestDevice(
         if (std::find(features.begin(), features.end(),
                       wgpu::FeatureName::ImplicitDeviceSynchronization) ==
             features.end()) {
-          features.push_back(
-              wgpu::FeatureName::ImplicitDeviceSynchronization);
+          features.push_back(wgpu::FeatureName::ImplicitDeviceSynchronization);
         }
       }
     }
@@ -211,7 +224,8 @@ async::AsyncTaskHandle GPUAdapter::requestDevice(
             }
           }
         }
-// TODO: in the latest version of Dawn, this won't be needed (https://issues.chromium.org/issues/42241591)
+// TODO: in the latest version of Dawn, this won't be needed
+// (https://issues.chromium.org/issues/42241591)
 #if defined(TARGET_OS_SIMULATOR) && TARGET_OS_SIMULATOR
         // The iOS Simulator only advertises MTLFeatureSet_iOS_GPUFamily2, so
         // Dawn defaults disable_base_instance/disable_base_vertex on and then

package/cpp/rnwgpu/api/GPUDevice.cpp

@@ -133,7 +133,8 @@ std::shared_ptr<GPUSupportedLimits> GPUDevice::getLimits() {
 
 std::shared_ptr<GPUQueue> GPUDevice::getQueue() {
   auto result = _instance.GetQueue();
-  return std::make_shared<GPUQueue>(result, _async, _label);
+  return std::make_shared<GPUQueue>(_instance, result, _async, _videoFrameBlit,
+                                    _label);
 }
 
 std::shared_ptr<GPUCommandEncoder> GPUDevice::createCommandEncoder(

package/cpp/rnwgpu/api/GPUDevice.h

@@ -54,6 +54,7 @@
 #include "GPUTexture.h"
 #include "GPUTextureDescriptor.h"
 #include "GPUUncapturedErrorEvent.h"
+#include "VideoFrameBlit.h"
 
 namespace rnwgpu {
 
@@ -285,6 +286,11 @@ private:
   wgpu::Device _instance;
   std::shared_ptr<async::RuntimeContext> _async;
   std::string _label;
+  // Renders native video frames into textures for
+  // GPUQueue.copyExternalImageToTexture; shared by every GPUQueue wrapper of
+  // this device so the pipelines are created once.
+  std::shared_ptr<VideoFrameBlit> _videoFrameBlit =
+      std::make_shared<VideoFrameBlit>();
   // Guards the device-lost state below. getLost() runs on a JS thread, but
   // Dawn's AllowSpontaneous device-lost callback (and device destruction) can
   // fire notifyDeviceLost() from other threads, so the mutex keeps these

package/cpp/rnwgpu/api/GPUExternalTexture.cpp

@@ -8,8 +8,47 @@
 
 #include "GPUExternalTextureDescriptor.h"
 
+#if defined(__ANDROID__)
+#include <android/hardware_buffer.h>
+#endif
+
 namespace rnwgpu {
 
+#if defined(__ANDROID__)
+// Whether Dawn imports an AHardwareBuffer of this format as an
+// OpaqueYCbCrAndroid texture. Mirrors FormatFromAHardwareBufferFormat
+// (dawn/native/AHBFunctions.cpp): the color and depth formats with a WebGPU
+// equivalent import as that format; the YUV formats and the formats outside
+// the public enum, which drivers use for the camera and the video decoder,
+// import as OpaqueYCbCrAndroid.
+static bool isOpaqueYCbCrFormat(uint32_t format) {
+  switch (format) {
+  case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM:
+  case AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM:
+  case AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM:
+  case AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM:
+  case AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT:
+  case AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM:
+  case AHARDWAREBUFFER_FORMAT_BLOB:
+  case AHARDWAREBUFFER_FORMAT_D16_UNORM:
+  case AHARDWAREBUFFER_FORMAT_D24_UNORM:
+  case AHARDWAREBUFFER_FORMAT_D24_UNORM_S8_UINT:
+  case AHARDWAREBUFFER_FORMAT_D32_FLOAT:
+  case AHARDWAREBUFFER_FORMAT_D32_FLOAT_S8_UINT:
+  case AHARDWAREBUFFER_FORMAT_S8_UINT:
+  // Single-channel formats of the API 33 and 34 headers, by value so that
+  // older NDKs compile.
+  case 0x38: // AHARDWAREBUFFER_FORMAT_R8_UNORM
+  case 0x39: // AHARDWAREBUFFER_FORMAT_R16_UINT
+  case 0x3a: // AHARDWAREBUFFER_FORMAT_R16G16_UINT
+  case 0x3b: // AHARDWAREBUFFER_FORMAT_R10G10B10A10_UNORM
+    return false;
+  default:
+    return true;
+  }
+}
+#endif
+
 // Identity gamut (BT.709 -> sRGB, same primaries) as a 3x3 column-major matrix.
 static const float kIdentityGamutMatrix[9] = {
     1.0f, 0.0f, 0.0f, //
@@ -254,6 +293,28 @@ std::shared_ptr<GPUExternalTexture> GPUExternalTexture::Create(
       // sampled texel is already RGB.
       kYuvPassthroughMatrix);
 #elif defined(__ANDROID__)
+  // 0. A YUV frame imports as an OpaqueYCbCrAndroid texture, which Dawn only
+  //    accepts on a device with OpaqueYCbCrAndroidForExternalTexture.
+  //    requestDevice() enables the feature whenever the adapter supports it;
+  //    a device created by another library (importDevice) has to request it
+  //    itself. Fail with the cause here: Dawn's own report is a bare
+  //    "Unsupported texture format TextureFormat::OpaqueYCbCrAndroid" device
+  //    error, followed by the failure of every call made with the result.
+  if (!device.HasFeature(
+          wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture)) {
+    AHardwareBuffer_Desc ahbDesc{};
+    AHardwareBuffer_describe(static_cast<AHardwareBuffer *>(frame.handle),
+                             &ahbDesc);
+    if (isOpaqueYCbCrFormat(ahbDesc.format)) {
+      throw std::runtime_error(
+          "GPUExternalTexture::Create(): a YUV frame can only be imported on "
+          "a device with the 'opaque-ycbcr-android-for-external-texture' "
+          "feature. requestDevice() enables it whenever the adapter supports "
+          "it; a device imported from another library has to request it "
+          "itself.");
+    }
+  }
+
   // 1. Import the AHardwareBuffer as SharedTextureMemory. For YUV AHBs this
   //    yields a Dawn texture in the implementation-defined OpaqueYCbCrAndroid
   //    format; for RGBA AHBs, a regular single-plane texture.

package/cpp/rnwgpu/api/GPUQueue.cpp

@@ -109,6 +109,68 @@ async::AsyncTaskHandle GPUQueue::onSubmittedWorkDone(jsi::Runtime &runtime) {
       });
 }
 
+void GPUQueue::copyVideoFrameToTexture(
+    const GPUImageCopyExternalImage &source,
+    const GPUImageCopyTextureTagged &destination, const wgpu::Extent3D &size) {
+  if (!destination.texture) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: destination.texture is required");
+  }
+  if (destination.aspect.value_or(wgpu::TextureAspect::All) !=
+      wgpu::TextureAspect::All) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: a native frame can only be copied into "
+        "the color aspect");
+  }
+  if (size.depthOrArrayLayers > 1) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: a native frame is a single 2D image");
+  }
+  constexpr double kMaxCoordinate =
+      static_cast<double>(std::numeric_limits<uint32_t>::max());
+  auto coordinate = [&](double value, const char *name) {
+    if (!(value >= 0) || value > kMaxCoordinate) {
+      throw std::runtime_error(std::string("copyExternalImageToTexture: ") +
+                               name +
+                               " must be a non-negative integer coordinate");
+    }
+    return static_cast<uint32_t>(value);
+  };
+  const double rotation = source.rotation.value_or(0);
+  if (rotation != 0 && rotation != 90 && rotation != 180 && rotation != 270) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: rotation must be 0, 90, 180 or 270");
+  }
+
+  VideoFrameBlit::Request request;
+  request.frame = source.videoFrame;
+  if (const auto origin = source.origin.value_or(nullptr)) {
+    request.originX = coordinate(origin->x, "origin.x");
+    request.originY = coordinate(origin->y, "origin.y");
+  }
+  request.width = size.width;
+  request.height = size.height;
+  request.flipY = source.flipY.value_or(false);
+  request.rotation = rotation;
+  request.mirrored = source.mirrored.value_or(false);
+  request.texture = destination.texture->get();
+  request.mipLevel = coordinate(destination.mipLevel.value_or(0), "mipLevel");
+  wgpu::Origin3D dstOrigin{};
+  Convertor conv;
+  if (!conv(dstOrigin, destination.origin)) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: invalid destination origin");
+  }
+  if (dstOrigin.z != 0) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: a native frame can only be copied into "
+        "the first layer of the texture");
+  }
+  request.dstX = dstOrigin.x;
+  request.dstY = dstOrigin.y;
+  _videoFrameBlit->copy(_device, _instance, request);
+}
+
 void GPUQueue::copyExternalImageToTexture(
     std::shared_ptr<GPUImageCopyExternalImage> source,
     std::shared_ptr<GPUImageCopyTextureTagged> destination,
@@ -117,6 +179,17 @@ void GPUQueue::copyExternalImageToTexture(
   wgpu::TexelCopyBufferLayout layout{};
   wgpu::Extent3D sz{};
   Convertor conv;
+  if (source->videoFrame) {
+    if (!conv(sz, size)) {
+      throw std::runtime_error("copyExternalImageToTexture: invalid copy size");
+    }
+    copyVideoFrameToTexture(*source, *destination, sz);
+    return;
+  }
+  if (!source->source) {
+    throw std::runtime_error("copyExternalImageToTexture: the source must be "
+                             "an ImageBitmap or a NativeVideoFrame");
+  }
   uint32_t bytesPerPixel =
       source->source->getSize() /
       (source->source->getWidth() * source->source->getHeight());
@@ -145,10 +218,8 @@ void GPUQueue::copyExternalImageToTexture(
     throw std::runtime_error(
         "The source origin must be a non-negative integer coordinate.");
   }
-  const size_t sourceOriginX =
-      origin ? static_cast<size_t>(origin->x) : 0;
-  const size_t sourceOriginY =
-      origin ? static_cast<size_t>(origin->y) : 0;
+  const size_t sourceOriginX = origin ? static_cast<size_t>(origin->x) : 0;
+  const size_t sourceOriginY = origin ? static_cast<size_t>(origin->y) : 0;
   if (sourceOriginX > source->source->getWidth() ||
       sz.width > source->source->getWidth() - sourceOriginX ||
       sourceOriginY > source->source->getHeight() ||
@@ -220,8 +291,7 @@ async::AsyncTaskHandle GPUQueue::drawElementImageToTexture(
   Convertor conv;
   if (!conv(dst.aspect, info->aspect) || !conv(dst.mipLevel, info->mipLevel) ||
       !conv(dst.origin, info->origin) || !conv(dst.texture, info->texture)) {
-    throw std::runtime_error(
-        "drawElementImageToTexture: invalid destination");
+    throw std::runtime_error("drawElementImageToTexture: invalid destination");
   }
   const wgpu::Texture texture = info->texture->get();
   bool destinationIsBgra = false;
@@ -335,10 +405,10 @@ async::AsyncTaskHandle GPUQueue::drawElementImageToTexture(
                        "pixels (is it mounted with a non-zero size?)");
                 return;
               }
-              if (image.width > mipWidth - std::min<uint32_t>(dst.origin.x,
-                                                              mipWidth) ||
-                  image.height > mipHeight - std::min<uint32_t>(dst.origin.y,
-                                                                mipHeight)) {
+              if (image.width >
+                      mipWidth - std::min<uint32_t>(dst.origin.x, mipWidth) ||
+                  image.height >
+                      mipHeight - std::min<uint32_t>(dst.origin.y, mipHeight)) {
                 reject("drawElementImageToTexture: the view's natural pixel "
                        "size (" +
                        std::to_string(image.width) + "x" +

package/cpp/rnwgpu/api/GPUQueue.h

@@ -20,6 +20,7 @@
 #include "GPUDrawElementImageSource.h"
 #include "GPUImageCopyExternalImage.h"
 #include "GPUImageCopyTextureTagged.h"
+#include "VideoFrameBlit.h"
 
 namespace rnwgpu {
 
@@ -29,10 +30,12 @@ class GPUQueue : public NativeObject<GPUQueue> {
 public:
   static constexpr const char *CLASS_NAME = "GPUQueue";
 
-  explicit GPUQueue(wgpu::Queue instance,
+  explicit GPUQueue(wgpu::Device device, wgpu::Queue instance,
                     std::shared_ptr<async::RuntimeContext> async,
+                    std::shared_ptr<VideoFrameBlit> videoFrameBlit,
                     std::string label)
-      : NativeObject(CLASS_NAME), _instance(instance), _async(async),
+      : NativeObject(CLASS_NAME), _device(device), _instance(instance),
+        _async(async), _videoFrameBlit(std::move(videoFrameBlit)),
         _label(label) {}
 
 public:
@@ -48,6 +51,9 @@ public:
                     std::shared_ptr<ArrayBuffer> data,
                     std::shared_ptr<GPUImageDataLayout> dataLayout,
                     std::shared_ptr<GPUExtent3D> size);
+  // The source is an ImageBitmap (uploaded from the CPU) or, as a React
+  // Native extension, a NativeVideoFrame rendered into the destination through
+  // an external texture (see VideoFrameBlit).
   void copyExternalImageToTexture(
       std::shared_ptr<GPUImageCopyExternalImage> source,
       std::shared_ptr<GPUImageCopyTextureTagged> destination,
@@ -61,8 +67,7 @@ public:
   // runtime's thread right before the promise resolves, so work submitted
   // after `await` observes the new contents.
   async::AsyncTaskHandle drawElementImageToTexture(
-      jsi::Runtime &runtime,
-      std::shared_ptr<GPUDrawElementImageSource> source,
+      jsi::Runtime &runtime, std::shared_ptr<GPUDrawElementImageSource> source,
       std::shared_ptr<GPUDrawElementImageDestination> destination);
 
   std::string getLabel() { return _label; }
@@ -89,8 +94,14 @@ public:
   inline const wgpu::Queue get() { return _instance; }
 
 private:
+  void copyVideoFrameToTexture(const GPUImageCopyExternalImage &source,
+                               const GPUImageCopyTextureTagged &destination,
+                               const wgpu::Extent3D &size);
+
+  wgpu::Device _device;
   wgpu::Queue _instance;
   std::shared_ptr<async::RuntimeContext> _async;
+  std::shared_ptr<VideoFrameBlit> _videoFrameBlit;
   std::string _label;
 };
 

package/cpp/rnwgpu/api/RnFeatures.h

@@ -31,11 +31,24 @@ inline std::vector<wgpu::FeatureName> rnNativeTextureBackingFeatures() {
 #endif
 }
 
+// Dawn features that complete the umbrella on the current platform. They are
+// requested by default when the adapter supports them, but they are not part
+// of the all-or-nothing backing set: on Android, Dawn imports the YUV buffers
+// of the video decoder and of the camera as OpaqueYCbCrAndroid textures, a
+// format it only accepts, and only samples as an external texture, with
+// OpaqueYCbCrAndroidForExternalTexture; RGBA buffers import without it.
+inline std::vector<wgpu::FeatureName> rnNativeTextureOptionalFeatures() {
+#if defined(__ANDROID__)
+  return {wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture};
+#else
+  return {};
+#endif
+}
+
 // If every Dawn feature backing the umbrella is in `enabled`, add the
 // umbrella name to `out`. Used by adapter.features / device.features so JS
 // callers can see (and call .has on) the same name they pass in.
-inline void
-maybeSynthesizeRnNativeTextureFeature(
+inline void maybeSynthesizeRnNativeTextureFeature(
     const std::unordered_set<wgpu::FeatureName> &enabled,
     std::unordered_set<std::string> &out) {
   auto backing = rnNativeTextureBackingFeatures();