react-native-webgpu 0.10.5 → 0.11.1

package/cpp/rnwgpu/RNWebGPUManager.cpp

@@ -67,6 +67,10 @@ RNWebGPUManager::RNWebGPUManager(
   // no invoker (best-effort; see README "Threading model").
   async::RuntimeContext::registerMainRuntime(_jsRuntime, _jsCallInvoker);
 
+  // Expose the platform context to API objects created later without a
+  // reference to it (GPUQueue::drawElementImageToTexture).
+  PlatformContext::setCurrent(_platformContext);
+
   auto gpu = std::make_shared<GPU>(*_jsRuntime);
   auto rnWebGPU =
       std::make_shared<RNWebGPU>(gpu, _platformContext, _jsCallInvoker);
@@ -248,6 +252,9 @@ RNWebGPUManager::~RNWebGPUManager() {
   // contextId counter, and stale entries would alias new canvases onto dead
   // surfaces.
   SurfaceRegistry::getInstance().clear();
+  if (PlatformContext::current() == _platformContext) {
+    PlatformContext::setCurrent(nullptr);
+  }
   _jsRuntime = nullptr;
   _jsCallInvoker = nullptr;
 }

package/cpp/rnwgpu/api/GPUDevice.cpp

@@ -133,7 +133,8 @@ std::shared_ptr<GPUSupportedLimits> GPUDevice::getLimits() {
 
 std::shared_ptr<GPUQueue> GPUDevice::getQueue() {
   auto result = _instance.GetQueue();
-  return std::make_shared<GPUQueue>(result, _async, _label);
+  return std::make_shared<GPUQueue>(_instance, result, _async, _videoFrameBlit,
+                                    _label);
 }
 
 std::shared_ptr<GPUCommandEncoder> GPUDevice::createCommandEncoder(

package/cpp/rnwgpu/api/GPUDevice.h

@@ -54,6 +54,7 @@
 #include "GPUTexture.h"
 #include "GPUTextureDescriptor.h"
 #include "GPUUncapturedErrorEvent.h"
+#include "VideoFrameBlit.h"
 
 namespace rnwgpu {
 
@@ -285,6 +286,11 @@ private:
   wgpu::Device _instance;
   std::shared_ptr<async::RuntimeContext> _async;
   std::string _label;
+  // Renders native video frames into textures for
+  // GPUQueue.copyExternalImageToTexture; shared by every GPUQueue wrapper of
+  // this device so the pipelines are created once.
+  std::shared_ptr<VideoFrameBlit> _videoFrameBlit =
+      std::make_shared<VideoFrameBlit>();
   // Guards the device-lost state below. getLost() runs on a JS thread, but
   // Dawn's AllowSpontaneous device-lost callback (and device destruction) can
   // fire notifyDeviceLost() from other threads, so the mutex keeps these

package/cpp/rnwgpu/api/GPUQueue.cpp

@@ -1,10 +1,16 @@
 #include "GPUQueue.h"
 
+#include <algorithm>
+#include <cmath>
 #include <limits>
 #include <memory>
+#include <string>
+#include <utility>
 #include <vector>
 
 #include "Convertors.h"
+#include "ImageBitmap.h"
+#include "PlatformContext.h"
 
 namespace rnwgpu {
 
@@ -103,6 +109,68 @@ async::AsyncTaskHandle GPUQueue::onSubmittedWorkDone(jsi::Runtime &runtime) {
       });
 }
 
+void GPUQueue::copyVideoFrameToTexture(
+    const GPUImageCopyExternalImage &source,
+    const GPUImageCopyTextureTagged &destination, const wgpu::Extent3D &size) {
+  if (!destination.texture) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: destination.texture is required");
+  }
+  if (destination.aspect.value_or(wgpu::TextureAspect::All) !=
+      wgpu::TextureAspect::All) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: a native frame can only be copied into "
+        "the color aspect");
+  }
+  if (size.depthOrArrayLayers > 1) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: a native frame is a single 2D image");
+  }
+  constexpr double kMaxCoordinate =
+      static_cast<double>(std::numeric_limits<uint32_t>::max());
+  auto coordinate = [&](double value, const char *name) {
+    if (!(value >= 0) || value > kMaxCoordinate) {
+      throw std::runtime_error(std::string("copyExternalImageToTexture: ") +
+                               name +
+                               " must be a non-negative integer coordinate");
+    }
+    return static_cast<uint32_t>(value);
+  };
+  const double rotation = source.rotation.value_or(0);
+  if (rotation != 0 && rotation != 90 && rotation != 180 && rotation != 270) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: rotation must be 0, 90, 180 or 270");
+  }
+
+  VideoFrameBlit::Request request;
+  request.frame = source.videoFrame;
+  if (const auto origin = source.origin.value_or(nullptr)) {
+    request.originX = coordinate(origin->x, "origin.x");
+    request.originY = coordinate(origin->y, "origin.y");
+  }
+  request.width = size.width;
+  request.height = size.height;
+  request.flipY = source.flipY.value_or(false);
+  request.rotation = rotation;
+  request.mirrored = source.mirrored.value_or(false);
+  request.texture = destination.texture->get();
+  request.mipLevel = coordinate(destination.mipLevel.value_or(0), "mipLevel");
+  wgpu::Origin3D dstOrigin{};
+  Convertor conv;
+  if (!conv(dstOrigin, destination.origin)) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: invalid destination origin");
+  }
+  if (dstOrigin.z != 0) {
+    throw std::runtime_error(
+        "copyExternalImageToTexture: a native frame can only be copied into "
+        "the first layer of the texture");
+  }
+  request.dstX = dstOrigin.x;
+  request.dstY = dstOrigin.y;
+  _videoFrameBlit->copy(_device, _instance, request);
+}
+
 void GPUQueue::copyExternalImageToTexture(
     std::shared_ptr<GPUImageCopyExternalImage> source,
     std::shared_ptr<GPUImageCopyTextureTagged> destination,
@@ -111,6 +179,17 @@ void GPUQueue::copyExternalImageToTexture(
   wgpu::TexelCopyBufferLayout layout{};
   wgpu::Extent3D sz{};
   Convertor conv;
+  if (source->videoFrame) {
+    if (!conv(sz, size)) {
+      throw std::runtime_error("copyExternalImageToTexture: invalid copy size");
+    }
+    copyVideoFrameToTexture(*source, *destination, sz);
+    return;
+  }
+  if (!source->source) {
+    throw std::runtime_error("copyExternalImageToTexture: the source must be "
+                             "an ImageBitmap or a NativeVideoFrame");
+  }
   uint32_t bytesPerPixel =
       source->source->getSize() /
       (source->source->getWidth() * source->source->getHeight());
@@ -139,10 +218,8 @@ void GPUQueue::copyExternalImageToTexture(
     throw std::runtime_error(
         "The source origin must be a non-negative integer coordinate.");
   }
-  const size_t sourceOriginX =
-      origin ? static_cast<size_t>(origin->x) : 0;
-  const size_t sourceOriginY =
-      origin ? static_cast<size_t>(origin->y) : 0;
+  const size_t sourceOriginX = origin ? static_cast<size_t>(origin->x) : 0;
+  const size_t sourceOriginY = origin ? static_cast<size_t>(origin->y) : 0;
   if (sourceOriginX > source->source->getWidth() ||
       sz.width > source->source->getWidth() - sourceOriginX ||
       sourceOriginY > source->source->getHeight() ||
@@ -193,6 +270,186 @@ void GPUQueue::copyExternalImageToTexture(
   }
 }
 
+async::AsyncTaskHandle GPUQueue::drawElementImageToTexture(
+    jsi::Runtime &runtime, std::shared_ptr<GPUDrawElementImageSource> source,
+    std::shared_ptr<GPUDrawElementImageDestination> destination) {
+  auto platform = PlatformContext::current();
+  if (!platform) {
+    throw std::runtime_error(
+        "drawElementImageToTexture: react-native-webgpu is not installed");
+  }
+  if (!destination || !destination->info || !destination->info->texture) {
+    throw std::runtime_error(
+        "drawElementImageToTexture: `destination.texture` is required");
+  }
+  const auto &info = destination->info;
+
+  // Destination texture: only the 8-bit RGBA/BGRA formats the platform
+  // rasterizers produce natively are accepted (with a byte swizzle between
+  // the two); anything else would need a GPU blit.
+  wgpu::TexelCopyTextureInfo dst{};
+  Convertor conv;
+  if (!conv(dst.aspect, info->aspect) || !conv(dst.mipLevel, info->mipLevel) ||
+      !conv(dst.origin, info->origin) || !conv(dst.texture, info->texture)) {
+    throw std::runtime_error("drawElementImageToTexture: invalid destination");
+  }
+  const wgpu::Texture texture = info->texture->get();
+  bool destinationIsBgra = false;
+  switch (texture.GetFormat()) {
+  case wgpu::TextureFormat::RGBA8Unorm:
+  case wgpu::TextureFormat::RGBA8UnormSrgb:
+    destinationIsBgra = false;
+    break;
+  case wgpu::TextureFormat::BGRA8Unorm:
+  case wgpu::TextureFormat::BGRA8UnormSrgb:
+    destinationIsBgra = true;
+    break;
+  default:
+    throw std::runtime_error(
+        "drawElementImageToTexture: the destination texture must be "
+        "rgba8unorm, rgba8unorm-srgb, bgra8unorm or bgra8unorm-srgb");
+  }
+  if (!(texture.GetUsage() & wgpu::TextureUsage::CopyDst)) {
+    throw std::runtime_error(
+        "drawElementImageToTexture: the destination texture must have "
+        "GPUTextureUsage.COPY_DST");
+  }
+  if (texture.GetDimension() != wgpu::TextureDimension::e2D ||
+      texture.GetSampleCount() != 1) {
+    throw std::runtime_error(
+        "drawElementImageToTexture: the destination must be a single-sampled "
+        "2D texture");
+  }
+  if (dst.mipLevel >= texture.GetMipLevelCount()) {
+    throw std::runtime_error(
+        "drawElementImageToTexture: destination.mipLevel is out of range");
+  }
+  const uint32_t mipWidth =
+      std::max<uint32_t>(1, texture.GetWidth() >> dst.mipLevel);
+  const uint32_t mipHeight =
+      std::max<uint32_t>(1, texture.GetHeight() >> dst.mipLevel);
+  if (dst.origin.z >= texture.GetDepthOrArrayLayers()) {
+    throw std::runtime_error(
+        "drawElementImageToTexture: destination.origin.z is out of range");
+  }
+
+  // Source rectangle (points, view space). Omitted width/height mean "the
+  // whole view", which only the platform knows.
+  ViewSnapshotRequest request{};
+  request.viewTag = source->viewTag;
+  request.sourceX = source->sourceX.value_or(0.0);
+  request.sourceY = source->sourceY.value_or(0.0);
+  request.sourceWidth = source->sourceWidth.value_or(0.0);
+  request.sourceHeight = source->sourceHeight.value_or(0.0);
+  if (!std::isfinite(request.sourceX) || !std::isfinite(request.sourceY) ||
+      !std::isfinite(request.sourceWidth) ||
+      !std::isfinite(request.sourceHeight) ||
+      (source->sourceWidth.has_value() && request.sourceWidth <= 0) ||
+      (source->sourceHeight.has_value() && request.sourceHeight <= 0)) {
+    throw std::runtime_error(
+        "drawElementImageToTexture: the source rectangle must be finite with "
+        "a positive width and height");
+  }
+
+  // Destination size (pixels). Omitted means the source rectangle's natural
+  // pixel size (points times the device pixel ratio), which the platform
+  // computes; the bounds check then happens once the pixels are back.
+  if (destination->size.has_value()) {
+    wgpu::Extent3D size{};
+    if (!conv(size, destination->size.value())) {
+      throw std::runtime_error(
+          "drawElementImageToTexture: invalid destination.size");
+    }
+    if (size.width == 0 || size.height == 0 || size.depthOrArrayLayers != 1) {
+      throw std::runtime_error(
+          "drawElementImageToTexture: destination.size must have a positive "
+          "width and height and a depthOrArrayLayers of 1");
+    }
+    if (size.width > mipWidth - std::min(dst.origin.x, mipWidth) ||
+        size.height > mipHeight - std::min(dst.origin.y, mipHeight)) {
+      throw std::runtime_error(
+          "drawElementImageToTexture: destination.origin + destination.size "
+          "exceeds the texture's extent at destination.mipLevel");
+    }
+    request.width = size.width;
+    request.height = size.height;
+  }
+
+  // premultipliedAlpha defaults to false, like copyExternalImageToTexture.
+  const bool destinationPremultiplied =
+      info->premultipliedAlpha.value_or(false);
+
+  // Settle on the CALLING runtime's context (see GPUBuffer::mapAsync): the
+  // snapshot completes on the UI thread and is deposited into that context's
+  // mailbox; the texture write and the resolve then run on the runtime's own
+  // thread during its next tick.
+  auto context =
+      async::RuntimeContext::getOrCreate(runtime, _async->instance());
+  auto queue = _instance;
+  // Keeps the destination texture alive until the write is issued.
+  auto gpuTexture = info->texture;
+  return context->postTask(
+      [platform, request, dst, mipWidth, mipHeight, destinationIsBgra,
+       destinationPremultiplied, queue,
+       gpuTexture](const async::AsyncTaskHandle::ResolveFunction &resolve,
+                   const async::AsyncTaskHandle::RejectFunction &reject) {
+        platform->snapshotView(
+            request,
+            [resolve, reject, dst, mipWidth, mipHeight, destinationIsBgra,
+             destinationPremultiplied, queue,
+             gpuTexture](ImageData image) mutable {
+              // Arbitrary (UI) thread: validate without touching JSI.
+              if (image.width == 0 || image.height == 0 ||
+                  image.data.size() < image.width * image.height * 4) {
+                reject("drawElementImageToTexture: the view produced no "
+                       "pixels (is it mounted with a non-zero size?)");
+                return;
+              }
+              if (image.width >
+                      mipWidth - std::min<uint32_t>(dst.origin.x, mipWidth) ||
+                  image.height >
+                      mipHeight - std::min<uint32_t>(dst.origin.y, mipHeight)) {
+                reject("drawElementImageToTexture: the view's natural pixel "
+                       "size (" +
+                       std::to_string(image.width) + "x" +
+                       std::to_string(image.height) +
+                       ") does not fit in the destination texture at "
+                       "destination.origin; pass destination.size to scale "
+                       "it, or create a larger texture");
+                return;
+              }
+              resolve([image = std::move(image), dst, destinationIsBgra,
+                       destinationPremultiplied, queue,
+                       gpuTexture](jsi::Runtime &) mutable -> jsi::Value {
+                // Owning runtime's thread. Bring the pixels into the
+                // destination's channel order and alpha representation, then
+                // issue the write.
+                const bool sourceIsBgra =
+                    image.format == wgpu::TextureFormat::BGRA8Unorm ||
+                    image.format == wgpu::TextureFormat::BGRA8UnormSrgb;
+                uint8_t *pixels = image.data.data();
+                const size_t byteLength = image.width * image.height * 4;
+                if (sourceIsBgra != destinationIsBgra) {
+                  for (size_t i = 0; i + 3 < byteLength; i += 4) {
+                    std::swap(pixels[i], pixels[i + 2]);
+                  }
+                }
+                convertAlpha(pixels, byteLength, image.premultiplied,
+                             destinationPremultiplied);
+                wgpu::TexelCopyBufferLayout layout{};
+                layout.offset = 0;
+                layout.bytesPerRow = static_cast<uint32_t>(image.width * 4);
+                layout.rowsPerImage = static_cast<uint32_t>(image.height);
+                wgpu::Extent3D extent{static_cast<uint32_t>(image.width),
+                                      static_cast<uint32_t>(image.height), 1};
+                queue.WriteTexture(&dst, pixels, byteLength, &layout, &extent);
+                return jsi::Value::undefined();
+              });
+            },
+            [reject](std::string error) { reject(std::move(error)); });
+      });
+}
+
 void GPUQueue::writeTexture(std::shared_ptr<GPUImageCopyTexture> destination,
                             std::shared_ptr<ArrayBuffer> data,
                             std::shared_ptr<GPUImageDataLayout> dataLayout,

package/cpp/rnwgpu/api/GPUQueue.h

@@ -16,8 +16,11 @@
 #include "ArrayBuffer.h"
 #include "GPUBuffer.h"
 #include "GPUCommandBuffer.h"
+#include "GPUDrawElementImageDestination.h"
+#include "GPUDrawElementImageSource.h"
 #include "GPUImageCopyExternalImage.h"
 #include "GPUImageCopyTextureTagged.h"
+#include "VideoFrameBlit.h"
 
 namespace rnwgpu {
 
@@ -27,10 +30,12 @@ class GPUQueue : public NativeObject<GPUQueue> {
 public:
   static constexpr const char *CLASS_NAME = "GPUQueue";
 
-  explicit GPUQueue(wgpu::Queue instance,
+  explicit GPUQueue(wgpu::Device device, wgpu::Queue instance,
                     std::shared_ptr<async::RuntimeContext> async,
+                    std::shared_ptr<VideoFrameBlit> videoFrameBlit,
                     std::string label)
-      : NativeObject(CLASS_NAME), _instance(instance), _async(async),
+      : NativeObject(CLASS_NAME), _device(device), _instance(instance),
+        _async(async), _videoFrameBlit(std::move(videoFrameBlit)),
         _label(label) {}
 
 public:
@@ -46,11 +51,25 @@ public:
                     std::shared_ptr<ArrayBuffer> data,
                     std::shared_ptr<GPUImageDataLayout> dataLayout,
                     std::shared_ptr<GPUExtent3D> size);
+  // The source is an ImageBitmap (uploaded from the CPU) or, as a React
+  // Native extension, a NativeVideoFrame rendered into the destination through
+  // an external texture (see VideoFrameBlit).
   void copyExternalImageToTexture(
       std::shared_ptr<GPUImageCopyExternalImage> source,
       std::shared_ptr<GPUImageCopyTextureTagged> destination,
       std::shared_ptr<GPUExtent3D> copySize);
 
+  // Non-spec React Native extension modeled on the html-in-canvas proposal's
+  // GPUQueue.drawElementImageToTexture: rasterizes a native view (the
+  // "element") and writes the pixels into `destination.texture`. Unlike the
+  // web API it returns a Promise, because the view can only be rasterized on
+  // the platform UI thread; the texture write is issued on the calling
+  // runtime's thread right before the promise resolves, so work submitted
+  // after `await` observes the new contents.
+  async::AsyncTaskHandle drawElementImageToTexture(
+      jsi::Runtime &runtime, std::shared_ptr<GPUDrawElementImageSource> source,
+      std::shared_ptr<GPUDrawElementImageDestination> destination);
+
   std::string getLabel() { return _label; }
   void setLabel(const std::string &label) {
     _label = label;
@@ -66,6 +85,8 @@ public:
     installMethod(runtime, prototype, "writeTexture", &GPUQueue::writeTexture);
     installMethod(runtime, prototype, "copyExternalImageToTexture",
                   &GPUQueue::copyExternalImageToTexture);
+    installMethodWithRuntime(runtime, prototype, "drawElementImageToTexture",
+                             &GPUQueue::drawElementImageToTexture);
     installGetterSetter(runtime, prototype, "label", &GPUQueue::getLabel,
                         &GPUQueue::setLabel);
   }
@@ -73,8 +94,14 @@ public:
   inline const wgpu::Queue get() { return _instance; }
 
 private:
+  void copyVideoFrameToTexture(const GPUImageCopyExternalImage &source,
+                               const GPUImageCopyTextureTagged &destination,
+                               const wgpu::Extent3D &size);
+
+  wgpu::Device _device;
   wgpu::Queue _instance;
   std::shared_ptr<async::RuntimeContext> _async;
+  std::shared_ptr<VideoFrameBlit> _videoFrameBlit;
   std::string _label;
 };