react-native-webgpu 0.10.5 → 0.11.1

package/android/cpp/AndroidPlatformContext.h

@@ -13,6 +13,7 @@
 
 #include "webgpu/webgpu_cpp.h"
 
+#include "AndroidVideoPlayer.h"
 #include "PlatformContext.h"
 #include "RNWebGPUManager.h"
 
@@ -21,9 +22,19 @@ namespace rnwgpu {
 namespace jsi = facebook::jsi;
 namespace jni = facebook::jni;
 
+// The completion callbacks of one snapshotView request, heap-allocated and
+// carried through Java as an opaque pointer (WebGPUModule.snapshotView ->
+// onViewSnapshot), which hands them back exactly once.
+struct ViewSnapshotCallbacks {
+  std::function<void(ImageData)> onSuccess;
+  std::function<void(std::string)> onError;
+};
+
 class AndroidPlatformContext : public PlatformContext {
 private:
   jobject _blobModule;
+  // Global ref to the com.webgpu.WebGPUModule instance (view snapshots).
+  jobject _module;
 
   std::vector<uint8_t> resolveBlob(JNIEnv *env, const std::string &blobId,
                                    double offset, double size) {
@@ -63,14 +74,61 @@ private:
   }
 
 public:
-  explicit AndroidPlatformContext(jobject blobModule)
-      : _blobModule(blobModule) {}
+  // Both arguments are global refs; the context releases them.
+  AndroidPlatformContext(jobject blobModule, jobject module)
+      : _blobModule(blobModule), _module(module) {}
   ~AndroidPlatformContext() {
+    JNIEnv *env = facebook::jni::Environment::current();
     if (_blobModule) {
-      JNIEnv *env = facebook::jni::Environment::current();
       env->DeleteGlobalRef(_blobModule);
       _blobModule = nullptr;
     }
+    if (_module) {
+      env->DeleteGlobalRef(_module);
+      _module = nullptr;
+    }
+  }
+
+  void snapshotView(const ViewSnapshotRequest &request,
+                    std::function<void(ImageData)> onSuccess,
+                    std::function<void(std::string)> onError) override {
+    if (!_module) {
+      onError("drawElementImageToTexture: the WebGPU module is gone");
+      return;
+    }
+    jni::Environment::ensureCurrentThreadIsAttached();
+    JNIEnv *env = facebook::jni::Environment::current();
+    if (!env) {
+      onError("drawElementImageToTexture: couldn't get the JNI environment");
+      return;
+    }
+    jclass moduleClass = env->GetObjectClass(_module);
+    jmethodID method =
+        env->GetMethodID(moduleClass, "snapshotView", "(IDDDDIIJ)V");
+    env->DeleteLocalRef(moduleClass);
+    if (!method) {
+      env->ExceptionClear();
+      onError("drawElementImageToTexture: WebGPUModule.snapshotView not found");
+      return;
+    }
+    // Ownership passes to Java; Java_com_webgpu_WebGPUModule_onViewSnapshot
+    // (cpp-adapter.cpp) reclaims it.
+    auto *callbacks =
+        new ViewSnapshotCallbacks{std::move(onSuccess), std::move(onError)};
+    env->CallVoidMethod(_module, method, static_cast<jint>(request.viewTag),
+                        static_cast<jdouble>(request.sourceX),
+                        static_cast<jdouble>(request.sourceY),
+                        static_cast<jdouble>(request.sourceWidth),
+                        static_cast<jdouble>(request.sourceHeight),
+                        static_cast<jint>(request.width),
+                        static_cast<jint>(request.height),
+                        reinterpret_cast<jlong>(callbacks));
+    if (env->ExceptionCheck()) {
+      env->ExceptionClear();
+      auto fail = std::move(callbacks->onError);
+      delete callbacks;
+      fail("drawElementImageToTexture: WebGPUModule.snapshotView threw");
+    }
   }
 
   wgpu::Surface makeSurface(wgpu::Instance instance, void *window, int width,
@@ -179,8 +237,9 @@ public:
     result.height = static_cast<int>(bitmapInfo.height);
     result.data.resize(bitmapInfo.height * bitmapInfo.stride);
     memcpy(result.data.data(), bitmapPixels, result.data.size());
-    // BitmapFactory hands back premultiplied ARGB_8888 pixels; createImageBitmap
-    // converts to the representation requested by premultiplyAlpha.
+    // BitmapFactory hands back premultiplied ARGB_8888 pixels;
+    // createImageBitmap converts to the representation requested by
+    // premultiplyAlpha.
     result.premultiplied = true;
 
     AndroidBitmap_unlockPixels(env, bitmap);
@@ -282,12 +341,44 @@ public:
     return handle;
   }
 
+  // The decoder writes its native YUV layout whatever format is requested:
+  // the frames are sampled through importExternalTexture (see
+  // WebGPUVideoPlayer.java).
   std::unique_ptr<IVideoPlayer>
-  createVideoPlayer(const std::string & /*path*/,
+  createVideoPlayer(const std::string &path,
                     VideoPixelFormat /*format*/) override {
-    // TODO: implement using MediaCodec -> ImageReader (AHardwareBuffer mode).
-    throw std::runtime_error(
-        "createVideoPlayer is not yet implemented on Android.");
+    if (!_module) {
+      throw std::runtime_error("createVideoPlayer: the WebGPU module is gone");
+    }
+    jni::Environment::ensureCurrentThreadIsAttached();
+    JNIEnv *env = facebook::jni::Environment::current();
+    if (!env) {
+      throw std::runtime_error(
+          "createVideoPlayer: couldn't get the JNI environment");
+    }
+    jclass moduleClass = env->GetObjectClass(_module);
+    jmethodID method =
+        env->GetMethodID(moduleClass, "createVideoPlayer",
+                         "(Ljava/lang/String;)Lcom/webgpu/WebGPUVideoPlayer;");
+    env->DeleteLocalRef(moduleClass);
+    if (!method) {
+      env->ExceptionClear();
+      throw std::runtime_error(
+          "createVideoPlayer: WebGPUModule.createVideoPlayer not found");
+    }
+    jstring jPath = env->NewStringUTF(path.c_str());
+    jobject player = env->CallObjectMethod(_module, method, jPath);
+    env->DeleteLocalRef(jPath);
+    if (env->ExceptionCheck()) {
+      throw std::runtime_error("createVideoPlayer: " +
+                               describeAndClearJavaException(env));
+    }
+    if (player == nullptr) {
+      throw std::runtime_error("createVideoPlayer: no player was created");
+    }
+    auto result = std::make_unique<AndroidVideoPlayer>(env, player);
+    env->DeleteLocalRef(player);
+    return result;
   }
 
   std::string writeTestVideoFile() override {

package/android/cpp/AndroidVideoPlayer.h

@@ -0,0 +1,307 @@
+#pragma once
+
+#include <android/hardware_buffer.h>
+#include <android/hardware_buffer_jni.h>
+#include <fbjni/fbjni.h>
+#include <jni.h>
+
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+#include "PlatformContext.h"
+
+namespace rnwgpu {
+
+// The message of the pending Java exception, which is cleared.
+inline std::string describeAndClearJavaException(JNIEnv *env) {
+  jthrowable throwable = env->ExceptionOccurred();
+  if (throwable == nullptr) {
+    return "unknown error";
+  }
+  env->ExceptionClear();
+  std::string message = "Java exception";
+  jclass throwableClass = env->GetObjectClass(throwable);
+  jmethodID getMessage =
+      env->GetMethodID(throwableClass, "getMessage", "()Ljava/lang/String;");
+  if (getMessage != nullptr) {
+    auto text =
+        static_cast<jstring>(env->CallObjectMethod(throwable, getMessage));
+    if (env->ExceptionCheck()) {
+      env->ExceptionClear();
+    } else if (text != nullptr) {
+      const char *chars = env->GetStringUTFChars(text, nullptr);
+      if (chars != nullptr) {
+        message = chars;
+        env->ReleaseStringUTFChars(text, chars);
+      }
+      env->DeleteLocalRef(text);
+    }
+  }
+  env->DeleteLocalRef(throwableClass);
+  env->DeleteLocalRef(throwable);
+  return message;
+}
+
+// A global reference to the Java player, shared between AndroidVideoPlayer
+// and the deleters of the frames it handed out (they close each frame's
+// Image through the player). The reference goes away with the last of them.
+class JavaVideoPlayerRef {
+public:
+  JavaVideoPlayerRef(JNIEnv *env, jobject local)
+      : _object(env->NewGlobalRef(local)) {}
+  ~JavaVideoPlayerRef() {
+    facebook::jni::Environment::ensureCurrentThreadIsAttached();
+    JNIEnv *env = facebook::jni::Environment::current();
+    if (env != nullptr && _object != nullptr) {
+      env->DeleteGlobalRef(_object);
+    }
+  }
+  JavaVideoPlayerRef(const JavaVideoPlayerRef &) = delete;
+  JavaVideoPlayerRef &operator=(const JavaVideoPlayerRef &) = delete;
+
+  jobject get() const { return _object; }
+
+private:
+  jobject _object;
+};
+
+// IVideoPlayer backed by com.webgpu.WebGPUVideoPlayer: MediaPlayer decodes
+// into an ImageReader whose buffers the GPU samples. The frames keep the
+// decoder's native YUV layout (reported as NV12), for importExternalTexture.
+class AndroidVideoPlayer : public IVideoPlayer {
+public:
+  AndroidVideoPlayer(JNIEnv *env, jobject player)
+      : _player(std::make_shared<JavaVideoPlayerRef>(env, player)) {
+    jclass playerClass = env->GetObjectClass(player);
+    _copyLatestFrame =
+        method(env, playerClass, "copyLatestFrame", "()Landroid/media/Image;");
+    _closeImage =
+        method(env, playerClass, "closeImage", "(Landroid/media/Image;)V");
+    _play = method(env, playerClass, "play", "()V");
+    _pause = method(env, playerClass, "pause", "()V");
+    _getPaused = method(env, playerClass, "getPaused", "()Z");
+    _getCurrentTime = method(env, playerClass, "getCurrentTime", "()D");
+    _seek = method(env, playerClass, "seek", "(D)V");
+    _getDuration = method(env, playerClass, "getDuration", "()D");
+    _getVolume = method(env, playerClass, "getVolume", "()D");
+    _setVolume = method(env, playerClass, "setVolume", "(D)V");
+    _getLoop = method(env, playerClass, "getLoop", "()Z");
+    _setLoop = method(env, playerClass, "setLoop", "(Z)V");
+    _getVideoWidth = method(env, playerClass, "getVideoWidth", "()I");
+    _getVideoHeight = method(env, playerClass, "getVideoHeight", "()I");
+    _getRotation = method(env, playerClass, "getRotation", "()I");
+    _getFrameRate = method(env, playerClass, "getFrameRate", "()D");
+    _release = method(env, playerClass, "release", "()V");
+    env->DeleteLocalRef(playerClass);
+
+    jclass imageClass = env->FindClass("android/media/Image");
+    _getHardwareBuffer = method(env, imageClass, "getHardwareBuffer",
+                                "()Landroid/hardware/HardwareBuffer;");
+    env->DeleteLocalRef(imageClass);
+  }
+
+  ~AndroidVideoPlayer() override {
+    JNIEnv *env = attach();
+    env->CallVoidMethod(_player->get(), _release);
+    if (env->ExceptionCheck()) {
+      env->ExceptionClear();
+    }
+  }
+
+  VideoFrameHandle copyLatestFrame() override {
+    JNIEnv *env = attach();
+    jobject image = env->CallObjectMethod(_player->get(), _copyLatestFrame);
+    if (env->ExceptionCheck()) {
+      throw std::runtime_error("copyLatestFrame: " +
+                               describeAndClearJavaException(env));
+    }
+    if (image == nullptr) {
+      return {};
+    }
+
+    jobject hardwareBuffer = env->CallObjectMethod(image, _getHardwareBuffer);
+    if (env->ExceptionCheck() || hardwareBuffer == nullptr) {
+      env->ExceptionClear();
+      closeImage(env, image);
+      env->DeleteLocalRef(image);
+      throw std::runtime_error(
+          "copyLatestFrame: the decoded frame has no HardwareBuffer");
+    }
+    AHardwareBuffer *buffer =
+        AHardwareBuffer_fromHardwareBuffer(env, hardwareBuffer);
+    env->DeleteLocalRef(hardwareBuffer);
+
+    AHardwareBuffer_Desc desc = {};
+    AHardwareBuffer_describe(buffer, &desc);
+    if ((desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) == 0) {
+      closeImage(env, image);
+      env->DeleteLocalRef(image);
+      throw std::runtime_error(
+          "copyLatestFrame: the decoder allocated frames without "
+          "AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, so the GPU cannot sample "
+          "them");
+    }
+
+    // The frame owns a reference to the buffer and keeps the Image acquired
+    // until it is released; closing the Image hands the buffer back to the
+    // decoder.
+    AHardwareBuffer_acquire(buffer);
+    jobject imageRef = env->NewGlobalRef(image);
+    env->DeleteLocalRef(image);
+
+    VideoFrameHandle handle;
+    handle.handle = static_cast<void *>(buffer);
+    handle.width = desc.width;
+    handle.height = desc.height;
+    handle.pixelFormat = isRgbFormat(desc.format) ? VideoPixelFormat::BGRA8
+                                                  : VideoPixelFormat::NV12;
+    auto player = _player;
+    jmethodID closeImageMethod = _closeImage;
+    handle.deleter = [player, closeImageMethod, imageRef, buffer]() {
+      JNIEnv *env = attach();
+      env->CallVoidMethod(player->get(), closeImageMethod, imageRef);
+      if (env->ExceptionCheck()) {
+        env->ExceptionClear();
+      }
+      env->DeleteGlobalRef(imageRef);
+      AHardwareBuffer_release(buffer);
+    };
+    return handle;
+  }
+
+  void play() override { callVoid(_play); }
+  void pause() override { callVoid(_pause); }
+  bool paused() override { return callBool(_getPaused); }
+
+  double currentTime() override { return callDouble(_getCurrentTime); }
+  void seek(double seconds) override {
+    JNIEnv *env = attach();
+    env->CallVoidMethod(_player->get(), _seek, static_cast<jdouble>(seconds));
+    clearException(env);
+  }
+  double duration() override { return callDouble(_getDuration); }
+
+  double volume() override { return callDouble(_getVolume); }
+  void setVolume(double volume) override {
+    JNIEnv *env = attach();
+    env->CallVoidMethod(_player->get(), _setVolume,
+                        static_cast<jdouble>(volume));
+    clearException(env);
+  }
+
+  bool loop() override { return callBool(_getLoop); }
+  void setLoop(bool loop) override {
+    JNIEnv *env = attach();
+    env->CallVoidMethod(_player->get(), _setLoop, static_cast<jboolean>(loop));
+    clearException(env);
+  }
+
+  uint32_t videoWidth() override {
+    return static_cast<uint32_t>(callInt(_getVideoWidth));
+  }
+  uint32_t videoHeight() override {
+    return static_cast<uint32_t>(callInt(_getVideoHeight));
+  }
+  int rotation() override { return callInt(_getRotation); }
+  double frameRate() override { return callDouble(_getFrameRate); }
+
+private:
+  static JNIEnv *attach() {
+    facebook::jni::Environment::ensureCurrentThreadIsAttached();
+    JNIEnv *env = facebook::jni::Environment::current();
+    if (env == nullptr) {
+      throw std::runtime_error("VideoPlayer: no JNI environment");
+    }
+    return env;
+  }
+
+  static jmethodID method(JNIEnv *env, jclass cls, const char *name,
+                          const char *signature) {
+    jmethodID id = env->GetMethodID(cls, name, signature);
+    if (id == nullptr) {
+      env->ExceptionClear();
+      throw std::runtime_error(std::string("VideoPlayer: method ") + name +
+                               " not found");
+    }
+    return id;
+  }
+
+  static void clearException(JNIEnv *env) {
+    if (env->ExceptionCheck()) {
+      env->ExceptionClear();
+    }
+  }
+
+  // Single-plane RGB buffers sample as regular textures; everything else
+  // (YUV 4:2:0, P010, implementation-defined codec formats) goes through
+  // Vulkan's YCbCr conversion as an external texture.
+  static bool isRgbFormat(uint32_t format) {
+    switch (format) {
+    case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM:
+    case AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM:
+    case AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM:
+    case AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM:
+    case AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT:
+    case AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  void closeImage(JNIEnv *env, jobject image) {
+    env->CallVoidMethod(_player->get(), _closeImage, image);
+    clearException(env);
+  }
+
+  void callVoid(jmethodID id) {
+    JNIEnv *env = attach();
+    env->CallVoidMethod(_player->get(), id);
+    clearException(env);
+  }
+
+  bool callBool(jmethodID id) {
+    JNIEnv *env = attach();
+    bool result = env->CallBooleanMethod(_player->get(), id) == JNI_TRUE;
+    clearException(env);
+    return result;
+  }
+
+  int callInt(jmethodID id) {
+    JNIEnv *env = attach();
+    int result = env->CallIntMethod(_player->get(), id);
+    clearException(env);
+    return result;
+  }
+
+  double callDouble(jmethodID id) {
+    JNIEnv *env = attach();
+    double result = env->CallDoubleMethod(_player->get(), id);
+    clearException(env);
+    return result;
+  }
+
+  std::shared_ptr<JavaVideoPlayerRef> _player;
+  jmethodID _copyLatestFrame;
+  jmethodID _closeImage;
+  jmethodID _play;
+  jmethodID _pause;
+  jmethodID _getPaused;
+  jmethodID _getCurrentTime;
+  jmethodID _seek;
+  jmethodID _getDuration;
+  jmethodID _getVolume;
+  jmethodID _setVolume;
+  jmethodID _getLoop;
+  jmethodID _setLoop;
+  jmethodID _getVideoWidth;
+  jmethodID _getVideoHeight;
+  jmethodID _getRotation;
+  jmethodID _getFrameRate;
+  jmethodID _release;
+  jmethodID _getHardwareBuffer;
+};
+
+} // namespace rnwgpu

package/android/cpp/cpp-adapter.cpp

@@ -1,11 +1,15 @@
+#include <cstring>
 #include <memory>
+#include <string>
 #include <unordered_map>
+#include <utility>
 
 #include <fbjni/fbjni.h>
 #include <jni.h>
 #include <jsi/jsi.h>
 
 #include <ReactCommon/CallInvokerHolder.h>
+#include <android/bitmap.h>
 #include <android/hardware_buffer_jni.h>
 #include <android/native_window_jni.h>
 #include <webgpu/webgpu_cpp.h>
@@ -19,21 +23,77 @@
 std::shared_ptr<rnwgpu::RNWebGPUManager> manager;
 
 extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUModule_initializeNative(
-    JNIEnv *env, jobject /* this */, jlong jsRuntime,
-    jobject jsCallInvokerHolder, jobject blobModule) {
+    JNIEnv *env, jobject thiz, jlong jsRuntime, jobject jsCallInvokerHolder,
+    jobject blobModule) {
   auto runtime = reinterpret_cast<facebook::jsi::Runtime *>(jsRuntime);
   jobject globalBlobModule = env->NewGlobalRef(blobModule);
+  jobject globalModule = env->NewGlobalRef(thiz);
   auto jsCallInvoker{
       facebook::jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>{
           reinterpret_cast<facebook::react::CallInvokerHolder::javaobject>(
-              jsCallInvokerHolder)} -> cthis()
-          ->getCallInvoker()};
-  auto platformContext =
-      std::make_shared<rnwgpu::AndroidPlatformContext>(globalBlobModule);
+              jsCallInvokerHolder)} -> cthis()->getCallInvoker()};
+  auto platformContext = std::make_shared<rnwgpu::AndroidPlatformContext>(
+      globalBlobModule, globalModule);
   manager = std::make_shared<rnwgpu::RNWebGPUManager>(runtime, jsCallInvoker,
                                                       platformContext);
 }
 
+// Completion of WebGPUModule.snapshotView (UI thread). Reclaims the callbacks
+// handed to Java and delivers either the bitmap's pixels or the error.
+extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUModule_onViewSnapshot(
+    JNIEnv *env, jobject /* this */, jlong callbackPointer, jobject bitmap,
+    jstring error) {
+  std::unique_ptr<rnwgpu::ViewSnapshotCallbacks> callbacks(
+      reinterpret_cast<rnwgpu::ViewSnapshotCallbacks *>(callbackPointer));
+  if (!callbacks) {
+    return;
+  }
+  if (error != nullptr) {
+    const char *chars = env->GetStringUTFChars(error, nullptr);
+    std::string message(chars ? chars : "drawElementImageToTexture failed");
+    if (chars) {
+      env->ReleaseStringUTFChars(error, chars);
+    }
+    callbacks->onError(std::move(message));
+    return;
+  }
+  if (bitmap == nullptr) {
+    callbacks->onError("drawElementImageToTexture: no bitmap was produced");
+    return;
+  }
+  AndroidBitmapInfo info;
+  if (AndroidBitmap_getInfo(env, bitmap, &info) !=
+          ANDROID_BITMAP_RESULT_SUCCESS ||
+      info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {
+    callbacks->onError(
+        "drawElementImageToTexture: unexpected snapshot bitmap format");
+    return;
+  }
+  void *pixels = nullptr;
+  if (AndroidBitmap_lockPixels(env, bitmap, &pixels) !=
+          ANDROID_BITMAP_RESULT_SUCCESS ||
+      pixels == nullptr) {
+    callbacks->onError(
+        "drawElementImageToTexture: couldn't lock the snapshot bitmap");
+    return;
+  }
+  rnwgpu::ImageData image;
+  image.width = info.width;
+  image.height = info.height;
+  image.format = wgpu::TextureFormat::RGBA8Unorm;
+  // Bitmap.Config.ARGB_8888 stores premultiplied RGBA bytes.
+  image.premultiplied = true;
+  const size_t rowBytes = static_cast<size_t>(info.width) * 4;
+  image.data.resize(rowBytes * info.height);
+  const auto *src = static_cast<const uint8_t *>(pixels);
+  for (uint32_t row = 0; row < info.height; ++row) {
+    std::memcpy(image.data.data() + row * rowBytes, src + row * info.stride,
+                rowBytes);
+  }
+  AndroidBitmap_unlockPixels(env, bitmap);
+  callbacks->onSuccess(std::move(image));
+}
+
 extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUView_onSurfaceChanged(
     JNIEnv *env, jobject thiz, jobject surface, jint contextId, jfloat width,
     jfloat height) {