react-native-webgpu 0.14.2 → 0.15.1

package/android/src/main/java/com/webgpu/WebGPUModule.java

@@ -1,6 +1,8 @@
 package com.webgpu;
 
 import android.graphics.Bitmap;
+import android.hardware.HardwareBuffer;
+import android.os.Build;
 import android.util.Log;
 import android.view.View;
 
@@ -57,9 +59,12 @@ public class WebGPUModule extends NativeWebGPUModuleSpec {
    * Called from C++ (AndroidPlatformContext::snapshotView) on an arbitrary
    * thread for GPUQueue.drawElementImageToTexture. Hops to the UI thread,
    * resolves the view for {@code tag}, rasterizes the requested rectangle and
-   * always reports back through {@link #onViewSnapshot}, with either a bitmap
-   * or an error message. {@code callback} is an opaque native pointer that
-   * must be handed back exactly once.
+   * always reports back exactly once: through {@link #onViewSnapshotBuffer}
+   * with the buffer the hardware renderer drew into, when the caller allows
+   * it and the platform can tell when the drawing is done (API 33), or else
+   * through {@link #onViewSnapshot} with a bitmap or an error message.
+   * {@code callback} is an opaque native pointer that must be handed back
+   * exactly once.
    *
    * The source rectangle is in density-independent points (React Native
    * layout units) and is converted to pixels here; a non-positive
@@ -75,11 +80,15 @@ public class WebGPUModule extends NativeWebGPUModuleSpec {
       final double sourceHeight,
       final int width,
       final int height,
+      final boolean allowHardwareBuffer,
       final long callback) {
     UiThreadUtil.runOnUiThread(
         () -> {
           Bitmap bitmap = null;
           String error = null;
+          // Set once the callback was handed to onViewSnapshotBuffer; nothing
+          // may report again after that.
+          final boolean[] handedOver = {false};
           try {
             ReactApplicationContext context = getReactApplicationContext();
             UIManager uiManager = UIManagerHelper.getUIManagerForReactTag(context, tag);
@@ -105,17 +114,72 @@ public class WebGPUModule extends NativeWebGPUModuleSpec {
               throw new IllegalArgumentException(
                   "drawElementImageToTexture: the source rectangle is empty");
             }
-            bitmap = ViewSnapshot.snapshot(view, sx, sy, sw, sh, w, h);
+            // The zero-copy path: the renderer's buffer goes to the GPU with
+            // the fence that says when the drawing is done. The fence is
+            // exposed from API 33; before that, and when the hardware render
+            // fails, the pixels are read back into a bitmap as usual.
+            if (allowHardwareBuffer
+                && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU
+                && view.isHardwareAccelerated()) {
+              try {
+                ViewSnapshot.renderHardware(
+                    view,
+                    sx,
+                    sy,
+                    sw,
+                    sh,
+                    w,
+                    h,
+                    image -> {
+                      HardwareBuffer buffer = image.getHardwareBuffer();
+                      if (buffer == null) {
+                        throw new IllegalStateException(
+                            "HardwareRenderer image has no HardwareBuffer");
+                      }
+                      int fenceFd = SyncFences.acquireFenceFd(image);
+                      // Native acquires its own reference to the buffer and
+                      // adopts the fence descriptor.
+                      handedOver[0] = true;
+                      onViewSnapshotBuffer(callback, buffer, fenceFd);
+                      buffer.close();
+                      return null;
+                    });
+              } catch (Throwable t) {
+                if (handedOver[0]) {
+                  throw t;
+                }
+                Log.w(NAME, "HardwareRenderer snapshot failed, reading the pixels back instead", t);
+              }
+            }
+            if (!handedOver[0]) {
+              bitmap = ViewSnapshot.snapshot(view, sx, sy, sw, sh, w, h);
+            }
           } catch (Throwable t) {
+            if (handedOver[0]) {
+              // The result is already on its way; only the cleanup failed.
+              Log.w(NAME, "drawElementImageToTexture: cleanup after the snapshot failed", t);
+              return;
+            }
             error = t.getMessage() != null ? t.getMessage() : t.toString();
           }
-          onViewSnapshot(callback, bitmap, error);
+          if (!handedOver[0]) {
+            onViewSnapshot(callback, bitmap, error);
+          }
         });
   }
 
   @DoNotStrip
   private native void onViewSnapshot(long callback, Bitmap bitmap, String error);
 
+  /**
+   * The zero-copy completion of {@link #snapshotView}: the buffer the hardware
+   * renderer drew the view into and the fence that signals when the drawing
+   * is done (a sync file descriptor native takes ownership of, -1 for none).
+   * Native acquires its own reference to the buffer before returning.
+   */
+  @DoNotStrip
+  private native void onViewSnapshotBuffer(long callback, HardwareBuffer buffer, int fenceFd);
+
   /**
    * Called from C++ (AndroidPlatformContext::createVideoPlayer) for
    * RNWebGPU.createVideoPlayer. Throws when the source cannot be opened; the

package/android/src/main/java/com/webgpu/WebGPUVideoPlayer.java

@@ -61,6 +61,23 @@ public class WebGPUVideoPlayer {
   private volatile int rotation = 0;
   private volatile double frameRate = 0;
   private volatile double durationSeconds = 0;
+  // The color space of the video track as the container declares it, packed
+  // like a data space (see dataSpaceFromContainer); 0 when it says nothing.
+  private volatile int containerDataSpace = 0;
+
+  // The layout of android.hardware.DataSpace (API 33), which is also the
+  // ADataSpace of the NDK: standard << 16, transfer << 22, range << 27. Spelled
+  // out so that the mapping compiles and runs on every supported API level.
+  private static final int DATASPACE_STANDARD_BT709 = 1 << 16;
+  private static final int DATASPACE_STANDARD_BT601_625 = 2 << 16;
+  private static final int DATASPACE_STANDARD_BT601_525 = 4 << 16;
+  private static final int DATASPACE_STANDARD_BT2020 = 6 << 16;
+  private static final int DATASPACE_TRANSFER_LINEAR = 1 << 22;
+  private static final int DATASPACE_TRANSFER_SMPTE_170M = 3 << 22;
+  private static final int DATASPACE_TRANSFER_ST2084 = 7 << 22;
+  private static final int DATASPACE_TRANSFER_HLG = 8 << 22;
+  private static final int DATASPACE_RANGE_FULL = 1 << 27;
+  private static final int DATASPACE_RANGE_LIMITED = 2 << 27;
 
   WebGPUVideoPlayer(Context context, String source) {
     Uri uri = parseSource(source);
@@ -160,6 +177,17 @@ public class WebGPUVideoPlayer {
                     if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
                       frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
                     }
+                    containerDataSpace =
+                        dataSpaceFromContainer(
+                            format.containsKey(MediaFormat.KEY_COLOR_STANDARD)
+                                ? format.getInteger(MediaFormat.KEY_COLOR_STANDARD)
+                                : 0,
+                            format.containsKey(MediaFormat.KEY_COLOR_TRANSFER)
+                                ? format.getInteger(MediaFormat.KEY_COLOR_TRANSFER)
+                                : 0,
+                            format.containsKey(MediaFormat.KEY_COLOR_RANGE)
+                                ? format.getInteger(MediaFormat.KEY_COLOR_RANGE)
+                                : 0);
                     break;
                   }
                 }
@@ -185,6 +213,77 @@ public class WebGPUVideoPlayer {
     }
   }
 
+  // The MediaFormat color keys of the video track (KEY_COLOR_STANDARD,
+  // KEY_COLOR_TRANSFER, KEY_COLOR_RANGE; 0 when absent) packed like a data
+  // space, so that the native side reads one representation whatever its
+  // source.
+  private static int dataSpaceFromContainer(int standard, int transfer, int range) {
+    int dataSpace = 0;
+    switch (standard) {
+      case MediaFormat.COLOR_STANDARD_BT709:
+        dataSpace |= DATASPACE_STANDARD_BT709;
+        break;
+      case MediaFormat.COLOR_STANDARD_BT601_PAL:
+        dataSpace |= DATASPACE_STANDARD_BT601_625;
+        break;
+      case MediaFormat.COLOR_STANDARD_BT601_NTSC:
+        dataSpace |= DATASPACE_STANDARD_BT601_525;
+        break;
+      case MediaFormat.COLOR_STANDARD_BT2020:
+        dataSpace |= DATASPACE_STANDARD_BT2020;
+        break;
+      default:
+        break;
+    }
+    switch (transfer) {
+      case MediaFormat.COLOR_TRANSFER_LINEAR:
+        dataSpace |= DATASPACE_TRANSFER_LINEAR;
+        break;
+      case MediaFormat.COLOR_TRANSFER_SDR_VIDEO:
+        dataSpace |= DATASPACE_TRANSFER_SMPTE_170M;
+        break;
+      case MediaFormat.COLOR_TRANSFER_ST2084:
+        dataSpace |= DATASPACE_TRANSFER_ST2084;
+        break;
+      case MediaFormat.COLOR_TRANSFER_HLG:
+        dataSpace |= DATASPACE_TRANSFER_HLG;
+        break;
+      default:
+        break;
+    }
+    switch (range) {
+      case MediaFormat.COLOR_RANGE_FULL:
+        dataSpace |= DATASPACE_RANGE_FULL;
+        break;
+      case MediaFormat.COLOR_RANGE_LIMITED:
+        dataSpace |= DATASPACE_RANGE_LIMITED;
+        break;
+      default:
+        break;
+    }
+    return dataSpace;
+  }
+
+  /**
+   * The color space of a frame returned by {@link #copyLatestFrame}, as a data space (the
+   * packing of {@code android.hardware.DataSpace}): the one the decoder stamps on the frame
+   * (API 33), or what the container declares for the video track. 0 when neither says anything.
+   */
+  @DoNotStrip
+  public int frameDataSpace(Image image) {
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+      try {
+        int dataSpace = image.getDataSpace();
+        if (dataSpace != 0) {
+          return dataSpace;
+        }
+      } catch (IllegalStateException e) {
+        // The frame was closed under us; fall back to the container.
+      }
+    }
+    return containerDataSpace;
+  }
+
   // The decoder reports the frame size (possibly more than once for adaptive
   // streams); the ImageReader that receives the frames is sized accordingly.
   private void onVideoSizeChanged(int w, int h) {
@@ -273,6 +372,16 @@ public class WebGPUVideoPlayer {
     return image;
   }
 
+  /**
+   * The acquire fence of a frame returned by {@link #copyLatestFrame}, as a sync file descriptor
+   * the caller owns, or -1 when there is none. The decoder may still be writing the frame when it
+   * is acquired; the GPU work that samples it has to wait on this fence (see {@link SyncFences}).
+   */
+  @DoNotStrip
+  public static int acquireFenceFd(Image image) {
+    return SyncFences.acquireFenceFd(image);
+  }
+
   /** Hands the buffer of a frame returned by {@link #copyLatestFrame} back to the decoder. */
   @DoNotStrip
   public void closeImage(Image image) {

package/apple/ApplePlatformContext.h

@@ -17,7 +17,8 @@ public:
   ~ApplePlatformContext() = default;
 
   void snapshotView(const ViewSnapshotRequest &request,
-                    std::function<void(ImageData)> onSuccess,
+                    std::function<void(ImageData)> onPixels,
+                    std::function<void(VideoFrameHandle)> onFrame,
                     std::function<void(std::string)> onError) override;
 
   ImageData createImageBitmap(std::string blobId, double offset,
@@ -36,12 +37,11 @@ public:
 
   VideoFrameHandle loadVideoFrame(const std::string &path) override;
 
-  VideoFrameHandle createTestVideoFrame(uint32_t width,
-                                        uint32_t height) override;
+  VideoFrameHandle createTestVideoFrame(uint32_t width, uint32_t height,
+                                        VideoPixelFormat format) override;
 
   std::unique_ptr<IVideoPlayer>
-  createVideoPlayer(const std::string &path,
-                    VideoPixelFormat format) override;
+  createVideoPlayer(const std::string &path, VideoPixelFormat format) override;
 
   std::string writeTestVideoFile() override;