react-native-webgpu 0.14.1 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/android/cpp/AndroidColorSpace.h +107 -0
  2. package/android/cpp/AndroidPlatformContext.h +100 -9
  3. package/android/cpp/AndroidVideoPlayer.h +46 -1
  4. package/android/cpp/cpp-adapter.cpp +53 -1
  5. package/android/src/main/java/com/webgpu/SyncFences.java +88 -0
  6. package/android/src/main/java/com/webgpu/ViewSnapshot.java +66 -23
  7. package/android/src/main/java/com/webgpu/WebGPUModule.java +69 -5
  8. package/android/src/main/java/com/webgpu/WebGPUVideoPlayer.java +109 -0
  9. package/apple/ApplePlatformContext.h +5 -5
  10. package/apple/ApplePlatformContext.mm +75 -3
  11. package/apple/AppleVideoPlayer.mm +68 -35
  12. package/cpp/dawn/common/ExternalTextureParams.h +67 -0
  13. package/cpp/rnwgpu/PlatformContext.h +65 -16
  14. package/cpp/rnwgpu/SurfaceRegistry.h +39 -5
  15. package/cpp/rnwgpu/api/GPUAdapter.cpp +35 -0
  16. package/cpp/rnwgpu/api/GPUExternalTexture.cpp +172 -156
  17. package/cpp/rnwgpu/api/GPUExternalTexture.h +2 -19
  18. package/cpp/rnwgpu/api/GPUQueue.cpp +98 -3
  19. package/cpp/rnwgpu/api/RNWebGPU.h +12 -3
  20. package/cpp/rnwgpu/api/VideoFrameBlit.h +39 -24
  21. package/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h +10 -3
  22. package/lib/commonjs/Canvas.js +36 -6
  23. package/lib/commonjs/Canvas.js.map +1 -1
  24. package/lib/commonjs/WebPolyfillGPUModule.js +6 -3
  25. package/lib/commonjs/WebPolyfillGPUModule.js.map +1 -1
  26. package/lib/commonjs/constants.js +6 -3
  27. package/lib/commonjs/constants.js.map +1 -1
  28. package/lib/commonjs/index.js +1 -0
  29. package/lib/commonjs/index.js.map +1 -1
  30. package/lib/commonjs/install.js +9 -22
  31. package/lib/commonjs/install.js.map +1 -1
  32. package/lib/commonjs/main/index.js +8 -7
  33. package/lib/commonjs/main/index.js.map +1 -1
  34. package/lib/commonjs/main/index.web.js +1 -1
  35. package/lib/commonjs/main/installNative.js +20 -0
  36. package/lib/commonjs/main/installNative.js.map +1 -0
  37. package/lib/commonjs/main/installNative.web.js +4 -0
  38. package/lib/commonjs/main/installNative.web.js.map +1 -0
  39. package/lib/commonjs/mock.js +60 -13
  40. package/lib/commonjs/mock.js.map +1 -1
  41. package/lib/module/Canvas.js +37 -7
  42. package/lib/module/Canvas.js.map +1 -1
  43. package/lib/module/WebPolyfillGPUModule.js +6 -3
  44. package/lib/module/WebPolyfillGPUModule.js.map +1 -1
  45. package/lib/module/constants.js +6 -3
  46. package/lib/module/constants.js.map +1 -1
  47. package/lib/module/index.js +8 -1
  48. package/lib/module/index.js.map +1 -1
  49. package/lib/module/install.js +8 -20
  50. package/lib/module/install.js.map +1 -1
  51. package/lib/module/main/index.js +6 -7
  52. package/lib/module/main/index.js.map +1 -1
  53. package/lib/module/main/index.web.js +1 -1
  54. package/lib/module/main/index.web.js.map +1 -1
  55. package/lib/module/main/installNative.js +18 -0
  56. package/lib/module/main/installNative.js.map +1 -0
  57. package/lib/module/main/installNative.web.js +6 -0
  58. package/lib/module/main/installNative.web.js.map +1 -0
  59. package/lib/module/mock.js +55 -1
  60. package/lib/module/mock.js.map +1 -1
  61. package/lib/typescript/lib/commonjs/Canvas.d.ts.map +1 -1
  62. package/lib/typescript/lib/commonjs/install.d.ts +41 -3
  63. package/lib/typescript/lib/commonjs/install.d.ts.map +1 -1
  64. package/lib/typescript/lib/commonjs/main/installNative.d.ts +2 -0
  65. package/lib/typescript/lib/commonjs/main/installNative.d.ts.map +1 -0
  66. package/lib/typescript/lib/commonjs/main/installNative.web.d.ts +2 -0
  67. package/lib/typescript/lib/commonjs/main/installNative.web.d.ts.map +1 -0
  68. package/lib/typescript/lib/commonjs/mock.d.ts +39 -0
  69. package/lib/typescript/lib/commonjs/mock.d.ts.map +1 -1
  70. package/lib/typescript/lib/module/Canvas.d.ts.map +1 -1
  71. package/lib/typescript/lib/module/install.d.ts +0 -1
  72. package/lib/typescript/lib/module/install.d.ts.map +1 -1
  73. package/lib/typescript/lib/module/main/installNative.d.ts +2 -0
  74. package/lib/typescript/lib/module/main/installNative.d.ts.map +1 -0
  75. package/lib/typescript/lib/module/main/installNative.web.d.ts +2 -0
  76. package/lib/typescript/lib/module/main/installNative.web.d.ts.map +1 -0
  77. package/lib/typescript/lib/module/mock.d.ts +39 -1
  78. package/lib/typescript/lib/module/mock.d.ts.map +1 -1
  79. package/lib/typescript/src/Canvas.d.ts.map +1 -1
  80. package/lib/typescript/src/constants.d.ts.map +1 -1
  81. package/lib/typescript/src/index.d.ts +2 -2
  82. package/lib/typescript/src/index.d.ts.map +1 -1
  83. package/lib/typescript/src/install.d.ts +0 -5
  84. package/lib/typescript/src/install.d.ts.map +1 -1
  85. package/lib/typescript/src/main/index.d.ts +1 -0
  86. package/lib/typescript/src/main/index.d.ts.map +1 -1
  87. package/lib/typescript/src/main/index.web.d.ts +1 -1
  88. package/lib/typescript/src/main/index.web.d.ts.map +1 -1
  89. package/lib/typescript/src/main/installNative.d.ts +2 -0
  90. package/lib/typescript/src/main/installNative.d.ts.map +1 -0
  91. package/lib/typescript/src/main/installNative.web.d.ts +2 -0
  92. package/lib/typescript/src/main/installNative.web.d.ts.map +1 -0
  93. package/lib/typescript/src/mock.d.ts +5 -1
  94. package/lib/typescript/src/mock.d.ts.map +1 -1
  95. package/package.json +1 -1
  96. package/src/Canvas.tsx +48 -29
  97. package/src/WebPolyfillGPUModule.ts +3 -1
  98. package/src/constants.ts +6 -3
  99. package/src/index.tsx +18 -10
  100. package/src/install.ts +9 -18
  101. package/src/main/index.tsx +7 -10
  102. package/src/main/index.web.tsx +1 -1
  103. package/src/main/installNative.ts +20 -0
  104. package/src/main/installNative.web.ts +5 -0
  105. package/src/mock.ts +60 -1
@@ -0,0 +1,107 @@
1
+ #pragma once
2
+
3
+ #include <android/data_space.h>
4
+
5
+ #include <cstdint>
6
+
7
+ #include "PlatformContext.h"
8
+
9
+ namespace rnwgpu {
10
+
11
+ // The data spaces that predate the standard / transfer / range bit fields
12
+ // (HAL_DATASPACE_* in system/graphics.h, "legacy" since Android 6) as the
13
+ // field form the NDK header documents for them. Decoders still report them:
14
+ // a Pixel 8 on Android 17 tags every H.264 frame with the legacy BT.709
15
+ // value. The modern header no longer defines these constants.
16
+ inline int32_t canonicalDataSpace(int32_t dataSpace) {
17
+ switch (dataSpace) {
18
+ case 0x101: // HAL_DATASPACE_JFIF
19
+ return STANDARD_BT601_625 | TRANSFER_SMPTE_170M | RANGE_FULL;
20
+ case 0x102: // HAL_DATASPACE_BT601_625
21
+ return STANDARD_BT601_625 | TRANSFER_SMPTE_170M | RANGE_LIMITED;
22
+ case 0x103: // HAL_DATASPACE_BT601_525
23
+ return STANDARD_BT601_525 | TRANSFER_SMPTE_170M | RANGE_LIMITED;
24
+ case 0x104: // HAL_DATASPACE_BT709
25
+ return STANDARD_BT709 | TRANSFER_SMPTE_170M | RANGE_LIMITED;
26
+ case 0x200: // HAL_DATASPACE_SRGB_LINEAR
27
+ return STANDARD_BT709 | TRANSFER_LINEAR | RANGE_FULL;
28
+ case 0x201: // HAL_DATASPACE_SRGB
29
+ return STANDARD_BT709 | TRANSFER_SRGB | RANGE_FULL;
30
+ default:
31
+ return dataSpace;
32
+ }
33
+ }
34
+
35
+ // The color space a frame's data space declares (android/data_space.h, the
36
+ // value of Image.getDataSpace() or of the container's color info packed the
37
+ // same way). The standard gives the YCbCr matrix and the primaries; the
38
+ // transfer function and the range are what their names say. The fields a data
39
+ // space leaves unspecified stay unset, and ADATASPACE_UNKNOWN (0) sets nothing.
40
+ inline VideoFrameColorSpace colorSpaceFromDataSpace(int32_t dataSpace) {
41
+ VideoFrameColorSpace colorSpace;
42
+ dataSpace = canonicalDataSpace(dataSpace);
43
+ switch (dataSpace & STANDARD_MASK) {
44
+ case STANDARD_BT709:
45
+ colorSpace.matrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec709;
46
+ // BT.709 shares the sRGB primaries (Dawn's helper only knows them under
47
+ // that name).
48
+ colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::SRGB;
49
+ break;
50
+ case STANDARD_BT601_625:
51
+ case STANDARD_BT601_625_UNADJUSTED:
52
+ case STANDARD_BT601_525:
53
+ case STANDARD_BT601_525_UNADJUSTED:
54
+ colorSpace.matrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec601;
55
+ colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::Rec601;
56
+ break;
57
+ case STANDARD_BT2020:
58
+ case STANDARD_BT2020_CONSTANT_LUMINANCE:
59
+ colorSpace.matrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec2020;
60
+ colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::Rec2020;
61
+ break;
62
+ case STANDARD_DCI_P3:
63
+ colorSpace.matrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec709;
64
+ colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::DisplayP3;
65
+ break;
66
+ default:
67
+ break;
68
+ }
69
+ switch (dataSpace & TRANSFER_MASK) {
70
+ case TRANSFER_LINEAR:
71
+ colorSpace.transfer = wgpu::ColorSpaceTransferDawn::Identity;
72
+ break;
73
+ case TRANSFER_SRGB:
74
+ case TRANSFER_GAMMA2_2: // sRGB is the closest curve Dawn has
75
+ colorSpace.transfer = wgpu::ColorSpaceTransferDawn::SRGB;
76
+ break;
77
+ case TRANSFER_SMPTE_170M:
78
+ colorSpace.transfer = wgpu::ColorSpaceTransferDawn::SMPTE_170M;
79
+ break;
80
+ case TRANSFER_GAMMA2_6:
81
+ case TRANSFER_GAMMA2_8: // BT.1886 (gamma 2.4) is the closest
82
+ colorSpace.transfer = wgpu::ColorSpaceTransferDawn::BT_1886;
83
+ break;
84
+ case TRANSFER_ST2084:
85
+ colorSpace.transfer = wgpu::ColorSpaceTransferDawn::PQ;
86
+ break;
87
+ case TRANSFER_HLG:
88
+ colorSpace.transfer = wgpu::ColorSpaceTransferDawn::HLG;
89
+ break;
90
+ default:
91
+ break;
92
+ }
93
+ switch (dataSpace & RANGE_MASK) {
94
+ case RANGE_FULL:
95
+ case RANGE_EXTENDED:
96
+ colorSpace.range = wgpu::ColorSpaceYCbCrRangeDawn::Full;
97
+ break;
98
+ case RANGE_LIMITED:
99
+ colorSpace.range = wgpu::ColorSpaceYCbCrRangeDawn::Narrow;
100
+ break;
101
+ default:
102
+ break;
103
+ }
104
+ return colorSpace;
105
+ }
106
+
107
+ } // namespace rnwgpu
@@ -1,7 +1,9 @@
1
1
  #pragma once
2
2
 
3
3
  #include <android/bitmap.h>
4
+ #include <android/data_space.h>
4
5
  #include <android/hardware_buffer.h>
6
+ #include <dlfcn.h>
5
7
  #include <jni.h>
6
8
 
7
9
  #include <algorithm>
@@ -9,10 +11,12 @@
9
11
  #include <memory>
10
12
  #include <string>
11
13
  #include <thread>
14
+ #include <utility>
12
15
  #include <vector>
13
16
 
14
17
  #include "webgpu/webgpu_cpp.h"
15
18
 
19
+ #include "AndroidColorSpace.h"
16
20
  #include "AndroidVideoPlayer.h"
17
21
  #include "PlatformContext.h"
18
22
  #include "RNWebGPUManager.h"
@@ -24,9 +28,10 @@ namespace jni = facebook::jni;
24
28
 
25
29
  // The completion callbacks of one snapshotView request, heap-allocated and
26
30
  // carried through Java as an opaque pointer (WebGPUModule.snapshotView ->
27
- // onViewSnapshot), which hands them back exactly once.
31
+ // onViewSnapshot / onViewSnapshotBuffer), which hands them back exactly once.
28
32
  struct ViewSnapshotCallbacks {
29
- std::function<void(ImageData)> onSuccess;
33
+ std::function<void(ImageData)> onPixels;
34
+ std::function<void(VideoFrameHandle)> onFrame;
30
35
  std::function<void(std::string)> onError;
31
36
  };
32
37
 
@@ -90,7 +95,8 @@ public:
90
95
  }
91
96
 
92
97
  void snapshotView(const ViewSnapshotRequest &request,
93
- std::function<void(ImageData)> onSuccess,
98
+ std::function<void(ImageData)> onPixels,
99
+ std::function<void(VideoFrameHandle)> onFrame,
94
100
  std::function<void(std::string)> onError) override {
95
101
  if (!_module) {
96
102
  onError("drawElementImageToTexture: the WebGPU module is gone");
@@ -104,7 +110,7 @@ public:
104
110
  }
105
111
  jclass moduleClass = env->GetObjectClass(_module);
106
112
  jmethodID method =
107
- env->GetMethodID(moduleClass, "snapshotView", "(IDDDDIIJ)V");
113
+ env->GetMethodID(moduleClass, "snapshotView", "(IDDDDIIZJ)V");
108
114
  env->DeleteLocalRef(moduleClass);
109
115
  if (!method) {
110
116
  env->ExceptionClear();
@@ -112,9 +118,9 @@ public:
112
118
  return;
113
119
  }
114
120
  // Ownership passes to Java; Java_com_webgpu_WebGPUModule_onViewSnapshot
115
- // (cpp-adapter.cpp) reclaims it.
116
- auto *callbacks =
117
- new ViewSnapshotCallbacks{std::move(onSuccess), std::move(onError)};
121
+ // or onViewSnapshotBuffer (cpp-adapter.cpp) reclaims it.
122
+ auto *callbacks = new ViewSnapshotCallbacks{
123
+ std::move(onPixels), std::move(onFrame), std::move(onError)};
118
124
  env->CallVoidMethod(_module, method, static_cast<jint>(request.viewTag),
119
125
  static_cast<jdouble>(request.sourceX),
120
126
  static_cast<jdouble>(request.sourceY),
@@ -122,6 +128,7 @@ public:
122
128
  static_cast<jdouble>(request.sourceHeight),
123
129
  static_cast<jint>(request.width),
124
130
  static_cast<jint>(request.height),
131
+ static_cast<jboolean>(request.allowFrame),
125
132
  reinterpret_cast<jlong>(callbacks));
126
133
  if (env->ExceptionCheck()) {
127
134
  env->ExceptionClear();
@@ -268,8 +275,92 @@ public:
268
275
  "device.importSharedTextureMemory.");
269
276
  }
270
277
 
271
- VideoFrameHandle createTestVideoFrame(uint32_t width,
272
- uint32_t height) override {
278
+ // The NV12 variant of createTestVideoFrame: a Y8Cb8Cr8_420 buffer, the YUV
279
+ // layout every gralloc supports, BT.709 limited range, red on the left half
280
+ // and green on the right. Filling it needs AHardwareBuffer_lockPlanes
281
+ // (API 29), resolved at run time since the library runs from API 26.
282
+ VideoFrameHandle createTestYuvFrame(uint32_t width, uint32_t height) {
283
+ using LockPlanes = int (*)(AHardwareBuffer *, uint64_t, int32_t,
284
+ const ARect *, AHardwareBuffer_Planes *);
285
+ static const auto lockPlanes = reinterpret_cast<LockPlanes>(
286
+ dlsym(RTLD_DEFAULT, "AHardwareBuffer_lockPlanes"));
287
+ if (lockPlanes == nullptr) {
288
+ throw std::runtime_error(
289
+ "createTestVideoFrame: unsupported: a YUV test frame needs "
290
+ "AHardwareBuffer_lockPlanes (API 29)");
291
+ }
292
+
293
+ AHardwareBuffer_Desc desc = {};
294
+ desc.width = width;
295
+ desc.height = height;
296
+ desc.layers = 1;
297
+ desc.format = AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420;
298
+ desc.usage = AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE |
299
+ AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY |
300
+ AHARDWAREBUFFER_USAGE_CPU_READ_RARELY;
301
+ AHardwareBuffer *buffer = nullptr;
302
+ int err = AHardwareBuffer_allocate(&desc, &buffer);
303
+ if (err != 0 || !buffer) {
304
+ throw std::runtime_error(
305
+ "createTestVideoFrame: unsupported: AHardwareBuffer_allocate failed "
306
+ "for a Y8Cb8Cr8_420 buffer (" +
307
+ std::to_string(err) + ")");
308
+ }
309
+
310
+ AHardwareBuffer_Planes planes = {};
311
+ int rc = lockPlanes(buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, -1,
312
+ nullptr, &planes);
313
+ if (rc != 0 || planes.planeCount < 3) {
314
+ AHardwareBuffer_release(buffer);
315
+ throw std::runtime_error(
316
+ "createTestVideoFrame: unsupported: AHardwareBuffer_lockPlanes "
317
+ "failed (" +
318
+ std::to_string(rc) + ")");
319
+ }
320
+ // Plane 0 is Y, 1 is Cb, 2 is Cr; the chroma planes are subsampled by two
321
+ // on both axes and the strides say whether Cb and Cr interleave.
322
+ const auto &y = planes.planes[0];
323
+ const auto &cb = planes.planes[1];
324
+ const auto &cr = planes.planes[2];
325
+ for (uint32_t row = 0; row < height; ++row) {
326
+ auto *line = static_cast<uint8_t *>(y.data) + row * y.rowStride;
327
+ for (uint32_t col = 0; col < width; ++col) {
328
+ line[col * y.pixelStride] =
329
+ col < width / 2 ? test_yuv_frame::kRedY : test_yuv_frame::kGreenY;
330
+ }
331
+ }
332
+ const uint32_t chromaWidth = (width + 1) / 2;
333
+ const uint32_t chromaHeight = (height + 1) / 2;
334
+ for (uint32_t row = 0; row < chromaHeight; ++row) {
335
+ auto *cbLine = static_cast<uint8_t *>(cb.data) + row * cb.rowStride;
336
+ auto *crLine = static_cast<uint8_t *>(cr.data) + row * cr.rowStride;
337
+ for (uint32_t col = 0; col < chromaWidth; ++col) {
338
+ bool red = col < chromaWidth / 2;
339
+ cbLine[col * cb.pixelStride] =
340
+ red ? test_yuv_frame::kRedCb : test_yuv_frame::kGreenCb;
341
+ crLine[col * cr.pixelStride] =
342
+ red ? test_yuv_frame::kRedCr : test_yuv_frame::kGreenCr;
343
+ }
344
+ }
345
+ AHardwareBuffer_unlock(buffer, nullptr);
346
+
347
+ VideoFrameHandle handle;
348
+ handle.handle = static_cast<void *>(buffer);
349
+ handle.width = width;
350
+ handle.height = height;
351
+ handle.pixelFormat = VideoPixelFormat::NV12;
352
+ // What a BT.709 SDR video declares: ADATASPACE_BT709 is STANDARD_BT709 |
353
+ // TRANSFER_SMPTE_170M | RANGE_LIMITED.
354
+ handle.colorSpace = colorSpaceFromDataSpace(ADATASPACE_BT709);
355
+ handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); };
356
+ return handle;
357
+ }
358
+
359
+ VideoFrameHandle createTestVideoFrame(uint32_t width, uint32_t height,
360
+ VideoPixelFormat format) override {
361
+ if (format == VideoPixelFormat::NV12) {
362
+ return createTestYuvFrame(width, height);
363
+ }
273
364
  // Dawn's Android backend already requires API 26+, so AHardwareBuffer_*
274
365
  // symbols are guaranteed to be available at link time on supported builds.
275
366
  AHardwareBuffer_Desc desc = {};
@@ -4,12 +4,14 @@
4
4
  #include <android/hardware_buffer_jni.h>
5
5
  #include <fbjni/fbjni.h>
6
6
  #include <jni.h>
7
+ #include <unistd.h>
7
8
 
8
9
  #include <memory>
9
10
  #include <stdexcept>
10
11
  #include <string>
11
12
  #include <utility>
12
13
 
14
+ #include "AndroidColorSpace.h"
13
15
  #include "PlatformContext.h"
14
16
 
15
17
  namespace rnwgpu {
@@ -94,6 +96,17 @@ public:
94
96
  _getRotation = method(env, playerClass, "getRotation", "()I");
95
97
  _getFrameRate = method(env, playerClass, "getFrameRate", "()D");
96
98
  _release = method(env, playerClass, "release", "()V");
99
+ _frameDataSpace =
100
+ method(env, playerClass, "frameDataSpace", "(Landroid/media/Image;)I");
101
+ _acquireFenceFd = env->GetStaticMethodID(playerClass, "acquireFenceFd",
102
+ "(Landroid/media/Image;)I");
103
+ if (_acquireFenceFd == nullptr) {
104
+ env->ExceptionClear();
105
+ env->DeleteLocalRef(playerClass);
106
+ throw std::runtime_error("VideoPlayer: method acquireFenceFd not found");
107
+ }
108
+ // The static call needs the class; keep it for the player's lifetime.
109
+ _playerClass = static_cast<jclass>(env->NewGlobalRef(playerClass));
97
110
  env->DeleteLocalRef(playerClass);
98
111
 
99
112
  jclass imageClass = env->FindClass("android/media/Image");
@@ -108,6 +121,7 @@ public:
108
121
  if (env->ExceptionCheck()) {
109
122
  env->ExceptionClear();
110
123
  }
124
+ env->DeleteGlobalRef(_playerClass);
111
125
  }
112
126
 
113
127
  VideoFrameHandle copyLatestFrame() override {
@@ -144,6 +158,28 @@ public:
144
158
  "them");
145
159
  }
146
160
 
161
+ // The decoder may still be writing the frame: ImageReader hands out the
162
+ // buffer as soon as it is queued and only waits on the producer's fence for
163
+ // CPU access (Image.getPlanes), not for Image.getHardwareBuffer. The GPU
164
+ // work that samples the frame waits on the fence instead (see
165
+ // GPUExternalTexture::Create). -1 when the platform cannot report it
166
+ // (before API 33) or the producer was already done.
167
+ int acquireFenceFd =
168
+ env->CallStaticIntMethod(_playerClass, _acquireFenceFd, image);
169
+ if (env->ExceptionCheck()) {
170
+ env->ExceptionClear();
171
+ acquireFenceFd = -1;
172
+ }
173
+
174
+ // The color space of the samples: the frame's data space (API 33) or the
175
+ // container's color info. What it leaves unknown is resolved at import.
176
+ int32_t dataSpace =
177
+ env->CallIntMethod(_player->get(), _frameDataSpace, image);
178
+ if (env->ExceptionCheck()) {
179
+ env->ExceptionClear();
180
+ dataSpace = 0;
181
+ }
182
+
147
183
  // The frame owns a reference to the buffer and keeps the Image acquired
148
184
  // until it is released; closing the Image hands the buffer back to the
149
185
  // decoder.
@@ -157,9 +193,12 @@ public:
157
193
  handle.height = desc.height;
158
194
  handle.pixelFormat = isRgbFormat(desc.format) ? VideoPixelFormat::BGRA8
159
195
  : VideoPixelFormat::NV12;
196
+ handle.acquireFenceFd = acquireFenceFd;
197
+ handle.colorSpace = colorSpaceFromDataSpace(dataSpace);
160
198
  auto player = _player;
161
199
  jmethodID closeImageMethod = _closeImage;
162
- handle.deleter = [player, closeImageMethod, imageRef, buffer]() {
200
+ handle.deleter = [player, closeImageMethod, imageRef, buffer,
201
+ acquireFenceFd]() {
163
202
  JNIEnv *env = attach();
164
203
  env->CallVoidMethod(player->get(), closeImageMethod, imageRef);
165
204
  if (env->ExceptionCheck()) {
@@ -167,6 +206,9 @@ public:
167
206
  }
168
207
  env->DeleteGlobalRef(imageRef);
169
208
  AHardwareBuffer_release(buffer);
209
+ if (acquireFenceFd >= 0) {
210
+ ::close(acquireFenceFd);
211
+ }
170
212
  };
171
213
  return handle;
172
214
  }
@@ -284,8 +326,11 @@ private:
284
326
  }
285
327
 
286
328
  std::shared_ptr<JavaVideoPlayerRef> _player;
329
+ jclass _playerClass;
287
330
  jmethodID _copyLatestFrame;
288
331
  jmethodID _closeImage;
332
+ jmethodID _frameDataSpace;
333
+ jmethodID _acquireFenceFd;
289
334
  jmethodID _play;
290
335
  jmethodID _pause;
291
336
  jmethodID _getPaused;
@@ -1,3 +1,5 @@
1
+ #include <unistd.h>
2
+
1
3
  #include <cstring>
2
4
  #include <memory>
3
5
  #include <string>
@@ -92,7 +94,57 @@ extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUModule_onViewSnapshot(
92
94
  rowBytes);
93
95
  }
94
96
  AndroidBitmap_unlockPixels(env, bitmap);
95
- callbacks->onSuccess(std::move(image));
97
+ callbacks->onPixels(std::move(image));
98
+ }
99
+
100
+ // Completion of WebGPUModule.snapshotView through a hardware buffer (UI
101
+ // thread, Android 13 and above): the buffer HardwareRenderer drew the view
102
+ // into, and the fence that signals when the drawing is done, as a sync fd
103
+ // this function takes ownership of. The GPU draws the buffer into the
104
+ // destination itself (GPUQueue::drawElementImageToTexture); nothing is read
105
+ // back through the CPU.
106
+ extern "C" JNIEXPORT void JNICALL
107
+ Java_com_webgpu_WebGPUModule_onViewSnapshotBuffer(JNIEnv *env,
108
+ jobject /* this */,
109
+ jlong callbackPointer,
110
+ jobject hardwareBuffer,
111
+ jint fenceFd) {
112
+ std::unique_ptr<rnwgpu::ViewSnapshotCallbacks> callbacks(
113
+ reinterpret_cast<rnwgpu::ViewSnapshotCallbacks *>(callbackPointer));
114
+ AHardwareBuffer *buffer =
115
+ hardwareBuffer != nullptr
116
+ ? AHardwareBuffer_fromHardwareBuffer(env, hardwareBuffer)
117
+ : nullptr;
118
+ if (!callbacks || buffer == nullptr) {
119
+ if (fenceFd >= 0) {
120
+ ::close(fenceFd);
121
+ }
122
+ if (callbacks) {
123
+ callbacks->onError(
124
+ "drawElementImageToTexture: the renderer produced no buffer");
125
+ }
126
+ return;
127
+ }
128
+ AHardwareBuffer_Desc desc{};
129
+ AHardwareBuffer_describe(buffer, &desc);
130
+ // Java closes its Image and HardwareBuffer as soon as this returns; this
131
+ // reference keeps the memory alive until the frame is released.
132
+ AHardwareBuffer_acquire(buffer);
133
+
134
+ rnwgpu::VideoFrameHandle frame;
135
+ frame.handle = buffer;
136
+ frame.width = desc.width;
137
+ frame.height = desc.height;
138
+ // An RGBA_8888 buffer: sampled as a plain color texture.
139
+ frame.pixelFormat = rnwgpu::VideoPixelFormat::BGRA8;
140
+ frame.acquireFenceFd = fenceFd;
141
+ frame.deleter = [buffer, fenceFd]() {
142
+ AHardwareBuffer_release(buffer);
143
+ if (fenceFd >= 0) {
144
+ ::close(fenceFd);
145
+ }
146
+ };
147
+ callbacks->onFrame(std::move(frame));
96
148
  }
97
149
 
98
150
  extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUView_onSurfaceChanged(
@@ -0,0 +1,88 @@
1
+ package com.webgpu;
2
+
3
+ import android.hardware.SyncFence;
4
+ import android.media.Image;
5
+ import android.os.Build;
6
+ import android.os.Parcel;
7
+ import android.os.ParcelFileDescriptor;
8
+ import android.util.Log;
9
+
10
+ import androidx.annotation.NonNull;
11
+ import androidx.annotation.RequiresApi;
12
+
13
+ import java.io.IOException;
14
+
15
+ /**
16
+ * Bridges the platform's {@link SyncFence} to the sync file descriptors Dawn imports as
17
+ * {@code shared-fence-sync-fd} fences.
18
+ *
19
+ * <p>A buffer handed out by an {@link android.media.ImageReader} carries an acquire fence: the
20
+ * producer (the video decoder, the camera, a {@link android.graphics.HardwareRenderer}) may still
21
+ * be writing it when the consumer acquires it. The framework waits on that fence for CPU access
22
+ * ({@link Image#getPlanes()}), but not for {@link Image#getHardwareBuffer()}, so a GPU consumer has
23
+ * to wait on it itself. Since API 33 the fence is exposed as {@link Image#getFence()}.
24
+ */
25
+ final class SyncFences {
26
+ private static final String TAG = "WebGPUSyncFences";
27
+
28
+ private SyncFences() {}
29
+
30
+ /**
31
+ * The acquire fence of {@code image} as a sync file descriptor the caller owns and must close,
32
+ * or -1 when the frame has none (the producer was done when it was acquired, or the platform
33
+ * predates {@link Image#getFence()}).
34
+ */
35
+ static int acquireFenceFd(@NonNull Image image) {
36
+ if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) {
37
+ return -1;
38
+ }
39
+ return acquireFenceFdApi33(image);
40
+ }
41
+
42
+ @RequiresApi(Build.VERSION_CODES.TIRAMISU)
43
+ private static int acquireFenceFdApi33(@NonNull Image image) {
44
+ SyncFence fence;
45
+ try {
46
+ fence = image.getFence();
47
+ } catch (IOException | RuntimeException e) {
48
+ Log.w(TAG, "could not read the acquire fence of the frame: " + e.getMessage());
49
+ return -1;
50
+ }
51
+ try {
52
+ return detachFd(fence);
53
+ } finally {
54
+ // getFence() returns a duplicate; the detached descriptor is ours, this one goes.
55
+ fence.close();
56
+ }
57
+ }
58
+
59
+ /**
60
+ * The file descriptor of {@code fence}, duplicated and detached, so it stays valid after the
61
+ * fence is closed and must be closed by the caller. -1 for an invalid fence.
62
+ *
63
+ * <p>{@link SyncFence} has no public accessor for its descriptor. Writing the fence to a {@link
64
+ * Parcel} and reading it back yields one: the parcel form is a validity flag followed by the
65
+ * file descriptor.
66
+ */
67
+ @RequiresApi(Build.VERSION_CODES.TIRAMISU)
68
+ static int detachFd(@NonNull SyncFence fence) {
69
+ if (!fence.isValid()) {
70
+ return -1;
71
+ }
72
+ Parcel parcel = Parcel.obtain();
73
+ try {
74
+ fence.writeToParcel(parcel, 0);
75
+ parcel.setDataPosition(0);
76
+ if (!parcel.readBoolean()) {
77
+ return -1;
78
+ }
79
+ ParcelFileDescriptor descriptor = parcel.readFileDescriptor();
80
+ return descriptor == null ? -1 : descriptor.detachFd();
81
+ } catch (RuntimeException e) {
82
+ Log.w(TAG, "could not read the file descriptor of a SyncFence: " + e.getMessage());
83
+ return -1;
84
+ } finally {
85
+ parcel.recycle();
86
+ }
87
+ }
88
+ }
@@ -24,7 +24,10 @@ import androidx.annotation.RequiresApi;
24
24
  * On API 29+ the view is rendered through {@link HardwareRenderer}: the
25
25
  * recording reuses the children's existing display lists, so the result is
26
26
  * exactly what the window's own renderer draws (shadows, hardware layers,
27
- * TextureView content). Older devices, or a failed hardware render, fall back
27
+ * TextureView content). The renderer draws into a GPU-sampleable buffer,
28
+ * which {@link #renderHardware} hands to its caller as is: the WebGPU side
29
+ * imports it directly when it can, and {@link #snapshot} reads it back into
30
+ * a bitmap otherwise. Older devices, or a failed hardware render, fall back
28
31
  * to drawing the view into a software canvas, which misses anything that is
29
32
  * only drawn hardware-accelerated. SurfaceView content is composed outside
30
33
  * the view hierarchy and is captured by neither path.
@@ -34,6 +37,11 @@ final class ViewSnapshot {
34
37
 
35
38
  private ViewSnapshot() {}
36
39
 
40
+ /** What a caller does with the frame the hardware renderer drew, while it is acquired. */
41
+ interface FrameConsumer<T> {
42
+ T consume(@NonNull Image image) throws Exception;
43
+ }
44
+
37
45
  /**
38
46
  * @param sourceX/Y/Width/Height the source rectangle in the view's own
39
47
  * coordinate space, in pixels (already scaled by the display density)
@@ -77,19 +85,27 @@ final class ViewSnapshot {
77
85
  return bitmap;
78
86
  }
79
87
 
88
+ /**
89
+ * Renders the view through {@link HardwareRenderer} into a GPU-sampleable
90
+ * RGBA_8888 buffer of exactly width x height pixels and hands the acquired
91
+ * frame to {@code consumer}; the reader and the renderer are torn down when
92
+ * it returns, so a consumer that keeps the buffer takes its own reference
93
+ * first. The renderer only submits the frame: the drawing may still be in
94
+ * flight on the GPU when the consumer runs. {@link Image#getFence()} (API 33)
95
+ * tells when it is done; a CPU readback of the buffer waits for it on its
96
+ * own.
97
+ */
80
98
  @RequiresApi(Build.VERSION_CODES.Q)
81
- @NonNull
82
- private static Bitmap snapshotHardware(
99
+ static <T> T renderHardware(
83
100
  @NonNull View view,
84
101
  float sourceX,
85
102
  float sourceY,
86
103
  float sourceWidth,
87
104
  float sourceHeight,
88
105
  int width,
89
- int height) {
90
- // The renderer draws into an ImageReader whose buffers are GPU-sampleable
91
- // (a later zero-copy path can import them directly); for now the frame is
92
- // read back into a software bitmap.
106
+ int height,
107
+ @NonNull FrameConsumer<T> consumer)
108
+ throws Exception {
93
109
  ImageReader reader =
94
110
  ImageReader.newInstance(
95
111
  width,
@@ -135,22 +151,7 @@ final class ViewSnapshot {
135
151
  if (image == null) {
136
152
  throw new IllegalStateException("HardwareRenderer produced no image");
137
153
  }
138
- HardwareBuffer buffer = image.getHardwareBuffer();
139
- if (buffer == null) {
140
- throw new IllegalStateException("HardwareRenderer image has no HardwareBuffer");
141
- }
142
- Bitmap hardware =
143
- Bitmap.wrapHardwareBuffer(buffer, ColorSpace.get(ColorSpace.Named.SRGB));
144
- buffer.close();
145
- if (hardware == null) {
146
- throw new IllegalStateException("Bitmap.wrapHardwareBuffer failed");
147
- }
148
- Bitmap software = hardware.copy(Bitmap.Config.ARGB_8888, false);
149
- hardware.recycle();
150
- if (software == null) {
151
- throw new IllegalStateException("could not read back the hardware bitmap");
152
- }
153
- return software;
154
+ return consumer.consume(image);
154
155
  } finally {
155
156
  if (image != null) {
156
157
  image.close();
@@ -159,4 +160,46 @@ final class ViewSnapshot {
159
160
  reader.close();
160
161
  }
161
162
  }
163
+
164
+ @RequiresApi(Build.VERSION_CODES.Q)
165
+ @NonNull
166
+ private static Bitmap snapshotHardware(
167
+ @NonNull View view,
168
+ float sourceX,
169
+ float sourceY,
170
+ float sourceWidth,
171
+ float sourceHeight,
172
+ int width,
173
+ int height)
174
+ throws Exception {
175
+ return renderHardware(
176
+ view,
177
+ sourceX,
178
+ sourceY,
179
+ sourceWidth,
180
+ sourceHeight,
181
+ width,
182
+ height,
183
+ image -> {
184
+ HardwareBuffer buffer = image.getHardwareBuffer();
185
+ if (buffer == null) {
186
+ throw new IllegalStateException("HardwareRenderer image has no HardwareBuffer");
187
+ }
188
+ // wrapHardwareBuffer takes its own reference, so the wrapper can be
189
+ // closed immediately; the copy reads the pixels back into software
190
+ // memory, waiting for the renderer to finish drawing.
191
+ Bitmap hardware =
192
+ Bitmap.wrapHardwareBuffer(buffer, ColorSpace.get(ColorSpace.Named.SRGB));
193
+ buffer.close();
194
+ if (hardware == null) {
195
+ throw new IllegalStateException("Bitmap.wrapHardwareBuffer failed");
196
+ }
197
+ Bitmap software = hardware.copy(Bitmap.Config.ARGB_8888, false);
198
+ hardware.recycle();
199
+ if (software == null) {
200
+ throw new IllegalStateException("could not read back the hardware bitmap");
201
+ }
202
+ return software;
203
+ });
204
+ }
162
205
  }