react-native-webgpu 0.14.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/cpp/AndroidColorSpace.h +107 -0
- package/android/cpp/AndroidPlatformContext.h +100 -9
- package/android/cpp/AndroidVideoPlayer.h +46 -1
- package/android/cpp/cpp-adapter.cpp +53 -1
- package/android/src/main/java/com/webgpu/SyncFences.java +88 -0
- package/android/src/main/java/com/webgpu/ViewSnapshot.java +66 -23
- package/android/src/main/java/com/webgpu/WebGPUModule.java +69 -5
- package/android/src/main/java/com/webgpu/WebGPUVideoPlayer.java +109 -0
- package/apple/ApplePlatformContext.h +5 -5
- package/apple/ApplePlatformContext.mm +75 -3
- package/apple/AppleVideoPlayer.mm +68 -35
- package/cpp/dawn/common/ExternalTextureParams.h +67 -0
- package/cpp/rnwgpu/PlatformContext.h +65 -16
- package/cpp/rnwgpu/SurfaceRegistry.h +39 -5
- package/cpp/rnwgpu/api/GPUAdapter.cpp +35 -0
- package/cpp/rnwgpu/api/GPUExternalTexture.cpp +172 -156
- package/cpp/rnwgpu/api/GPUExternalTexture.h +2 -19
- package/cpp/rnwgpu/api/GPUQueue.cpp +98 -3
- package/cpp/rnwgpu/api/RNWebGPU.h +12 -3
- package/cpp/rnwgpu/api/VideoFrameBlit.h +39 -24
- package/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h +10 -3
- package/lib/commonjs/Canvas.js +36 -6
- package/lib/commonjs/Canvas.js.map +1 -1
- package/lib/commonjs/WebPolyfillGPUModule.js +6 -3
- package/lib/commonjs/WebPolyfillGPUModule.js.map +1 -1
- package/lib/commonjs/constants.js +6 -3
- package/lib/commonjs/constants.js.map +1 -1
- package/lib/commonjs/index.js +1 -0
- package/lib/commonjs/index.js.map +1 -1
- package/lib/commonjs/install.js +9 -22
- package/lib/commonjs/install.js.map +1 -1
- package/lib/commonjs/main/index.js +8 -7
- package/lib/commonjs/main/index.js.map +1 -1
- package/lib/commonjs/main/index.web.js +1 -1
- package/lib/commonjs/main/installNative.js +20 -0
- package/lib/commonjs/main/installNative.js.map +1 -0
- package/lib/commonjs/main/installNative.web.js +4 -0
- package/lib/commonjs/main/installNative.web.js.map +1 -0
- package/lib/commonjs/mock.js +60 -13
- package/lib/commonjs/mock.js.map +1 -1
- package/lib/module/Canvas.js +37 -7
- package/lib/module/Canvas.js.map +1 -1
- package/lib/module/WebPolyfillGPUModule.js +6 -3
- package/lib/module/WebPolyfillGPUModule.js.map +1 -1
- package/lib/module/constants.js +6 -3
- package/lib/module/constants.js.map +1 -1
- package/lib/module/index.js +8 -1
- package/lib/module/index.js.map +1 -1
- package/lib/module/install.js +8 -20
- package/lib/module/install.js.map +1 -1
- package/lib/module/main/index.js +6 -7
- package/lib/module/main/index.js.map +1 -1
- package/lib/module/main/index.web.js +1 -1
- package/lib/module/main/index.web.js.map +1 -1
- package/lib/module/main/installNative.js +18 -0
- package/lib/module/main/installNative.js.map +1 -0
- package/lib/module/main/installNative.web.js +6 -0
- package/lib/module/main/installNative.web.js.map +1 -0
- package/lib/module/mock.js +55 -1
- package/lib/module/mock.js.map +1 -1
- package/lib/typescript/lib/commonjs/Canvas.d.ts.map +1 -1
- package/lib/typescript/lib/commonjs/install.d.ts +41 -3
- package/lib/typescript/lib/commonjs/install.d.ts.map +1 -1
- package/lib/typescript/lib/commonjs/main/installNative.d.ts +2 -0
- package/lib/typescript/lib/commonjs/main/installNative.d.ts.map +1 -0
- package/lib/typescript/lib/commonjs/main/installNative.web.d.ts +2 -0
- package/lib/typescript/lib/commonjs/main/installNative.web.d.ts.map +1 -0
- package/lib/typescript/lib/commonjs/mock.d.ts +39 -0
- package/lib/typescript/lib/commonjs/mock.d.ts.map +1 -1
- package/lib/typescript/lib/module/Canvas.d.ts.map +1 -1
- package/lib/typescript/lib/module/install.d.ts +0 -1
- package/lib/typescript/lib/module/install.d.ts.map +1 -1
- package/lib/typescript/lib/module/main/installNative.d.ts +2 -0
- package/lib/typescript/lib/module/main/installNative.d.ts.map +1 -0
- package/lib/typescript/lib/module/main/installNative.web.d.ts +2 -0
- package/lib/typescript/lib/module/main/installNative.web.d.ts.map +1 -0
- package/lib/typescript/lib/module/mock.d.ts +39 -1
- package/lib/typescript/lib/module/mock.d.ts.map +1 -1
- package/lib/typescript/src/Canvas.d.ts.map +1 -1
- package/lib/typescript/src/constants.d.ts.map +1 -1
- package/lib/typescript/src/index.d.ts +2 -2
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/install.d.ts +0 -5
- package/lib/typescript/src/install.d.ts.map +1 -1
- package/lib/typescript/src/main/index.d.ts +1 -0
- package/lib/typescript/src/main/index.d.ts.map +1 -1
- package/lib/typescript/src/main/index.web.d.ts +1 -1
- package/lib/typescript/src/main/index.web.d.ts.map +1 -1
- package/lib/typescript/src/main/installNative.d.ts +2 -0
- package/lib/typescript/src/main/installNative.d.ts.map +1 -0
- package/lib/typescript/src/main/installNative.web.d.ts +2 -0
- package/lib/typescript/src/main/installNative.web.d.ts.map +1 -0
- package/lib/typescript/src/mock.d.ts +5 -1
- package/lib/typescript/src/mock.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/Canvas.tsx +48 -29
- package/src/WebPolyfillGPUModule.ts +3 -1
- package/src/constants.ts +6 -3
- package/src/index.tsx +18 -10
- package/src/install.ts +9 -18
- package/src/main/index.tsx +7 -10
- package/src/main/index.web.tsx +1 -1
- package/src/main/installNative.ts +20 -0
- package/src/main/installNative.web.ts +5 -0
- package/src/mock.ts +60 -1
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <android/data_space.h>
|
|
4
|
+
|
|
5
|
+
#include <cstdint>
|
|
6
|
+
|
|
7
|
+
#include "PlatformContext.h"
|
|
8
|
+
|
|
9
|
+
namespace rnwgpu {
|
|
10
|
+
|
|
11
|
+
// The data spaces that predate the standard / transfer / range bit fields
|
|
12
|
+
// (HAL_DATASPACE_* in system/graphics.h, "legacy" since Android 6) as the
|
|
13
|
+
// field form the NDK header documents for them. Decoders still report them:
|
|
14
|
+
// a Pixel 8 on Android 17 tags every H.264 frame with the legacy BT.709
|
|
15
|
+
// value. The modern header no longer defines these constants.
|
|
16
|
+
inline int32_t canonicalDataSpace(int32_t dataSpace) {
|
|
17
|
+
switch (dataSpace) {
|
|
18
|
+
case 0x101: // HAL_DATASPACE_JFIF
|
|
19
|
+
return STANDARD_BT601_625 | TRANSFER_SMPTE_170M | RANGE_FULL;
|
|
20
|
+
case 0x102: // HAL_DATASPACE_BT601_625
|
|
21
|
+
return STANDARD_BT601_625 | TRANSFER_SMPTE_170M | RANGE_LIMITED;
|
|
22
|
+
case 0x103: // HAL_DATASPACE_BT601_525
|
|
23
|
+
return STANDARD_BT601_525 | TRANSFER_SMPTE_170M | RANGE_LIMITED;
|
|
24
|
+
case 0x104: // HAL_DATASPACE_BT709
|
|
25
|
+
return STANDARD_BT709 | TRANSFER_SMPTE_170M | RANGE_LIMITED;
|
|
26
|
+
case 0x200: // HAL_DATASPACE_SRGB_LINEAR
|
|
27
|
+
return STANDARD_BT709 | TRANSFER_LINEAR | RANGE_FULL;
|
|
28
|
+
case 0x201: // HAL_DATASPACE_SRGB
|
|
29
|
+
return STANDARD_BT709 | TRANSFER_SRGB | RANGE_FULL;
|
|
30
|
+
default:
|
|
31
|
+
return dataSpace;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// The color space a frame's data space declares (android/data_space.h, the
|
|
36
|
+
// value of Image.getDataSpace() or of the container's color info packed the
|
|
37
|
+
// same way). The standard gives the YCbCr matrix and the primaries; the
|
|
38
|
+
// transfer function and the range are what their names say. The fields a data
|
|
39
|
+
// space leaves unspecified stay unset, and ADATASPACE_UNKNOWN (0) sets nothing.
|
|
40
|
+
inline VideoFrameColorSpace colorSpaceFromDataSpace(int32_t dataSpace) {
|
|
41
|
+
VideoFrameColorSpace colorSpace;
|
|
42
|
+
dataSpace = canonicalDataSpace(dataSpace);
|
|
43
|
+
switch (dataSpace & STANDARD_MASK) {
|
|
44
|
+
case STANDARD_BT709:
|
|
45
|
+
colorSpace.matrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec709;
|
|
46
|
+
// BT.709 shares the sRGB primaries (Dawn's helper only knows them under
|
|
47
|
+
// that name).
|
|
48
|
+
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::SRGB;
|
|
49
|
+
break;
|
|
50
|
+
case STANDARD_BT601_625:
|
|
51
|
+
case STANDARD_BT601_625_UNADJUSTED:
|
|
52
|
+
case STANDARD_BT601_525:
|
|
53
|
+
case STANDARD_BT601_525_UNADJUSTED:
|
|
54
|
+
colorSpace.matrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec601;
|
|
55
|
+
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::Rec601;
|
|
56
|
+
break;
|
|
57
|
+
case STANDARD_BT2020:
|
|
58
|
+
case STANDARD_BT2020_CONSTANT_LUMINANCE:
|
|
59
|
+
colorSpace.matrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec2020;
|
|
60
|
+
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::Rec2020;
|
|
61
|
+
break;
|
|
62
|
+
case STANDARD_DCI_P3:
|
|
63
|
+
colorSpace.matrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec709;
|
|
64
|
+
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::DisplayP3;
|
|
65
|
+
break;
|
|
66
|
+
default:
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
switch (dataSpace & TRANSFER_MASK) {
|
|
70
|
+
case TRANSFER_LINEAR:
|
|
71
|
+
colorSpace.transfer = wgpu::ColorSpaceTransferDawn::Identity;
|
|
72
|
+
break;
|
|
73
|
+
case TRANSFER_SRGB:
|
|
74
|
+
case TRANSFER_GAMMA2_2: // sRGB is the closest curve Dawn has
|
|
75
|
+
colorSpace.transfer = wgpu::ColorSpaceTransferDawn::SRGB;
|
|
76
|
+
break;
|
|
77
|
+
case TRANSFER_SMPTE_170M:
|
|
78
|
+
colorSpace.transfer = wgpu::ColorSpaceTransferDawn::SMPTE_170M;
|
|
79
|
+
break;
|
|
80
|
+
case TRANSFER_GAMMA2_6:
|
|
81
|
+
case TRANSFER_GAMMA2_8: // BT.1886 (gamma 2.4) is the closest
|
|
82
|
+
colorSpace.transfer = wgpu::ColorSpaceTransferDawn::BT_1886;
|
|
83
|
+
break;
|
|
84
|
+
case TRANSFER_ST2084:
|
|
85
|
+
colorSpace.transfer = wgpu::ColorSpaceTransferDawn::PQ;
|
|
86
|
+
break;
|
|
87
|
+
case TRANSFER_HLG:
|
|
88
|
+
colorSpace.transfer = wgpu::ColorSpaceTransferDawn::HLG;
|
|
89
|
+
break;
|
|
90
|
+
default:
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
switch (dataSpace & RANGE_MASK) {
|
|
94
|
+
case RANGE_FULL:
|
|
95
|
+
case RANGE_EXTENDED:
|
|
96
|
+
colorSpace.range = wgpu::ColorSpaceYCbCrRangeDawn::Full;
|
|
97
|
+
break;
|
|
98
|
+
case RANGE_LIMITED:
|
|
99
|
+
colorSpace.range = wgpu::ColorSpaceYCbCrRangeDawn::Narrow;
|
|
100
|
+
break;
|
|
101
|
+
default:
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
104
|
+
return colorSpace;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
} // namespace rnwgpu
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
#pragma once
|
|
2
2
|
|
|
3
3
|
#include <android/bitmap.h>
|
|
4
|
+
#include <android/data_space.h>
|
|
4
5
|
#include <android/hardware_buffer.h>
|
|
6
|
+
#include <dlfcn.h>
|
|
5
7
|
#include <jni.h>
|
|
6
8
|
|
|
7
9
|
#include <algorithm>
|
|
@@ -9,10 +11,12 @@
|
|
|
9
11
|
#include <memory>
|
|
10
12
|
#include <string>
|
|
11
13
|
#include <thread>
|
|
14
|
+
#include <utility>
|
|
12
15
|
#include <vector>
|
|
13
16
|
|
|
14
17
|
#include "webgpu/webgpu_cpp.h"
|
|
15
18
|
|
|
19
|
+
#include "AndroidColorSpace.h"
|
|
16
20
|
#include "AndroidVideoPlayer.h"
|
|
17
21
|
#include "PlatformContext.h"
|
|
18
22
|
#include "RNWebGPUManager.h"
|
|
@@ -24,9 +28,10 @@ namespace jni = facebook::jni;
|
|
|
24
28
|
|
|
25
29
|
// The completion callbacks of one snapshotView request, heap-allocated and
|
|
26
30
|
// carried through Java as an opaque pointer (WebGPUModule.snapshotView ->
|
|
27
|
-
// onViewSnapshot), which hands them back exactly once.
|
|
31
|
+
// onViewSnapshot / onViewSnapshotBuffer), which hands them back exactly once.
|
|
28
32
|
struct ViewSnapshotCallbacks {
|
|
29
|
-
std::function<void(ImageData)>
|
|
33
|
+
std::function<void(ImageData)> onPixels;
|
|
34
|
+
std::function<void(VideoFrameHandle)> onFrame;
|
|
30
35
|
std::function<void(std::string)> onError;
|
|
31
36
|
};
|
|
32
37
|
|
|
@@ -90,7 +95,8 @@ public:
|
|
|
90
95
|
}
|
|
91
96
|
|
|
92
97
|
void snapshotView(const ViewSnapshotRequest &request,
|
|
93
|
-
std::function<void(ImageData)>
|
|
98
|
+
std::function<void(ImageData)> onPixels,
|
|
99
|
+
std::function<void(VideoFrameHandle)> onFrame,
|
|
94
100
|
std::function<void(std::string)> onError) override {
|
|
95
101
|
if (!_module) {
|
|
96
102
|
onError("drawElementImageToTexture: the WebGPU module is gone");
|
|
@@ -104,7 +110,7 @@ public:
|
|
|
104
110
|
}
|
|
105
111
|
jclass moduleClass = env->GetObjectClass(_module);
|
|
106
112
|
jmethodID method =
|
|
107
|
-
env->GetMethodID(moduleClass, "snapshotView", "(
|
|
113
|
+
env->GetMethodID(moduleClass, "snapshotView", "(IDDDDIIZJ)V");
|
|
108
114
|
env->DeleteLocalRef(moduleClass);
|
|
109
115
|
if (!method) {
|
|
110
116
|
env->ExceptionClear();
|
|
@@ -112,9 +118,9 @@ public:
|
|
|
112
118
|
return;
|
|
113
119
|
}
|
|
114
120
|
// Ownership passes to Java; Java_com_webgpu_WebGPUModule_onViewSnapshot
|
|
115
|
-
// (cpp-adapter.cpp) reclaims it.
|
|
116
|
-
auto *callbacks =
|
|
117
|
-
|
|
121
|
+
// or onViewSnapshotBuffer (cpp-adapter.cpp) reclaims it.
|
|
122
|
+
auto *callbacks = new ViewSnapshotCallbacks{
|
|
123
|
+
std::move(onPixels), std::move(onFrame), std::move(onError)};
|
|
118
124
|
env->CallVoidMethod(_module, method, static_cast<jint>(request.viewTag),
|
|
119
125
|
static_cast<jdouble>(request.sourceX),
|
|
120
126
|
static_cast<jdouble>(request.sourceY),
|
|
@@ -122,6 +128,7 @@ public:
|
|
|
122
128
|
static_cast<jdouble>(request.sourceHeight),
|
|
123
129
|
static_cast<jint>(request.width),
|
|
124
130
|
static_cast<jint>(request.height),
|
|
131
|
+
static_cast<jboolean>(request.allowFrame),
|
|
125
132
|
reinterpret_cast<jlong>(callbacks));
|
|
126
133
|
if (env->ExceptionCheck()) {
|
|
127
134
|
env->ExceptionClear();
|
|
@@ -268,8 +275,92 @@ public:
|
|
|
268
275
|
"device.importSharedTextureMemory.");
|
|
269
276
|
}
|
|
270
277
|
|
|
271
|
-
|
|
272
|
-
|
|
278
|
+
// The NV12 variant of createTestVideoFrame: a Y8Cb8Cr8_420 buffer, the YUV
|
|
279
|
+
// layout every gralloc supports, BT.709 limited range, red on the left half
|
|
280
|
+
// and green on the right. Filling it needs AHardwareBuffer_lockPlanes
|
|
281
|
+
// (API 29), resolved at run time since the library runs from API 26.
|
|
282
|
+
VideoFrameHandle createTestYuvFrame(uint32_t width, uint32_t height) {
|
|
283
|
+
using LockPlanes = int (*)(AHardwareBuffer *, uint64_t, int32_t,
|
|
284
|
+
const ARect *, AHardwareBuffer_Planes *);
|
|
285
|
+
static const auto lockPlanes = reinterpret_cast<LockPlanes>(
|
|
286
|
+
dlsym(RTLD_DEFAULT, "AHardwareBuffer_lockPlanes"));
|
|
287
|
+
if (lockPlanes == nullptr) {
|
|
288
|
+
throw std::runtime_error(
|
|
289
|
+
"createTestVideoFrame: unsupported: a YUV test frame needs "
|
|
290
|
+
"AHardwareBuffer_lockPlanes (API 29)");
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
AHardwareBuffer_Desc desc = {};
|
|
294
|
+
desc.width = width;
|
|
295
|
+
desc.height = height;
|
|
296
|
+
desc.layers = 1;
|
|
297
|
+
desc.format = AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420;
|
|
298
|
+
desc.usage = AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE |
|
|
299
|
+
AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY |
|
|
300
|
+
AHARDWAREBUFFER_USAGE_CPU_READ_RARELY;
|
|
301
|
+
AHardwareBuffer *buffer = nullptr;
|
|
302
|
+
int err = AHardwareBuffer_allocate(&desc, &buffer);
|
|
303
|
+
if (err != 0 || !buffer) {
|
|
304
|
+
throw std::runtime_error(
|
|
305
|
+
"createTestVideoFrame: unsupported: AHardwareBuffer_allocate failed "
|
|
306
|
+
"for a Y8Cb8Cr8_420 buffer (" +
|
|
307
|
+
std::to_string(err) + ")");
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
AHardwareBuffer_Planes planes = {};
|
|
311
|
+
int rc = lockPlanes(buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, -1,
|
|
312
|
+
nullptr, &planes);
|
|
313
|
+
if (rc != 0 || planes.planeCount < 3) {
|
|
314
|
+
AHardwareBuffer_release(buffer);
|
|
315
|
+
throw std::runtime_error(
|
|
316
|
+
"createTestVideoFrame: unsupported: AHardwareBuffer_lockPlanes "
|
|
317
|
+
"failed (" +
|
|
318
|
+
std::to_string(rc) + ")");
|
|
319
|
+
}
|
|
320
|
+
// Plane 0 is Y, 1 is Cb, 2 is Cr; the chroma planes are subsampled by two
|
|
321
|
+
// on both axes and the strides say whether Cb and Cr interleave.
|
|
322
|
+
const auto &y = planes.planes[0];
|
|
323
|
+
const auto &cb = planes.planes[1];
|
|
324
|
+
const auto &cr = planes.planes[2];
|
|
325
|
+
for (uint32_t row = 0; row < height; ++row) {
|
|
326
|
+
auto *line = static_cast<uint8_t *>(y.data) + row * y.rowStride;
|
|
327
|
+
for (uint32_t col = 0; col < width; ++col) {
|
|
328
|
+
line[col * y.pixelStride] =
|
|
329
|
+
col < width / 2 ? test_yuv_frame::kRedY : test_yuv_frame::kGreenY;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
const uint32_t chromaWidth = (width + 1) / 2;
|
|
333
|
+
const uint32_t chromaHeight = (height + 1) / 2;
|
|
334
|
+
for (uint32_t row = 0; row < chromaHeight; ++row) {
|
|
335
|
+
auto *cbLine = static_cast<uint8_t *>(cb.data) + row * cb.rowStride;
|
|
336
|
+
auto *crLine = static_cast<uint8_t *>(cr.data) + row * cr.rowStride;
|
|
337
|
+
for (uint32_t col = 0; col < chromaWidth; ++col) {
|
|
338
|
+
bool red = col < chromaWidth / 2;
|
|
339
|
+
cbLine[col * cb.pixelStride] =
|
|
340
|
+
red ? test_yuv_frame::kRedCb : test_yuv_frame::kGreenCb;
|
|
341
|
+
crLine[col * cr.pixelStride] =
|
|
342
|
+
red ? test_yuv_frame::kRedCr : test_yuv_frame::kGreenCr;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
AHardwareBuffer_unlock(buffer, nullptr);
|
|
346
|
+
|
|
347
|
+
VideoFrameHandle handle;
|
|
348
|
+
handle.handle = static_cast<void *>(buffer);
|
|
349
|
+
handle.width = width;
|
|
350
|
+
handle.height = height;
|
|
351
|
+
handle.pixelFormat = VideoPixelFormat::NV12;
|
|
352
|
+
// What a BT.709 SDR video declares: ADATASPACE_BT709 is STANDARD_BT709 |
|
|
353
|
+
// TRANSFER_SMPTE_170M | RANGE_LIMITED.
|
|
354
|
+
handle.colorSpace = colorSpaceFromDataSpace(ADATASPACE_BT709);
|
|
355
|
+
handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); };
|
|
356
|
+
return handle;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
VideoFrameHandle createTestVideoFrame(uint32_t width, uint32_t height,
|
|
360
|
+
VideoPixelFormat format) override {
|
|
361
|
+
if (format == VideoPixelFormat::NV12) {
|
|
362
|
+
return createTestYuvFrame(width, height);
|
|
363
|
+
}
|
|
273
364
|
// Dawn's Android backend already requires API 26+, so AHardwareBuffer_*
|
|
274
365
|
// symbols are guaranteed to be available at link time on supported builds.
|
|
275
366
|
AHardwareBuffer_Desc desc = {};
|
|
@@ -4,12 +4,14 @@
|
|
|
4
4
|
#include <android/hardware_buffer_jni.h>
|
|
5
5
|
#include <fbjni/fbjni.h>
|
|
6
6
|
#include <jni.h>
|
|
7
|
+
#include <unistd.h>
|
|
7
8
|
|
|
8
9
|
#include <memory>
|
|
9
10
|
#include <stdexcept>
|
|
10
11
|
#include <string>
|
|
11
12
|
#include <utility>
|
|
12
13
|
|
|
14
|
+
#include "AndroidColorSpace.h"
|
|
13
15
|
#include "PlatformContext.h"
|
|
14
16
|
|
|
15
17
|
namespace rnwgpu {
|
|
@@ -94,6 +96,17 @@ public:
|
|
|
94
96
|
_getRotation = method(env, playerClass, "getRotation", "()I");
|
|
95
97
|
_getFrameRate = method(env, playerClass, "getFrameRate", "()D");
|
|
96
98
|
_release = method(env, playerClass, "release", "()V");
|
|
99
|
+
_frameDataSpace =
|
|
100
|
+
method(env, playerClass, "frameDataSpace", "(Landroid/media/Image;)I");
|
|
101
|
+
_acquireFenceFd = env->GetStaticMethodID(playerClass, "acquireFenceFd",
|
|
102
|
+
"(Landroid/media/Image;)I");
|
|
103
|
+
if (_acquireFenceFd == nullptr) {
|
|
104
|
+
env->ExceptionClear();
|
|
105
|
+
env->DeleteLocalRef(playerClass);
|
|
106
|
+
throw std::runtime_error("VideoPlayer: method acquireFenceFd not found");
|
|
107
|
+
}
|
|
108
|
+
// The static call needs the class; keep it for the player's lifetime.
|
|
109
|
+
_playerClass = static_cast<jclass>(env->NewGlobalRef(playerClass));
|
|
97
110
|
env->DeleteLocalRef(playerClass);
|
|
98
111
|
|
|
99
112
|
jclass imageClass = env->FindClass("android/media/Image");
|
|
@@ -108,6 +121,7 @@ public:
|
|
|
108
121
|
if (env->ExceptionCheck()) {
|
|
109
122
|
env->ExceptionClear();
|
|
110
123
|
}
|
|
124
|
+
env->DeleteGlobalRef(_playerClass);
|
|
111
125
|
}
|
|
112
126
|
|
|
113
127
|
VideoFrameHandle copyLatestFrame() override {
|
|
@@ -144,6 +158,28 @@ public:
|
|
|
144
158
|
"them");
|
|
145
159
|
}
|
|
146
160
|
|
|
161
|
+
// The decoder may still be writing the frame: ImageReader hands out the
|
|
162
|
+
// buffer as soon as it is queued and only waits on the producer's fence for
|
|
163
|
+
// CPU access (Image.getPlanes), not for Image.getHardwareBuffer. The GPU
|
|
164
|
+
// work that samples the frame waits on the fence instead (see
|
|
165
|
+
// GPUExternalTexture::Create). -1 when the platform cannot report it
|
|
166
|
+
// (before API 33) or the producer was already done.
|
|
167
|
+
int acquireFenceFd =
|
|
168
|
+
env->CallStaticIntMethod(_playerClass, _acquireFenceFd, image);
|
|
169
|
+
if (env->ExceptionCheck()) {
|
|
170
|
+
env->ExceptionClear();
|
|
171
|
+
acquireFenceFd = -1;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// The color space of the samples: the frame's data space (API 33) or the
|
|
175
|
+
// container's color info. What it leaves unknown is resolved at import.
|
|
176
|
+
int32_t dataSpace =
|
|
177
|
+
env->CallIntMethod(_player->get(), _frameDataSpace, image);
|
|
178
|
+
if (env->ExceptionCheck()) {
|
|
179
|
+
env->ExceptionClear();
|
|
180
|
+
dataSpace = 0;
|
|
181
|
+
}
|
|
182
|
+
|
|
147
183
|
// The frame owns a reference to the buffer and keeps the Image acquired
|
|
148
184
|
// until it is released; closing the Image hands the buffer back to the
|
|
149
185
|
// decoder.
|
|
@@ -157,9 +193,12 @@ public:
|
|
|
157
193
|
handle.height = desc.height;
|
|
158
194
|
handle.pixelFormat = isRgbFormat(desc.format) ? VideoPixelFormat::BGRA8
|
|
159
195
|
: VideoPixelFormat::NV12;
|
|
196
|
+
handle.acquireFenceFd = acquireFenceFd;
|
|
197
|
+
handle.colorSpace = colorSpaceFromDataSpace(dataSpace);
|
|
160
198
|
auto player = _player;
|
|
161
199
|
jmethodID closeImageMethod = _closeImage;
|
|
162
|
-
handle.deleter = [player, closeImageMethod, imageRef, buffer
|
|
200
|
+
handle.deleter = [player, closeImageMethod, imageRef, buffer,
|
|
201
|
+
acquireFenceFd]() {
|
|
163
202
|
JNIEnv *env = attach();
|
|
164
203
|
env->CallVoidMethod(player->get(), closeImageMethod, imageRef);
|
|
165
204
|
if (env->ExceptionCheck()) {
|
|
@@ -167,6 +206,9 @@ public:
|
|
|
167
206
|
}
|
|
168
207
|
env->DeleteGlobalRef(imageRef);
|
|
169
208
|
AHardwareBuffer_release(buffer);
|
|
209
|
+
if (acquireFenceFd >= 0) {
|
|
210
|
+
::close(acquireFenceFd);
|
|
211
|
+
}
|
|
170
212
|
};
|
|
171
213
|
return handle;
|
|
172
214
|
}
|
|
@@ -284,8 +326,11 @@ private:
|
|
|
284
326
|
}
|
|
285
327
|
|
|
286
328
|
std::shared_ptr<JavaVideoPlayerRef> _player;
|
|
329
|
+
jclass _playerClass;
|
|
287
330
|
jmethodID _copyLatestFrame;
|
|
288
331
|
jmethodID _closeImage;
|
|
332
|
+
jmethodID _frameDataSpace;
|
|
333
|
+
jmethodID _acquireFenceFd;
|
|
289
334
|
jmethodID _play;
|
|
290
335
|
jmethodID _pause;
|
|
291
336
|
jmethodID _getPaused;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
#include <unistd.h>
|
|
2
|
+
|
|
1
3
|
#include <cstring>
|
|
2
4
|
#include <memory>
|
|
3
5
|
#include <string>
|
|
@@ -92,7 +94,57 @@ extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUModule_onViewSnapshot(
|
|
|
92
94
|
rowBytes);
|
|
93
95
|
}
|
|
94
96
|
AndroidBitmap_unlockPixels(env, bitmap);
|
|
95
|
-
callbacks->
|
|
97
|
+
callbacks->onPixels(std::move(image));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Completion of WebGPUModule.snapshotView through a hardware buffer (UI
|
|
101
|
+
// thread, Android 13 and above): the buffer HardwareRenderer drew the view
|
|
102
|
+
// into, and the fence that signals when the drawing is done, as a sync fd
|
|
103
|
+
// this function takes ownership of. The GPU draws the buffer into the
|
|
104
|
+
// destination itself (GPUQueue::drawElementImageToTexture); nothing is read
|
|
105
|
+
// back through the CPU.
|
|
106
|
+
extern "C" JNIEXPORT void JNICALL
|
|
107
|
+
Java_com_webgpu_WebGPUModule_onViewSnapshotBuffer(JNIEnv *env,
|
|
108
|
+
jobject /* this */,
|
|
109
|
+
jlong callbackPointer,
|
|
110
|
+
jobject hardwareBuffer,
|
|
111
|
+
jint fenceFd) {
|
|
112
|
+
std::unique_ptr<rnwgpu::ViewSnapshotCallbacks> callbacks(
|
|
113
|
+
reinterpret_cast<rnwgpu::ViewSnapshotCallbacks *>(callbackPointer));
|
|
114
|
+
AHardwareBuffer *buffer =
|
|
115
|
+
hardwareBuffer != nullptr
|
|
116
|
+
? AHardwareBuffer_fromHardwareBuffer(env, hardwareBuffer)
|
|
117
|
+
: nullptr;
|
|
118
|
+
if (!callbacks || buffer == nullptr) {
|
|
119
|
+
if (fenceFd >= 0) {
|
|
120
|
+
::close(fenceFd);
|
|
121
|
+
}
|
|
122
|
+
if (callbacks) {
|
|
123
|
+
callbacks->onError(
|
|
124
|
+
"drawElementImageToTexture: the renderer produced no buffer");
|
|
125
|
+
}
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
AHardwareBuffer_Desc desc{};
|
|
129
|
+
AHardwareBuffer_describe(buffer, &desc);
|
|
130
|
+
// Java closes its Image and HardwareBuffer as soon as this returns; this
|
|
131
|
+
// reference keeps the memory alive until the frame is released.
|
|
132
|
+
AHardwareBuffer_acquire(buffer);
|
|
133
|
+
|
|
134
|
+
rnwgpu::VideoFrameHandle frame;
|
|
135
|
+
frame.handle = buffer;
|
|
136
|
+
frame.width = desc.width;
|
|
137
|
+
frame.height = desc.height;
|
|
138
|
+
// An RGBA_8888 buffer: sampled as a plain color texture.
|
|
139
|
+
frame.pixelFormat = rnwgpu::VideoPixelFormat::BGRA8;
|
|
140
|
+
frame.acquireFenceFd = fenceFd;
|
|
141
|
+
frame.deleter = [buffer, fenceFd]() {
|
|
142
|
+
AHardwareBuffer_release(buffer);
|
|
143
|
+
if (fenceFd >= 0) {
|
|
144
|
+
::close(fenceFd);
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
callbacks->onFrame(std::move(frame));
|
|
96
148
|
}
|
|
97
149
|
|
|
98
150
|
extern "C" JNIEXPORT void JNICALL Java_com_webgpu_WebGPUView_onSurfaceChanged(
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
package com.webgpu;
|
|
2
|
+
|
|
3
|
+
import android.hardware.SyncFence;
|
|
4
|
+
import android.media.Image;
|
|
5
|
+
import android.os.Build;
|
|
6
|
+
import android.os.Parcel;
|
|
7
|
+
import android.os.ParcelFileDescriptor;
|
|
8
|
+
import android.util.Log;
|
|
9
|
+
|
|
10
|
+
import androidx.annotation.NonNull;
|
|
11
|
+
import androidx.annotation.RequiresApi;
|
|
12
|
+
|
|
13
|
+
import java.io.IOException;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Bridges the platform's {@link SyncFence} to the sync file descriptors Dawn imports as
|
|
17
|
+
* {@code shared-fence-sync-fd} fences.
|
|
18
|
+
*
|
|
19
|
+
* <p>A buffer handed out by an {@link android.media.ImageReader} carries an acquire fence: the
|
|
20
|
+
* producer (the video decoder, the camera, a {@link android.graphics.HardwareRenderer}) may still
|
|
21
|
+
* be writing it when the consumer acquires it. The framework waits on that fence for CPU access
|
|
22
|
+
* ({@link Image#getPlanes()}), but not for {@link Image#getHardwareBuffer()}, so a GPU consumer has
|
|
23
|
+
* to wait on it itself. Since API 33 the fence is exposed as {@link Image#getFence()}.
|
|
24
|
+
*/
|
|
25
|
+
final class SyncFences {
|
|
26
|
+
private static final String TAG = "WebGPUSyncFences";
|
|
27
|
+
|
|
28
|
+
private SyncFences() {}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The acquire fence of {@code image} as a sync file descriptor the caller owns and must close,
|
|
32
|
+
* or -1 when the frame has none (the producer was done when it was acquired, or the platform
|
|
33
|
+
* predates {@link Image#getFence()}).
|
|
34
|
+
*/
|
|
35
|
+
static int acquireFenceFd(@NonNull Image image) {
|
|
36
|
+
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) {
|
|
37
|
+
return -1;
|
|
38
|
+
}
|
|
39
|
+
return acquireFenceFdApi33(image);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
|
43
|
+
private static int acquireFenceFdApi33(@NonNull Image image) {
|
|
44
|
+
SyncFence fence;
|
|
45
|
+
try {
|
|
46
|
+
fence = image.getFence();
|
|
47
|
+
} catch (IOException | RuntimeException e) {
|
|
48
|
+
Log.w(TAG, "could not read the acquire fence of the frame: " + e.getMessage());
|
|
49
|
+
return -1;
|
|
50
|
+
}
|
|
51
|
+
try {
|
|
52
|
+
return detachFd(fence);
|
|
53
|
+
} finally {
|
|
54
|
+
// getFence() returns a duplicate; the detached descriptor is ours, this one goes.
|
|
55
|
+
fence.close();
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The file descriptor of {@code fence}, duplicated and detached, so it stays valid after the
|
|
61
|
+
* fence is closed and must be closed by the caller. -1 for an invalid fence.
|
|
62
|
+
*
|
|
63
|
+
* <p>{@link SyncFence} has no public accessor for its descriptor. Writing the fence to a {@link
|
|
64
|
+
* Parcel} and reading it back yields one: the parcel form is a validity flag followed by the
|
|
65
|
+
* file descriptor.
|
|
66
|
+
*/
|
|
67
|
+
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
|
68
|
+
static int detachFd(@NonNull SyncFence fence) {
|
|
69
|
+
if (!fence.isValid()) {
|
|
70
|
+
return -1;
|
|
71
|
+
}
|
|
72
|
+
Parcel parcel = Parcel.obtain();
|
|
73
|
+
try {
|
|
74
|
+
fence.writeToParcel(parcel, 0);
|
|
75
|
+
parcel.setDataPosition(0);
|
|
76
|
+
if (!parcel.readBoolean()) {
|
|
77
|
+
return -1;
|
|
78
|
+
}
|
|
79
|
+
ParcelFileDescriptor descriptor = parcel.readFileDescriptor();
|
|
80
|
+
return descriptor == null ? -1 : descriptor.detachFd();
|
|
81
|
+
} catch (RuntimeException e) {
|
|
82
|
+
Log.w(TAG, "could not read the file descriptor of a SyncFence: " + e.getMessage());
|
|
83
|
+
return -1;
|
|
84
|
+
} finally {
|
|
85
|
+
parcel.recycle();
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -24,7 +24,10 @@ import androidx.annotation.RequiresApi;
|
|
|
24
24
|
* On API 29+ the view is rendered through {@link HardwareRenderer}: the
|
|
25
25
|
* recording reuses the children's existing display lists, so the result is
|
|
26
26
|
* exactly what the window's own renderer draws (shadows, hardware layers,
|
|
27
|
-
* TextureView content).
|
|
27
|
+
* TextureView content). The renderer draws into a GPU-sampleable buffer,
|
|
28
|
+
* which {@link #renderHardware} hands to its caller as is: the WebGPU side
|
|
29
|
+
* imports it directly when it can, and {@link #snapshot} reads it back into
|
|
30
|
+
* a bitmap otherwise. Older devices, or a failed hardware render, fall back
|
|
28
31
|
* to drawing the view into a software canvas, which misses anything that is
|
|
29
32
|
* only drawn hardware-accelerated. SurfaceView content is composed outside
|
|
30
33
|
* the view hierarchy and is captured by neither path.
|
|
@@ -34,6 +37,11 @@ final class ViewSnapshot {
|
|
|
34
37
|
|
|
35
38
|
private ViewSnapshot() {}
|
|
36
39
|
|
|
40
|
+
/** What a caller does with the frame the hardware renderer drew, while it is acquired. */
|
|
41
|
+
interface FrameConsumer<T> {
|
|
42
|
+
T consume(@NonNull Image image) throws Exception;
|
|
43
|
+
}
|
|
44
|
+
|
|
37
45
|
/**
|
|
38
46
|
* @param sourceX/Y/Width/Height the source rectangle in the view's own
|
|
39
47
|
* coordinate space, in pixels (already scaled by the display density)
|
|
@@ -77,19 +85,27 @@ final class ViewSnapshot {
|
|
|
77
85
|
return bitmap;
|
|
78
86
|
}
|
|
79
87
|
|
|
88
|
+
/**
|
|
89
|
+
* Renders the view through {@link HardwareRenderer} into a GPU-sampleable
|
|
90
|
+
* RGBA_8888 buffer of exactly width x height pixels and hands the acquired
|
|
91
|
+
* frame to {@code consumer}; the reader and the renderer are torn down when
|
|
92
|
+
* it returns, so a consumer that keeps the buffer takes its own reference
|
|
93
|
+
* first. The renderer only submits the frame: the drawing may still be in
|
|
94
|
+
* flight on the GPU when the consumer runs. {@link Image#getFence()} (API 33)
|
|
95
|
+
* tells when it is done; a CPU readback of the buffer waits for it on its
|
|
96
|
+
* own.
|
|
97
|
+
*/
|
|
80
98
|
@RequiresApi(Build.VERSION_CODES.Q)
|
|
81
|
-
|
|
82
|
-
private static Bitmap snapshotHardware(
|
|
99
|
+
static <T> T renderHardware(
|
|
83
100
|
@NonNull View view,
|
|
84
101
|
float sourceX,
|
|
85
102
|
float sourceY,
|
|
86
103
|
float sourceWidth,
|
|
87
104
|
float sourceHeight,
|
|
88
105
|
int width,
|
|
89
|
-
int height
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
// read back into a software bitmap.
|
|
106
|
+
int height,
|
|
107
|
+
@NonNull FrameConsumer<T> consumer)
|
|
108
|
+
throws Exception {
|
|
93
109
|
ImageReader reader =
|
|
94
110
|
ImageReader.newInstance(
|
|
95
111
|
width,
|
|
@@ -135,22 +151,7 @@ final class ViewSnapshot {
|
|
|
135
151
|
if (image == null) {
|
|
136
152
|
throw new IllegalStateException("HardwareRenderer produced no image");
|
|
137
153
|
}
|
|
138
|
-
|
|
139
|
-
if (buffer == null) {
|
|
140
|
-
throw new IllegalStateException("HardwareRenderer image has no HardwareBuffer");
|
|
141
|
-
}
|
|
142
|
-
Bitmap hardware =
|
|
143
|
-
Bitmap.wrapHardwareBuffer(buffer, ColorSpace.get(ColorSpace.Named.SRGB));
|
|
144
|
-
buffer.close();
|
|
145
|
-
if (hardware == null) {
|
|
146
|
-
throw new IllegalStateException("Bitmap.wrapHardwareBuffer failed");
|
|
147
|
-
}
|
|
148
|
-
Bitmap software = hardware.copy(Bitmap.Config.ARGB_8888, false);
|
|
149
|
-
hardware.recycle();
|
|
150
|
-
if (software == null) {
|
|
151
|
-
throw new IllegalStateException("could not read back the hardware bitmap");
|
|
152
|
-
}
|
|
153
|
-
return software;
|
|
154
|
+
return consumer.consume(image);
|
|
154
155
|
} finally {
|
|
155
156
|
if (image != null) {
|
|
156
157
|
image.close();
|
|
@@ -159,4 +160,46 @@ final class ViewSnapshot {
|
|
|
159
160
|
reader.close();
|
|
160
161
|
}
|
|
161
162
|
}
|
|
163
|
+
|
|
164
|
+
@RequiresApi(Build.VERSION_CODES.Q)
|
|
165
|
+
@NonNull
|
|
166
|
+
private static Bitmap snapshotHardware(
|
|
167
|
+
@NonNull View view,
|
|
168
|
+
float sourceX,
|
|
169
|
+
float sourceY,
|
|
170
|
+
float sourceWidth,
|
|
171
|
+
float sourceHeight,
|
|
172
|
+
int width,
|
|
173
|
+
int height)
|
|
174
|
+
throws Exception {
|
|
175
|
+
return renderHardware(
|
|
176
|
+
view,
|
|
177
|
+
sourceX,
|
|
178
|
+
sourceY,
|
|
179
|
+
sourceWidth,
|
|
180
|
+
sourceHeight,
|
|
181
|
+
width,
|
|
182
|
+
height,
|
|
183
|
+
image -> {
|
|
184
|
+
HardwareBuffer buffer = image.getHardwareBuffer();
|
|
185
|
+
if (buffer == null) {
|
|
186
|
+
throw new IllegalStateException("HardwareRenderer image has no HardwareBuffer");
|
|
187
|
+
}
|
|
188
|
+
// wrapHardwareBuffer takes its own reference, so the wrapper can be
|
|
189
|
+
// closed immediately; the copy reads the pixels back into software
|
|
190
|
+
// memory, waiting for the renderer to finish drawing.
|
|
191
|
+
Bitmap hardware =
|
|
192
|
+
Bitmap.wrapHardwareBuffer(buffer, ColorSpace.get(ColorSpace.Named.SRGB));
|
|
193
|
+
buffer.close();
|
|
194
|
+
if (hardware == null) {
|
|
195
|
+
throw new IllegalStateException("Bitmap.wrapHardwareBuffer failed");
|
|
196
|
+
}
|
|
197
|
+
Bitmap software = hardware.copy(Bitmap.Config.ARGB_8888, false);
|
|
198
|
+
hardware.recycle();
|
|
199
|
+
if (software == null) {
|
|
200
|
+
throw new IllegalStateException("could not read back the hardware bitmap");
|
|
201
|
+
}
|
|
202
|
+
return software;
|
|
203
|
+
});
|
|
204
|
+
}
|
|
162
205
|
}
|