react-native-webgpu 0.15.2 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/CMakeLists.txt +5 -0
- package/android/cpp/cpp-adapter.cpp +109 -11
- package/android/src/main/java/com/webgpu/SurfaceFrameRateVote.java +97 -85
- package/android/src/main/java/com/webgpu/WebGPUView.java +13 -8
- package/apple/ApplePlatformContext.h +1 -0
- package/cpp/jsi/JSICache.cpp +2 -0
- package/cpp/jsi/NativeObject.h +36 -12
- package/cpp/rnwgpu/RNWebGPUManager.cpp +4 -0
- package/cpp/rnwgpu/SurfaceRegistry.h +4 -4
- package/cpp/rnwgpu/api/GPUDevice.cpp +96 -0
- package/cpp/rnwgpu/api/GPUDevice.h +22 -0
- package/cpp/rnwgpu/api/GPUError.h +1 -0
- package/cpp/rnwgpu/api/GPUExtent2D.h +56 -0
- package/cpp/rnwgpu/api/GPUExternalTexture.cpp +32 -408
- package/cpp/rnwgpu/api/GPUExternalTexture.h +12 -37
- package/cpp/rnwgpu/api/GPUInternalError.h +1 -0
- package/cpp/rnwgpu/api/GPUNativeBufferExternalTexture.h +60 -0
- package/cpp/rnwgpu/api/GPUNativeBufferTexture.h +52 -0
- package/cpp/rnwgpu/api/GPUNativeBufferWrapper.cpp +144 -0
- package/cpp/rnwgpu/api/GPUNativeBufferWrapper.h +142 -0
- package/cpp/rnwgpu/api/GPUOutOfMemoryError.h +1 -0
- package/cpp/rnwgpu/api/GPUQueue.h +1 -0
- package/cpp/rnwgpu/api/GPUSharedFence.h +1 -0
- package/cpp/rnwgpu/api/GPUSharedTextureMemory.cpp +12 -44
- package/cpp/rnwgpu/api/GPUSharedTextureMemory.h +1 -0
- package/cpp/rnwgpu/api/GPUUncapturedErrorEvent.h +1 -0
- package/cpp/rnwgpu/api/GPUValidationError.h +1 -0
- package/cpp/rnwgpu/api/NativeFrameImport.cpp +638 -0
- package/cpp/rnwgpu/api/NativeFrameImport.h +86 -0
- package/cpp/rnwgpu/api/VideoFrame.h +21 -7
- package/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h +153 -1
- package/cpp/rnwgpu/api/descriptors/GPUNativeBufferTextureDescriptor.h +55 -0
- package/cpp/rnwgpu/async/RuntimeContext.cpp +1 -0
- package/lib/commonjs/Canvas.js.map +1 -1
- package/lib/commonjs/index.js.map +1 -1
- package/lib/module/Canvas.js.map +1 -1
- package/lib/module/index.js.map +1 -1
- package/lib/typescript/src/Canvas.d.ts +2 -31
- package/lib/typescript/src/Canvas.d.ts.map +1 -1
- package/lib/typescript/src/index.d.ts +4 -2
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/types.d.ts +33 -0
- package/lib/typescript/src/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/Canvas.tsx +3 -36
- package/src/index.tsx +24 -0
- package/src/types.ts +97 -0
|
@@ -1,441 +1,65 @@
|
|
|
1
1
|
#include "GPUExternalTexture.h"
|
|
2
2
|
|
|
3
|
-
#include <
|
|
4
|
-
#include <cmath>
|
|
3
|
+
#include <cstdint>
|
|
5
4
|
#include <memory>
|
|
6
|
-
#include <
|
|
5
|
+
#include <stdexcept>
|
|
7
6
|
#include <string>
|
|
8
7
|
#include <utility>
|
|
8
|
+
#include <vector>
|
|
9
9
|
|
|
10
10
|
#include "GPUExternalTextureDescriptor.h"
|
|
11
|
-
|
|
12
|
-
#include "dawn/common/ExternalTextureParams.h"
|
|
13
|
-
|
|
14
|
-
#if defined(__ANDROID__)
|
|
15
|
-
#include <android/hardware_buffer.h>
|
|
16
|
-
#include <android/log.h>
|
|
17
|
-
#include <poll.h>
|
|
18
|
-
#endif
|
|
11
|
+
#include "NativeFrameImport.h"
|
|
19
12
|
|
|
20
13
|
namespace rnwgpu {
|
|
21
14
|
|
|
22
|
-
#if defined(__ANDROID__)
|
|
23
|
-
// Whether Dawn imports an AHardwareBuffer of this format as an
|
|
24
|
-
// OpaqueYCbCrAndroid texture. Mirrors FormatFromAHardwareBufferFormat
|
|
25
|
-
// (dawn/native/AHBFunctions.cpp): the color and depth formats with a WebGPU
|
|
26
|
-
// equivalent import as that format; the YUV formats and the formats outside
|
|
27
|
-
// the public enum, which drivers use for the camera and the video decoder,
|
|
28
|
-
// import as OpaqueYCbCrAndroid.
|
|
29
|
-
static bool isOpaqueYCbCrFormat(uint32_t format) {
|
|
30
|
-
switch (format) {
|
|
31
|
-
case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM:
|
|
32
|
-
case AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM:
|
|
33
|
-
case AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM:
|
|
34
|
-
case AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM:
|
|
35
|
-
case AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT:
|
|
36
|
-
case AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM:
|
|
37
|
-
case AHARDWAREBUFFER_FORMAT_BLOB:
|
|
38
|
-
case AHARDWAREBUFFER_FORMAT_D16_UNORM:
|
|
39
|
-
case AHARDWAREBUFFER_FORMAT_D24_UNORM:
|
|
40
|
-
case AHARDWAREBUFFER_FORMAT_D24_UNORM_S8_UINT:
|
|
41
|
-
case AHARDWAREBUFFER_FORMAT_D32_FLOAT:
|
|
42
|
-
case AHARDWAREBUFFER_FORMAT_D32_FLOAT_S8_UINT:
|
|
43
|
-
case AHARDWAREBUFFER_FORMAT_S8_UINT:
|
|
44
|
-
// Single-channel formats of the API 33 and 34 headers, by value so that
|
|
45
|
-
// older NDKs compile.
|
|
46
|
-
case 0x38: // AHARDWAREBUFFER_FORMAT_R8_UNORM
|
|
47
|
-
case 0x39: // AHARDWAREBUFFER_FORMAT_R16_UINT
|
|
48
|
-
case 0x3a: // AHARDWAREBUFFER_FORMAT_R16G16_UINT
|
|
49
|
-
case 0x3b: // AHARDWAREBUFFER_FORMAT_R10G10B10A10_UNORM
|
|
50
|
-
return false;
|
|
51
|
-
default:
|
|
52
|
-
return true;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
#endif
|
|
56
|
-
|
|
57
|
-
// The color space of a YUV frame with every member known. The producer reports
|
|
58
|
-
// what it knows (VideoFrameHandle.colorSpace); the matrix and range it leaves
|
|
59
|
-
// out come from `fallbackMatrix` / `fallbackRange`, the platform's best guess
|
|
60
|
-
// for the buffer, and the rest follows from the matrix: the primaries of the
|
|
61
|
-
// standard that defines it, and the SMPTE 170M transfer function that BT.601,
|
|
62
|
-
// BT.709 and BT.2020 SDR video is encoded with. An identity matrix means the
|
|
63
|
-
// samples are RGB already, which the identity transfer leaves untouched.
|
|
64
|
-
static wgpu::ColorSpaceDawn
|
|
65
|
-
resolveColorSpace(const VideoFrameColorSpace &reported,
|
|
66
|
-
wgpu::ColorSpaceYCbCrMatrixDawn fallbackMatrix,
|
|
67
|
-
wgpu::ColorSpaceYCbCrRangeDawn fallbackRange) {
|
|
68
|
-
wgpu::ColorSpaceDawn colorSpace{};
|
|
69
|
-
colorSpace.yCbCrMatrix = reported.matrix.value_or(fallbackMatrix);
|
|
70
|
-
colorSpace.yCbCrRange = reported.range.value_or(fallbackRange);
|
|
71
|
-
if (reported.primaries.has_value()) {
|
|
72
|
-
colorSpace.primaries = *reported.primaries;
|
|
73
|
-
} else {
|
|
74
|
-
switch (colorSpace.yCbCrMatrix) {
|
|
75
|
-
case wgpu::ColorSpaceYCbCrMatrixDawn::Rec601:
|
|
76
|
-
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::Rec601;
|
|
77
|
-
break;
|
|
78
|
-
case wgpu::ColorSpaceYCbCrMatrixDawn::Rec2020:
|
|
79
|
-
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::Rec2020;
|
|
80
|
-
break;
|
|
81
|
-
default:
|
|
82
|
-
// BT.709 shares the sRGB primaries (Dawn's helper only knows them under
|
|
83
|
-
// that name).
|
|
84
|
-
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::SRGB;
|
|
85
|
-
break;
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
if (reported.transfer.has_value()) {
|
|
89
|
-
colorSpace.transfer = *reported.transfer;
|
|
90
|
-
} else {
|
|
91
|
-
colorSpace.transfer =
|
|
92
|
-
colorSpace.yCbCrMatrix == wgpu::ColorSpaceYCbCrMatrixDawn::Identity
|
|
93
|
-
? wgpu::ColorSpaceTransferDawn::Identity
|
|
94
|
-
: wgpu::ColorSpaceTransferDawn::SMPTE_170M;
|
|
95
|
-
}
|
|
96
|
-
return colorSpace;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
// The ExternalTextureDescriptor parameters that convert samples in `source` to
|
|
100
|
-
// RGB in the color space the descriptor asks for ("srgb" by default,
|
|
101
|
-
// "display-p3"): the YCbCr matrix, the range, the source transfer function,
|
|
102
|
-
// the gamut conversion and the destination transfer function. Dawn applies
|
|
103
|
-
// them while sampling the external texture. The helper is Dawn's own, the one
|
|
104
|
-
// Chromium feeds video frames through.
|
|
105
|
-
static dawn::ExternalTextureColorSpaceParams
|
|
106
|
-
colorSpaceParams(const wgpu::ColorSpaceDawn &source,
|
|
107
|
-
const std::optional<std::string> &destination) {
|
|
108
|
-
auto target = destination.has_value() && *destination == "display-p3"
|
|
109
|
-
? wgpu::PredefinedColorSpace::DisplayP3
|
|
110
|
-
: wgpu::PredefinedColorSpace::SRGB;
|
|
111
|
-
dawn::ExternalTextureColorSpaceParams params;
|
|
112
|
-
if (dawn::ComputeExternalTextureParams(source, target, ¶ms) !=
|
|
113
|
-
wgpu::Status::Success) {
|
|
114
|
-
throw std::runtime_error(
|
|
115
|
-
"GPUExternalTexture::Create(): the color space of the frame cannot be "
|
|
116
|
-
"converted to '" +
|
|
117
|
-
destination.value_or("srgb") + "'");
|
|
118
|
-
}
|
|
119
|
-
return params;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// Dawn dereferences the four arrays unconditionally, so a descriptor always
|
|
123
|
-
// carries a full set, the no-op one for frames that are RGB already.
|
|
124
|
-
static void
|
|
125
|
-
applyColorSpaceParams(wgpu::ExternalTextureDescriptor &descriptor,
|
|
126
|
-
const dawn::ExternalTextureColorSpaceParams ¶ms) {
|
|
127
|
-
descriptor.yuvToRgbConversionMatrix = params.yuvToRgbConversionMatrix.data();
|
|
128
|
-
descriptor.srcTransferFunctionParameters = params.srcTransferFunction.data();
|
|
129
|
-
descriptor.dstTransferFunctionParameters = params.dstTransferFunction.data();
|
|
130
|
-
descriptor.gamutConversionMatrix = params.gamutConversionMatrix.data();
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
// Map a rotation in degrees (0 / 90 / 180 / 270) to Dawn's enum. Anything that
|
|
134
|
-
// isn't a clean multiple of 90 snaps to the nearest quadrant; Dawn only
|
|
135
|
-
// supports those four steps for external textures.
|
|
136
|
-
static wgpu::ExternalTextureRotation toExternalTextureRotation(double degrees) {
|
|
137
|
-
int quadrant = static_cast<int>(std::lround(degrees / 90.0)) & 3;
|
|
138
|
-
switch (quadrant) {
|
|
139
|
-
case 1:
|
|
140
|
-
return wgpu::ExternalTextureRotation::Rotate90Degrees;
|
|
141
|
-
case 2:
|
|
142
|
-
return wgpu::ExternalTextureRotation::Rotate180Degrees;
|
|
143
|
-
case 3:
|
|
144
|
-
return wgpu::ExternalTextureRotation::Rotate270Degrees;
|
|
145
|
-
default:
|
|
146
|
-
return wgpu::ExternalTextureRotation::Rotate0Degrees;
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
|
|
150
15
|
std::shared_ptr<GPUExternalTexture> GPUExternalTexture::Create(
|
|
151
16
|
wgpu::Device device,
|
|
152
17
|
std::shared_ptr<GPUExternalTextureDescriptor> descriptor) {
|
|
18
|
+
static constexpr const char *kCaller = "GPUDevice::importExternalTexture";
|
|
153
19
|
if (!descriptor || !descriptor->source) {
|
|
154
20
|
throw std::runtime_error(
|
|
155
|
-
"
|
|
156
|
-
"is required");
|
|
157
|
-
}
|
|
158
|
-
const auto &source = descriptor->source;
|
|
159
|
-
const auto &frame = source->handle();
|
|
160
|
-
if (frame.handle == nullptr) {
|
|
161
|
-
throw std::runtime_error(
|
|
162
|
-
"GPUExternalTexture::Create(): VideoFrame has been released");
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
#if defined(__APPLE__)
|
|
166
|
-
// 1. Import the IOSurface as SharedTextureMemory. For NV12 surfaces this
|
|
167
|
-
// yields a biplanar texture; for BGRA, a single-plane one.
|
|
168
|
-
wgpu::SharedTextureMemoryDescriptor memDesc{};
|
|
169
|
-
std::string label = descriptor->label.value_or("external-texture");
|
|
170
|
-
if (!label.empty()) {
|
|
171
|
-
memDesc.label = wgpu::StringView(label.c_str(), label.size());
|
|
172
|
-
}
|
|
173
|
-
wgpu::SharedTextureMemoryIOSurfaceDescriptor platformDesc{};
|
|
174
|
-
platformDesc.ioSurface = frame.handle;
|
|
175
|
-
// ExternalTexture views are sampled-only; storage binding isn't needed and
|
|
176
|
-
// for biplanar formats it would fail validation.
|
|
177
|
-
platformDesc.allowStorageBinding = false;
|
|
178
|
-
memDesc.nextInChain = &platformDesc;
|
|
179
|
-
auto memory = device.ImportSharedTextureMemory(&memDesc);
|
|
180
|
-
if (memory == nullptr) {
|
|
181
|
-
throw std::runtime_error(
|
|
182
|
-
"GPUExternalTexture::Create(): ImportSharedTextureMemory "
|
|
183
|
-
"returned null. Is 'shared-texture-memory-iosurface' enabled?");
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
// 2. Create the texture from the surface. We pass the right format
|
|
187
|
-
// explicitly so Dawn picks the multi-planar variant on NV12.
|
|
188
|
-
bool isYuv = frame.pixelFormat == VideoPixelFormat::NV12;
|
|
189
|
-
auto texture = memory.CreateTexture();
|
|
190
|
-
if (texture == nullptr) {
|
|
191
|
-
throw std::runtime_error(
|
|
192
|
-
"GPUExternalTexture::Create(): CreateTexture returned null");
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
// 3. Begin access on the underlying memory. The matching EndAccess runs when
|
|
196
|
-
// the GPUExternalTexture is destroyed (explicitly via destroy() or at GC).
|
|
197
|
-
wgpu::SharedTextureMemoryBeginAccessDescriptor begin{};
|
|
198
|
-
begin.initialized = true;
|
|
199
|
-
begin.concurrentRead = false;
|
|
200
|
-
if (memory.BeginAccess(texture, &begin) != wgpu::Status::Success) {
|
|
201
|
-
throw std::runtime_error(
|
|
202
|
-
"GPUExternalTexture::Create(): BeginAccess failed");
|
|
21
|
+
"GPUDevice::importExternalTexture(): descriptor.source "
|
|
22
|
+
"(NativeVideoFrame) is required");
|
|
203
23
|
}
|
|
204
|
-
|
|
205
|
-
// 4. Build plane views. For NV12 we need plane0 = R8 luma and plane1 = RG8
|
|
206
|
-
// chroma; for BGRA we only set plane0.
|
|
207
|
-
wgpu::TextureView plane0;
|
|
208
|
-
wgpu::TextureView plane1;
|
|
209
|
-
{
|
|
210
|
-
wgpu::TextureViewDescriptor v{};
|
|
211
|
-
v.aspect =
|
|
212
|
-
isYuv ? wgpu::TextureAspect::Plane0Only : wgpu::TextureAspect::All;
|
|
213
|
-
plane0 = texture.CreateView(&v);
|
|
214
|
-
}
|
|
215
|
-
if (isYuv) {
|
|
216
|
-
wgpu::TextureViewDescriptor v{};
|
|
217
|
-
v.aspect = wgpu::TextureAspect::Plane1Only;
|
|
218
|
-
plane1 = texture.CreateView(&v);
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
// 5. Build the ExternalTextureDescriptor. Dawn converts the YUV planes to
|
|
222
|
-
// RGB in the destination color space while sampling, with the matrix,
|
|
223
|
-
// range, transfer function and gamut of the frame's color space (from the
|
|
224
|
-
// CVPixelBuffer attachments; BT.709 limited range, the norm for HD video,
|
|
225
|
-
// where they are missing). A BGRA frame is RGB already: no-op parameters.
|
|
226
|
-
dawn::ExternalTextureColorSpaceParams params =
|
|
227
|
-
isYuv ? colorSpaceParams(
|
|
228
|
-
resolveColorSpace(frame.colorSpace,
|
|
229
|
-
wgpu::ColorSpaceYCbCrMatrixDawn::Rec709,
|
|
230
|
-
wgpu::ColorSpaceYCbCrRangeDawn::Narrow),
|
|
231
|
-
descriptor->colorSpace)
|
|
232
|
-
: dawn::GetNoopColorSpaceParams();
|
|
233
|
-
wgpu::ExternalTextureDescriptor extDesc{};
|
|
234
|
-
if (!label.empty()) {
|
|
235
|
-
extDesc.label = wgpu::StringView(label.c_str(), label.size());
|
|
236
|
-
}
|
|
237
|
-
extDesc.plane0 = plane0;
|
|
238
|
-
if (isYuv) {
|
|
239
|
-
extDesc.plane1 = plane1;
|
|
240
|
-
}
|
|
241
|
-
applyColorSpaceParams(extDesc, params);
|
|
242
|
-
extDesc.cropOrigin = {0, 0};
|
|
243
|
-
extDesc.cropSize = {frame.width, frame.height};
|
|
244
|
-
extDesc.apparentSize = {frame.width, frame.height};
|
|
245
|
-
extDesc.mirrored = descriptor->mirrored.value_or(false);
|
|
246
|
-
extDesc.rotation =
|
|
247
|
-
toExternalTextureRotation(descriptor->rotation.value_or(0));
|
|
248
|
-
|
|
249
|
-
auto external = device.CreateExternalTexture(&extDesc);
|
|
250
|
-
if (external == nullptr) {
|
|
251
|
-
endAccess(memory, texture);
|
|
252
|
-
throw std::runtime_error(
|
|
253
|
-
"GPUExternalTexture::Create(): CreateExternalTexture returned "
|
|
254
|
-
"null");
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
return std::make_shared<GPUExternalTexture>(
|
|
258
|
-
std::move(external), std::move(memory), std::move(texture),
|
|
259
|
-
std::move(descriptor->source), std::move(label));
|
|
260
|
-
#elif defined(__ANDROID__)
|
|
261
|
-
// 0. A YUV frame imports as an OpaqueYCbCrAndroid texture, which Dawn only
|
|
262
|
-
// accepts on a device with OpaqueYCbCrAndroidForExternalTexture.
|
|
263
|
-
// requestDevice() enables the feature whenever the adapter supports it;
|
|
264
|
-
// a device created by another library (importDevice) has to request it
|
|
265
|
-
// itself. Fail with the cause here: Dawn's own report is a bare
|
|
266
|
-
// "Unsupported texture format TextureFormat::OpaqueYCbCrAndroid" device
|
|
267
|
-
// error, followed by the failure of every call made with the result.
|
|
268
|
-
if (!device.HasFeature(
|
|
269
|
-
wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture)) {
|
|
270
|
-
AHardwareBuffer_Desc ahbDesc{};
|
|
271
|
-
AHardwareBuffer_describe(static_cast<AHardwareBuffer *>(frame.handle),
|
|
272
|
-
&ahbDesc);
|
|
273
|
-
if (isOpaqueYCbCrFormat(ahbDesc.format)) {
|
|
274
|
-
throw std::runtime_error(
|
|
275
|
-
"GPUExternalTexture::Create(): a YUV frame can only be imported on "
|
|
276
|
-
"a device with the 'opaque-ycbcr-android-for-external-texture' "
|
|
277
|
-
"feature. requestDevice() enables it whenever the adapter supports "
|
|
278
|
-
"it; a device imported from another library has to request it "
|
|
279
|
-
"itself.");
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
// 1. Import the AHardwareBuffer as SharedTextureMemory. For YUV AHBs this
|
|
284
|
-
// yields a Dawn texture in the implementation-defined OpaqueYCbCrAndroid
|
|
285
|
-
// format; for RGBA AHBs, a regular single-plane texture.
|
|
286
|
-
wgpu::SharedTextureMemoryDescriptor memDesc{};
|
|
24
|
+
const auto &frame = descriptor->source->handle();
|
|
287
25
|
std::string label = descriptor->label.value_or("external-texture");
|
|
288
|
-
if (!label.empty()) {
|
|
289
|
-
memDesc.label = wgpu::StringView(label.c_str(), label.size());
|
|
290
|
-
}
|
|
291
|
-
wgpu::SharedTextureMemoryAHardwareBufferDescriptor platformDesc{};
|
|
292
|
-
platformDesc.handle = frame.handle;
|
|
293
|
-
memDesc.nextInChain = &platformDesc;
|
|
294
|
-
auto memory = device.ImportSharedTextureMemory(&memDesc);
|
|
295
|
-
if (memory == nullptr) {
|
|
296
|
-
throw std::runtime_error(
|
|
297
|
-
"GPUExternalTexture::Create(): ImportSharedTextureMemory "
|
|
298
|
-
"returned null. Is 'shared-texture-memory-ahardware-buffer' enabled?");
|
|
299
|
-
}
|
|
300
26
|
|
|
301
|
-
//
|
|
302
|
-
//
|
|
27
|
+
// 1. Import the buffer as shared texture memory, and create the texture.
|
|
28
|
+
// No descriptor: Dawn picks the right format (biplanar for an NV12
|
|
29
|
+
// IOSurface, OpaqueYCbCrAndroid for a YUV AHardwareBuffer, R8 / RGBA8 /
|
|
30
|
+
// ... for color buffers).
|
|
31
|
+
auto memory = native_frame::importMemory(device, frame, label, kCaller);
|
|
303
32
|
auto texture = memory.CreateTexture();
|
|
304
33
|
if (texture == nullptr) {
|
|
305
34
|
throw std::runtime_error(
|
|
306
|
-
"
|
|
35
|
+
"GPUDevice::importExternalTexture(): CreateTexture returned null");
|
|
307
36
|
}
|
|
308
37
|
|
|
309
|
-
//
|
|
310
|
-
//
|
|
311
|
-
//
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
// makes the GPU wait on it before any work that samples the texture.
|
|
318
|
-
// Dawn duplicates the descriptor on import; the frame keeps its own.
|
|
319
|
-
wgpu::SharedTextureMemoryBeginAccessDescriptor begin{};
|
|
320
|
-
begin.initialized = true;
|
|
321
|
-
begin.concurrentRead = false;
|
|
322
|
-
wgpu::SharedTextureMemoryVkImageLayoutBeginState beginLayout{};
|
|
323
|
-
begin.nextInChain = &beginLayout;
|
|
324
|
-
wgpu::SharedFence acquireFence;
|
|
325
|
-
const uint64_t kBinaryFenceSignaled = 1;
|
|
326
|
-
if (frame.acquireFenceFd >= 0) {
|
|
327
|
-
if (device.HasFeature(wgpu::FeatureName::SharedFenceSyncFD)) {
|
|
328
|
-
wgpu::SharedFenceSyncFDDescriptor fdDesc{};
|
|
329
|
-
fdDesc.handle = frame.acquireFenceFd;
|
|
330
|
-
wgpu::SharedFenceDescriptor fenceDesc{};
|
|
331
|
-
fenceDesc.nextInChain = &fdDesc;
|
|
332
|
-
acquireFence = device.ImportSharedFence(&fenceDesc);
|
|
333
|
-
if (acquireFence) {
|
|
334
|
-
begin.fenceCount = 1;
|
|
335
|
-
begin.fences = &acquireFence;
|
|
336
|
-
begin.signaledValueCount = 1;
|
|
337
|
-
begin.signaledValues = &kBinaryFenceSignaled;
|
|
338
|
-
}
|
|
339
|
-
} else {
|
|
340
|
-
// A device without shared-fence-sync-fd (imported from another library
|
|
341
|
-
// that did not request it) cannot wait on the GPU: wait here instead. A
|
|
342
|
-
// sync fd becomes readable once it signals.
|
|
343
|
-
pollfd pfd{};
|
|
344
|
-
pfd.fd = frame.acquireFenceFd;
|
|
345
|
-
pfd.events = POLLIN;
|
|
346
|
-
int polled = poll(&pfd, 1, 3000);
|
|
347
|
-
if (polled <= 0) {
|
|
348
|
-
__android_log_print(ANDROID_LOG_WARN, "GPUExternalTexture",
|
|
349
|
-
"the acquire fence of the frame did not signal "
|
|
350
|
-
"within 3s; sampling it anyway");
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
if (memory.BeginAccess(texture, &begin) != wgpu::Status::Success) {
|
|
355
|
-
throw std::runtime_error(
|
|
356
|
-
"GPUExternalTexture::Create(): BeginAccess failed");
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
// 4. The color conversion. Dawn imports every YUV AHardwareBuffer as an
|
|
360
|
-
// OpaqueYCbCrAndroid texture and samples it through a Vulkan YCbCr
|
|
361
|
-
// sampler whose own conversion is the identity (SamplerVk.cpp), so the
|
|
362
|
-
// sample is raw Y, Cb, Cr; the external texture then applies the
|
|
363
|
-
// descriptor's YCbCr matrix, transfer functions and gamut conversion
|
|
364
|
-
// while sampling (Dawn's YUV AHB specialization), the same way it does
|
|
365
|
-
// for the biplanar textures of the other backends. The frame's color
|
|
366
|
-
// space comes from its producer (Image.getDataSpace on API 33, the
|
|
367
|
-
// container's color info before that); what it leaves unknown falls back
|
|
368
|
-
// to the YCbCr model and range the driver suggests for the buffer, read
|
|
369
|
-
// from the shared memory's properties, and to BT.601 limited range, the
|
|
370
|
-
// camera norm, when the driver has no opinion. A color-format AHB (RGBA8,
|
|
371
|
-
// R8, ...) samples as RGB directly: no-op parameters.
|
|
372
|
-
dawn::ExternalTextureColorSpaceParams params =
|
|
373
|
-
dawn::GetNoopColorSpaceParams();
|
|
374
|
-
if (texture.GetFormat() == wgpu::TextureFormat::OpaqueYCbCrAndroid) {
|
|
375
|
-
auto fallbackMatrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec601;
|
|
376
|
-
auto fallbackRange = wgpu::ColorSpaceYCbCrRangeDawn::Narrow;
|
|
377
|
-
wgpu::SharedTextureMemoryAHardwareBufferProperties ahbProps{};
|
|
378
|
-
wgpu::SharedTextureMemoryProperties props{};
|
|
379
|
-
props.nextInChain = &ahbProps;
|
|
380
|
-
if (memory.GetProperties(&props) == wgpu::Status::Success) {
|
|
381
|
-
// VkSamplerYcbcrModelConversion / VkSamplerYcbcrRange values, inlined
|
|
382
|
-
// to keep this cross-platform file free of Vulkan headers.
|
|
383
|
-
switch (ahbProps.yCbCrInfo.vkYCbCrModel) {
|
|
384
|
-
case 0: // VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY: RGB data
|
|
385
|
-
fallbackMatrix = wgpu::ColorSpaceYCbCrMatrixDawn::Identity;
|
|
386
|
-
fallbackRange = wgpu::ColorSpaceYCbCrRangeDawn::Identity;
|
|
387
|
-
break;
|
|
388
|
-
case 2: // VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709
|
|
389
|
-
fallbackMatrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec709;
|
|
390
|
-
break;
|
|
391
|
-
case 4: // VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_2020
|
|
392
|
-
fallbackMatrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec2020;
|
|
393
|
-
break;
|
|
394
|
-
default: // YCBCR_601, YCBCR_IDENTITY and unknown models
|
|
395
|
-
break;
|
|
396
|
-
}
|
|
397
|
-
if (fallbackMatrix != wgpu::ColorSpaceYCbCrMatrixDawn::Identity) {
|
|
398
|
-
fallbackRange = ahbProps.yCbCrInfo.vkYCbCrRange == 0
|
|
399
|
-
? wgpu::ColorSpaceYCbCrRangeDawn::Full
|
|
400
|
-
: wgpu::ColorSpaceYCbCrRangeDawn::Narrow;
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
params = colorSpaceParams(
|
|
404
|
-
resolveColorSpace(frame.colorSpace, fallbackMatrix, fallbackRange),
|
|
405
|
-
descriptor->colorSpace);
|
|
38
|
+
// 2. Begin the access, after the producer's fence when the frame has one.
|
|
39
|
+
// The matching EndAccess runs when the GPUExternalTexture is destroyed
|
|
40
|
+
// (explicitly via destroy() or at GC).
|
|
41
|
+
std::vector<wgpu::SharedFence> fences;
|
|
42
|
+
std::vector<uint64_t> signaledValues;
|
|
43
|
+
if (auto acquire = native_frame::importAcquireFence(device, frame)) {
|
|
44
|
+
fences.push_back(std::move(acquire));
|
|
45
|
+
signaledValues.push_back(1);
|
|
406
46
|
}
|
|
47
|
+
native_frame::beginAccess(memory, texture, true, fences, signaledValues,
|
|
48
|
+
kCaller);
|
|
407
49
|
|
|
408
|
-
//
|
|
409
|
-
wgpu::
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
wgpu::TextureView plane0 = texture.CreateView();
|
|
417
|
-
extDesc.plane0 = plane0;
|
|
418
|
-
applyColorSpaceParams(extDesc, params);
|
|
419
|
-
extDesc.mirrored = descriptor->mirrored.value_or(false);
|
|
420
|
-
extDesc.rotation =
|
|
421
|
-
toExternalTextureRotation(descriptor->rotation.value_or(0));
|
|
422
|
-
|
|
423
|
-
auto external = device.CreateExternalTexture(&extDesc);
|
|
424
|
-
if (external == nullptr) {
|
|
425
|
-
endAccess(memory, texture);
|
|
426
|
-
throw std::runtime_error(
|
|
427
|
-
"GPUExternalTexture::Create(): CreateExternalTexture returned "
|
|
428
|
-
"null");
|
|
50
|
+
// 3. The external texture that samples the frame as RGB.
|
|
51
|
+
wgpu::ExternalTexture external;
|
|
52
|
+
try {
|
|
53
|
+
external = native_frame::createExternalTexture(
|
|
54
|
+
device, memory, texture, frame, *descriptor, label, kCaller);
|
|
55
|
+
} catch (...) {
|
|
56
|
+
native_frame::endAccess(memory, texture);
|
|
57
|
+
throw;
|
|
429
58
|
}
|
|
430
59
|
|
|
431
60
|
return std::make_shared<GPUExternalTexture>(
|
|
432
61
|
std::move(external), std::move(memory), std::move(texture),
|
|
433
62
|
std::move(descriptor->source), std::move(label));
|
|
434
|
-
#else
|
|
435
|
-
throw std::runtime_error(
|
|
436
|
-
"GPUExternalTexture::Create(): not yet implemented on this "
|
|
437
|
-
"platform");
|
|
438
|
-
#endif
|
|
439
63
|
}
|
|
440
64
|
|
|
441
65
|
} // namespace rnwgpu
|
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
#pragma once
|
|
2
2
|
|
|
3
|
-
#include <array>
|
|
4
3
|
#include <memory>
|
|
5
4
|
#include <string>
|
|
6
5
|
#include <utility>
|
|
7
|
-
#include <vector>
|
|
8
|
-
|
|
9
|
-
#include "Unions.h"
|
|
10
6
|
|
|
7
|
+
#include "NativeFrameImport.h"
|
|
11
8
|
#include "NativeObject.h"
|
|
12
9
|
#include "VideoFrame.h"
|
|
13
10
|
|
|
@@ -23,19 +20,20 @@ class GPUExternalTexture : public NativeObject<GPUExternalTexture> {
|
|
|
23
20
|
public:
|
|
24
21
|
static constexpr const char *CLASS_NAME = "GPUExternalTexture";
|
|
25
22
|
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
// destructor. Defined in GPUExternalTexture.cpp.
|
|
23
|
+
// importExternalTexture(): imports the frame of descriptor.source as shared
|
|
24
|
+
// texture memory, begins an access to it, and wraps the wgpu::ExternalTexture
|
|
25
|
+
// that samples it together with the import it owns. The matching EndAccess
|
|
26
|
+
// runs in destroy() / the destructor. Defined in GPUExternalTexture.cpp.
|
|
31
27
|
static std::shared_ptr<GPUExternalTexture>
|
|
32
28
|
Create(wgpu::Device device,
|
|
33
29
|
std::shared_ptr<GPUExternalTextureDescriptor> descriptor);
|
|
34
30
|
|
|
35
|
-
//
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
31
|
+
// `memory` and `texture` are the import this external texture owns:
|
|
32
|
+
// destroy() ends their access and releases them, so the producer (e.g. an
|
|
33
|
+
// AVPlayer) can reclaim the surface. A GPUNativeBufferExternalTexture
|
|
34
|
+
// passes null for both: it owns the import itself and brackets each use with
|
|
35
|
+
// its own beginAccess() / endAccess(), so destroy() here only releases the
|
|
36
|
+
// external texture.
|
|
39
37
|
GPUExternalTexture(wgpu::ExternalTexture instance,
|
|
40
38
|
wgpu::SharedTextureMemory memory, wgpu::Texture texture,
|
|
41
39
|
std::shared_ptr<VideoFrame> source, std::string label)
|
|
@@ -63,13 +61,7 @@ public:
|
|
|
63
61
|
// sampled it) is garbage-collected.
|
|
64
62
|
void destroy() {
|
|
65
63
|
if (_memory && _texture) {
|
|
66
|
-
endAccess(_memory, _texture);
|
|
67
|
-
}
|
|
68
|
-
if (_texture) {
|
|
69
|
-
_texture.Destroy();
|
|
70
|
-
}
|
|
71
|
-
if (_instance) {
|
|
72
|
-
_instance.Destroy();
|
|
64
|
+
native_frame::endAccess(_memory, _texture);
|
|
73
65
|
}
|
|
74
66
|
if (_texture) {
|
|
75
67
|
_texture.Destroy();
|
|
@@ -101,23 +93,6 @@ public:
|
|
|
101
93
|
|
|
102
94
|
inline const wgpu::ExternalTexture get() { return _instance; }
|
|
103
95
|
|
|
104
|
-
// End the shared-memory access window on `texture`. Shared by destroy() and
|
|
105
|
-
// the Create() error paths so every EndAccess carries the same chain: Dawn's
|
|
106
|
-
// Vulkan backend rejects the call unless a
|
|
107
|
-
// SharedTextureMemoryVkImageLayoutEndState is chained (it writes the
|
|
108
|
-
// released VkImage layouts there), while the Metal backend only accepts its
|
|
109
|
-
// own optional end state. The returned fences are dropped: Dawn keeps the
|
|
110
|
-
// texture alive for in-flight GPU work on its own.
|
|
111
|
-
static void endAccess(const wgpu::SharedTextureMemory &memory,
|
|
112
|
-
const wgpu::Texture &texture) {
|
|
113
|
-
wgpu::SharedTextureMemoryEndAccessState state{};
|
|
114
|
-
#if defined(__ANDROID__)
|
|
115
|
-
wgpu::SharedTextureMemoryVkImageLayoutEndState vkLayout{};
|
|
116
|
-
state.nextInChain = &vkLayout;
|
|
117
|
-
#endif
|
|
118
|
-
(void)memory.EndAccess(texture, &state);
|
|
119
|
-
}
|
|
120
|
-
|
|
121
96
|
private:
|
|
122
97
|
wgpu::ExternalTexture _instance;
|
|
123
98
|
wgpu::SharedTextureMemory _memory;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <memory>
|
|
4
|
+
#include <string>
|
|
5
|
+
#include <utility>
|
|
6
|
+
|
|
7
|
+
#include "webgpu/webgpu_cpp.h"
|
|
8
|
+
|
|
9
|
+
#include "GPUExternalTexture.h"
|
|
10
|
+
#include "GPUNativeBufferWrapper.h"
|
|
11
|
+
#include "VideoFrame.h"
|
|
12
|
+
|
|
13
|
+
namespace rnwgpu {
|
|
14
|
+
|
|
15
|
+
namespace jsi = facebook::jsi;
|
|
16
|
+
|
|
17
|
+
// A native frame buffer (the YUV AHardwareBuffer of a decoder or a camera, an
|
|
18
|
+
// NV12 or BGRA IOSurface) imported once as an external texture, with the
|
|
19
|
+
// access window of each use explicit: GPUHardwareBufferExternalTexture of
|
|
20
|
+
// Dawn's Android layer. Created by
|
|
21
|
+
// GPUDevice.createNativeBufferExternalTexture(). Unlike
|
|
22
|
+
// importExternalTexture(), which imports the buffer and begins the access on
|
|
23
|
+
// every call, the external texture here is built once per buffer, color
|
|
24
|
+
// conversion included, and only beginAccess() / endAccess() run per frame.
|
|
25
|
+
class GPUNativeBufferExternalTexture
|
|
26
|
+
: public GPUNativeBufferWrapper<GPUNativeBufferExternalTexture,
|
|
27
|
+
GPUExternalTexture> {
|
|
28
|
+
public:
|
|
29
|
+
static constexpr const char *CLASS_NAME = "GPUNativeBufferExternalTexture";
|
|
30
|
+
|
|
31
|
+
GPUNativeBufferExternalTexture(
|
|
32
|
+
wgpu::Device device, wgpu::SharedTextureMemory memory,
|
|
33
|
+
wgpu::Texture texture,
|
|
34
|
+
std::shared_ptr<GPUExternalTexture> externalTexture,
|
|
35
|
+
std::shared_ptr<VideoFrame> source, std::string label)
|
|
36
|
+
: GPUNativeBufferWrapper(std::move(device), std::move(memory),
|
|
37
|
+
std::move(texture), std::move(externalTexture),
|
|
38
|
+
std::move(source), std::move(label)) {}
|
|
39
|
+
|
|
40
|
+
// The external texture to bind (a texture_external binding). Sample it only
|
|
41
|
+
// between beginAccess() and endAccess().
|
|
42
|
+
std::shared_ptr<GPUExternalTexture> getExternalTexture() { return _wrapped; }
|
|
43
|
+
|
|
44
|
+
// Releases the external texture and the import (see
|
|
45
|
+
// NativeBufferAccess::destroy), the order of Dawn's layer.
|
|
46
|
+
void destroy() {
|
|
47
|
+
_wrapped->destroy();
|
|
48
|
+
_access.destroy();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
static void definePrototype(jsi::Runtime &runtime, jsi::Object &prototype) {
|
|
52
|
+
defineWrapperPrototype(runtime, prototype);
|
|
53
|
+
installGetter(runtime, prototype, "externalTexture",
|
|
54
|
+
&GPUNativeBufferExternalTexture::getExternalTexture);
|
|
55
|
+
installMethod(runtime, prototype, "destroy",
|
|
56
|
+
&GPUNativeBufferExternalTexture::destroy);
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
} // namespace rnwgpu
|