react-native-webgpu 0.15.2 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/CMakeLists.txt +5 -0
- package/android/cpp/cpp-adapter.cpp +109 -11
- package/android/src/main/java/com/webgpu/SurfaceFrameRateVote.java +97 -85
- package/android/src/main/java/com/webgpu/WebGPUView.java +13 -8
- package/apple/ApplePlatformContext.h +1 -0
- package/cpp/jsi/JSICache.cpp +2 -0
- package/cpp/jsi/NativeObject.h +36 -12
- package/cpp/rnwgpu/RNWebGPUManager.cpp +4 -0
- package/cpp/rnwgpu/SurfaceRegistry.h +4 -4
- package/cpp/rnwgpu/api/GPUDevice.cpp +96 -0
- package/cpp/rnwgpu/api/GPUDevice.h +22 -0
- package/cpp/rnwgpu/api/GPUError.h +1 -0
- package/cpp/rnwgpu/api/GPUExtent2D.h +56 -0
- package/cpp/rnwgpu/api/GPUExternalTexture.cpp +32 -408
- package/cpp/rnwgpu/api/GPUExternalTexture.h +12 -37
- package/cpp/rnwgpu/api/GPUInternalError.h +1 -0
- package/cpp/rnwgpu/api/GPUNativeBufferExternalTexture.h +60 -0
- package/cpp/rnwgpu/api/GPUNativeBufferTexture.h +52 -0
- package/cpp/rnwgpu/api/GPUNativeBufferWrapper.cpp +144 -0
- package/cpp/rnwgpu/api/GPUNativeBufferWrapper.h +142 -0
- package/cpp/rnwgpu/api/GPUOutOfMemoryError.h +1 -0
- package/cpp/rnwgpu/api/GPUQueue.h +1 -0
- package/cpp/rnwgpu/api/GPUSharedFence.h +1 -0
- package/cpp/rnwgpu/api/GPUSharedTextureMemory.cpp +12 -44
- package/cpp/rnwgpu/api/GPUSharedTextureMemory.h +1 -0
- package/cpp/rnwgpu/api/GPUUncapturedErrorEvent.h +1 -0
- package/cpp/rnwgpu/api/GPUValidationError.h +1 -0
- package/cpp/rnwgpu/api/NativeFrameImport.cpp +638 -0
- package/cpp/rnwgpu/api/NativeFrameImport.h +86 -0
- package/cpp/rnwgpu/api/VideoFrame.h +21 -7
- package/cpp/rnwgpu/api/descriptors/GPUExternalTextureDescriptor.h +153 -1
- package/cpp/rnwgpu/api/descriptors/GPUNativeBufferTextureDescriptor.h +55 -0
- package/cpp/rnwgpu/async/RuntimeContext.cpp +1 -0
- package/lib/commonjs/Canvas.js.map +1 -1
- package/lib/commonjs/index.js.map +1 -1
- package/lib/module/Canvas.js.map +1 -1
- package/lib/module/index.js.map +1 -1
- package/lib/typescript/src/Canvas.d.ts +2 -31
- package/lib/typescript/src/Canvas.d.ts.map +1 -1
- package/lib/typescript/src/index.d.ts +4 -2
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/types.d.ts +33 -0
- package/lib/typescript/src/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/Canvas.tsx +3 -36
- package/src/index.tsx +24 -0
- package/src/types.ts +97 -0
|
@@ -0,0 +1,638 @@
|
|
|
1
|
+
#include "NativeFrameImport.h"
|
|
2
|
+
|
|
3
|
+
#include <array>
|
|
4
|
+
#include <cmath>
|
|
5
|
+
#include <optional>
|
|
6
|
+
#include <stdexcept>
|
|
7
|
+
#include <string>
|
|
8
|
+
#include <utility>
|
|
9
|
+
#include <vector>
|
|
10
|
+
|
|
11
|
+
#include "GPUExternalTextureDescriptor.h"
|
|
12
|
+
|
|
13
|
+
#include "dawn/common/ExternalTextureParams.h"
|
|
14
|
+
|
|
15
|
+
#if defined(__ANDROID__)
|
|
16
|
+
#include <android/hardware_buffer.h>
|
|
17
|
+
#include <android/log.h>
|
|
18
|
+
#include <android/sync.h>
|
|
19
|
+
#include <fcntl.h>
|
|
20
|
+
#include <poll.h>
|
|
21
|
+
#include <unistd.h>
|
|
22
|
+
#endif
|
|
23
|
+
|
|
24
|
+
namespace rnwgpu {
|
|
25
|
+
namespace native_frame {
|
|
26
|
+
|
|
27
|
+
namespace {
|
|
28
|
+
|
|
29
|
+
std::string prefix(const char *caller) { return std::string(caller) + "(): "; }
|
|
30
|
+
|
|
31
|
+
#if defined(__ANDROID__)
|
|
32
|
+
// VK_IMAGE_LAYOUT_UNDEFINED / VK_IMAGE_LAYOUT_GENERAL, inlined to keep this
|
|
33
|
+
// cross-platform file free of Vulkan headers.
|
|
34
|
+
constexpr uint32_t kVkImageLayoutUndefined = 0;
|
|
35
|
+
constexpr uint32_t kVkImageLayoutGeneral = 1;
|
|
36
|
+
|
|
37
|
+
// Whether Dawn imports an AHardwareBuffer of this format as an
|
|
38
|
+
// OpaqueYCbCrAndroid texture. Mirrors FormatFromAHardwareBufferFormat
|
|
39
|
+
// (dawn/native/AHBFunctions.cpp): the color and depth formats with a WebGPU
|
|
40
|
+
// equivalent import as that format; the YUV formats and the formats outside
|
|
41
|
+
// the public enum, which drivers use for the camera and the video decoder,
|
|
42
|
+
// import as OpaqueYCbCrAndroid.
|
|
43
|
+
bool isOpaqueYCbCrFormat(uint32_t format) {
|
|
44
|
+
switch (format) {
|
|
45
|
+
case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM:
|
|
46
|
+
case AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM:
|
|
47
|
+
case AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM:
|
|
48
|
+
case AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM:
|
|
49
|
+
case AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT:
|
|
50
|
+
case AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM:
|
|
51
|
+
case AHARDWAREBUFFER_FORMAT_BLOB:
|
|
52
|
+
case AHARDWAREBUFFER_FORMAT_D16_UNORM:
|
|
53
|
+
case AHARDWAREBUFFER_FORMAT_D24_UNORM:
|
|
54
|
+
case AHARDWAREBUFFER_FORMAT_D24_UNORM_S8_UINT:
|
|
55
|
+
case AHARDWAREBUFFER_FORMAT_D32_FLOAT:
|
|
56
|
+
case AHARDWAREBUFFER_FORMAT_D32_FLOAT_S8_UINT:
|
|
57
|
+
case AHARDWAREBUFFER_FORMAT_S8_UINT:
|
|
58
|
+
// Single-channel formats of the API 33 and 34 headers, by value so that
|
|
59
|
+
// older NDKs compile.
|
|
60
|
+
case 0x38: // AHARDWAREBUFFER_FORMAT_R8_UNORM
|
|
61
|
+
case 0x39: // AHARDWAREBUFFER_FORMAT_R16_UINT
|
|
62
|
+
case 0x3a: // AHARDWAREBUFFER_FORMAT_R16G16_UINT
|
|
63
|
+
case 0x3b: // AHARDWAREBUFFER_FORMAT_R10G10B10A10_UNORM
|
|
64
|
+
return false;
|
|
65
|
+
default:
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// A sync fd owned by this scope.
|
|
71
|
+
struct UniqueFd {
|
|
72
|
+
int fd = -1;
|
|
73
|
+
UniqueFd() = default;
|
|
74
|
+
explicit UniqueFd(int f) : fd(f) {}
|
|
75
|
+
UniqueFd(const UniqueFd &) = delete;
|
|
76
|
+
UniqueFd &operator=(const UniqueFd &) = delete;
|
|
77
|
+
UniqueFd(UniqueFd &&other) noexcept : fd(other.fd) { other.fd = -1; }
|
|
78
|
+
UniqueFd &operator=(UniqueFd &&other) noexcept {
|
|
79
|
+
if (this != &other) {
|
|
80
|
+
reset();
|
|
81
|
+
fd = other.fd;
|
|
82
|
+
other.fd = -1;
|
|
83
|
+
}
|
|
84
|
+
return *this;
|
|
85
|
+
}
|
|
86
|
+
~UniqueFd() { reset(); }
|
|
87
|
+
void reset() {
|
|
88
|
+
if (fd >= 0) {
|
|
89
|
+
::close(fd);
|
|
90
|
+
}
|
|
91
|
+
fd = -1;
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
#endif
|
|
95
|
+
|
|
96
|
+
// The color space of a YUV frame with every member known. The producer reports
|
|
97
|
+
// what it knows (VideoFrameHandle.colorSpace), the caller may override members
|
|
98
|
+
// (GPUExternalTextureDescriptor.sourceColorSpace), and the matrix and range
|
|
99
|
+
// both leave out come from `fallbackMatrix` / `fallbackRange`, the platform's
|
|
100
|
+
// best guess for the buffer. The rest follows from the matrix: the primaries
|
|
101
|
+
// of the standard that defines it, and the SMPTE 170M transfer function that
|
|
102
|
+
// BT.601, BT.709 and BT.2020 SDR video is encoded with. An identity matrix
|
|
103
|
+
// means the samples are RGB already, which the identity transfer leaves
|
|
104
|
+
// untouched.
|
|
105
|
+
wgpu::ColorSpaceDawn
|
|
106
|
+
resolveColorSpace(const VideoFrameColorSpace &reported,
|
|
107
|
+
const std::optional<VideoFrameColorSpace> &override,
|
|
108
|
+
wgpu::ColorSpaceYCbCrMatrixDawn fallbackMatrix,
|
|
109
|
+
wgpu::ColorSpaceYCbCrRangeDawn fallbackRange) {
|
|
110
|
+
VideoFrameColorSpace source = reported;
|
|
111
|
+
if (override.has_value()) {
|
|
112
|
+
if (override->matrix.has_value()) {
|
|
113
|
+
source.matrix = override->matrix;
|
|
114
|
+
}
|
|
115
|
+
if (override->range.has_value()) {
|
|
116
|
+
source.range = override->range;
|
|
117
|
+
}
|
|
118
|
+
if (override->primaries.has_value()) {
|
|
119
|
+
source.primaries = override->primaries;
|
|
120
|
+
}
|
|
121
|
+
if (override->transfer.has_value()) {
|
|
122
|
+
source.transfer = override->transfer;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
wgpu::ColorSpaceDawn colorSpace{};
|
|
126
|
+
colorSpace.yCbCrMatrix = source.matrix.value_or(fallbackMatrix);
|
|
127
|
+
colorSpace.yCbCrRange = source.range.value_or(fallbackRange);
|
|
128
|
+
if (source.primaries.has_value()) {
|
|
129
|
+
colorSpace.primaries = *source.primaries;
|
|
130
|
+
} else {
|
|
131
|
+
switch (colorSpace.yCbCrMatrix) {
|
|
132
|
+
case wgpu::ColorSpaceYCbCrMatrixDawn::Rec601:
|
|
133
|
+
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::Rec601;
|
|
134
|
+
break;
|
|
135
|
+
case wgpu::ColorSpaceYCbCrMatrixDawn::Rec2020:
|
|
136
|
+
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::Rec2020;
|
|
137
|
+
break;
|
|
138
|
+
default:
|
|
139
|
+
// BT.709 shares the sRGB primaries (Dawn's helper only knows them under
|
|
140
|
+
// that name).
|
|
141
|
+
colorSpace.primaries = wgpu::ColorSpacePrimariesDawn::SRGB;
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (source.transfer.has_value()) {
|
|
146
|
+
colorSpace.transfer = *source.transfer;
|
|
147
|
+
} else {
|
|
148
|
+
colorSpace.transfer =
|
|
149
|
+
colorSpace.yCbCrMatrix == wgpu::ColorSpaceYCbCrMatrixDawn::Identity
|
|
150
|
+
? wgpu::ColorSpaceTransferDawn::Identity
|
|
151
|
+
: wgpu::ColorSpaceTransferDawn::SMPTE_170M;
|
|
152
|
+
}
|
|
153
|
+
return colorSpace;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// The ExternalTextureDescriptor parameters that convert samples in `source` to
|
|
157
|
+
// RGB in the color space the descriptor asks for ("srgb" by default,
|
|
158
|
+
// "display-p3"): the YCbCr matrix, the range, the source transfer function,
|
|
159
|
+
// the gamut conversion and the destination transfer function. Dawn applies
|
|
160
|
+
// them while sampling the external texture. The helper is Dawn's own, the one
|
|
161
|
+
// Chromium feeds video frames through.
|
|
162
|
+
dawn::ExternalTextureColorSpaceParams
|
|
163
|
+
colorSpaceParams(const wgpu::ColorSpaceDawn &source,
|
|
164
|
+
const std::optional<std::string> &destination,
|
|
165
|
+
const char *caller) {
|
|
166
|
+
auto target = destination.has_value() && *destination == "display-p3"
|
|
167
|
+
? wgpu::PredefinedColorSpace::DisplayP3
|
|
168
|
+
: wgpu::PredefinedColorSpace::SRGB;
|
|
169
|
+
dawn::ExternalTextureColorSpaceParams params;
|
|
170
|
+
if (dawn::ComputeExternalTextureParams(source, target, ¶ms) !=
|
|
171
|
+
wgpu::Status::Success) {
|
|
172
|
+
throw std::runtime_error(
|
|
173
|
+
prefix(caller) +
|
|
174
|
+
"the color space of the frame cannot be converted to '" +
|
|
175
|
+
destination.value_or("srgb") + "'");
|
|
176
|
+
}
|
|
177
|
+
return params;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// Dawn dereferences the four arrays unconditionally, so a descriptor always
|
|
181
|
+
// carries a full set, the no-op one for frames that are RGB already.
|
|
182
|
+
void applyColorSpaceParams(
|
|
183
|
+
wgpu::ExternalTextureDescriptor &descriptor,
|
|
184
|
+
const dawn::ExternalTextureColorSpaceParams ¶ms) {
|
|
185
|
+
descriptor.yuvToRgbConversionMatrix = params.yuvToRgbConversionMatrix.data();
|
|
186
|
+
descriptor.srcTransferFunctionParameters = params.srcTransferFunction.data();
|
|
187
|
+
descriptor.dstTransferFunctionParameters = params.dstTransferFunction.data();
|
|
188
|
+
descriptor.gamutConversionMatrix = params.gamutConversionMatrix.data();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Map a rotation in degrees (0 / 90 / 180 / 270) to Dawn's enum. Anything that
|
|
192
|
+
// isn't a clean multiple of 90 snaps to the nearest quadrant; Dawn only
|
|
193
|
+
// supports those four steps for external textures.
|
|
194
|
+
wgpu::ExternalTextureRotation toExternalTextureRotation(double degrees) {
|
|
195
|
+
int quadrant = static_cast<int>(std::lround(degrees / 90.0)) & 3;
|
|
196
|
+
switch (quadrant) {
|
|
197
|
+
case 1:
|
|
198
|
+
return wgpu::ExternalTextureRotation::Rotate90Degrees;
|
|
199
|
+
case 2:
|
|
200
|
+
return wgpu::ExternalTextureRotation::Rotate180Degrees;
|
|
201
|
+
case 3:
|
|
202
|
+
return wgpu::ExternalTextureRotation::Rotate270Degrees;
|
|
203
|
+
default:
|
|
204
|
+
return wgpu::ExternalTextureRotation::Rotate0Degrees;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// A crop coordinate from JS, truncated like a GPUIntegerCoordinate. Throws
|
|
209
|
+
// unless it is a number in [0, max]: a cast of a negative or out of range
|
|
210
|
+
// double to uint32_t is undefined.
|
|
211
|
+
uint32_t cropCoordinate(double value, uint32_t max, const char *name,
|
|
212
|
+
const char *caller) {
|
|
213
|
+
double truncated = std::trunc(value);
|
|
214
|
+
if (!std::isfinite(truncated) || truncated < 0 ||
|
|
215
|
+
truncated > static_cast<double>(max)) {
|
|
216
|
+
throw std::runtime_error(prefix(caller) + name + " must be in [0, " +
|
|
217
|
+
std::to_string(max) + "], got " +
|
|
218
|
+
std::to_string(value));
|
|
219
|
+
}
|
|
220
|
+
return static_cast<uint32_t>(truncated);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
} // namespace
|
|
224
|
+
|
|
225
|
+
wgpu::SharedTextureMemory importMemory(const wgpu::Device &device,
|
|
226
|
+
const VideoFrameHandle &frame,
|
|
227
|
+
const std::string &label,
|
|
228
|
+
const char *caller) {
|
|
229
|
+
if (frame.handle == nullptr) {
|
|
230
|
+
throw std::runtime_error(prefix(caller) + "the frame has been released");
|
|
231
|
+
}
|
|
232
|
+
wgpu::SharedTextureMemoryDescriptor memDesc{};
|
|
233
|
+
if (!label.empty()) {
|
|
234
|
+
memDesc.label = wgpu::StringView(label.c_str(), label.size());
|
|
235
|
+
}
|
|
236
|
+
#if defined(__APPLE__)
|
|
237
|
+
// The IOSurface imports as a biplanar texture when it is NV12, as a
|
|
238
|
+
// single-plane one when it is BGRA. ExternalTexture views are sampled-only;
|
|
239
|
+
// storage binding isn't needed and for biplanar formats it would fail
|
|
240
|
+
// validation.
|
|
241
|
+
wgpu::SharedTextureMemoryIOSurfaceDescriptor platformDesc{};
|
|
242
|
+
platformDesc.ioSurface = frame.handle;
|
|
243
|
+
platformDesc.allowStorageBinding = false;
|
|
244
|
+
memDesc.nextInChain = &platformDesc;
|
|
245
|
+
#elif defined(__ANDROID__)
|
|
246
|
+
// A YUV buffer imports as an OpaqueYCbCrAndroid texture, which Dawn only
|
|
247
|
+
// accepts on a device with OpaqueYCbCrAndroidForExternalTexture.
|
|
248
|
+
// requestDevice() enables the feature whenever the adapter supports it; a
|
|
249
|
+
// device created by another library (importDevice) has to request it
|
|
250
|
+
// itself. Fail with the cause here: Dawn's own report is a bare
|
|
251
|
+
// "Unsupported texture format TextureFormat::OpaqueYCbCrAndroid" device
|
|
252
|
+
// error, followed by the failure of every call made with the result.
|
|
253
|
+
if (!device.HasFeature(
|
|
254
|
+
wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture)) {
|
|
255
|
+
AHardwareBuffer_Desc ahbDesc{};
|
|
256
|
+
AHardwareBuffer_describe(static_cast<AHardwareBuffer *>(frame.handle),
|
|
257
|
+
&ahbDesc);
|
|
258
|
+
if (isOpaqueYCbCrFormat(ahbDesc.format)) {
|
|
259
|
+
throw std::runtime_error(
|
|
260
|
+
prefix(caller) +
|
|
261
|
+
"a YUV buffer can only be imported on a device with the "
|
|
262
|
+
"'opaque-ycbcr-android-for-external-texture' feature. "
|
|
263
|
+
"requestDevice() enables it whenever the adapter supports it; a "
|
|
264
|
+
"device imported from another library has to request it itself.");
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
wgpu::SharedTextureMemoryAHardwareBufferDescriptor platformDesc{};
|
|
268
|
+
platformDesc.handle = frame.handle;
|
|
269
|
+
memDesc.nextInChain = &platformDesc;
|
|
270
|
+
#else
|
|
271
|
+
throw std::runtime_error(prefix(caller) + "unsupported platform");
|
|
272
|
+
#endif
|
|
273
|
+
auto memory = device.ImportSharedTextureMemory(&memDesc);
|
|
274
|
+
// Dawn reports a failed import (feature missing, unusable handle) as a
|
|
275
|
+
// validation error and returns an error object rather than null, with no
|
|
276
|
+
// format in its properties. Fail the import instead.
|
|
277
|
+
wgpu::SharedTextureMemoryProperties properties{};
|
|
278
|
+
if (memory == nullptr ||
|
|
279
|
+
memory.GetProperties(&properties) != wgpu::Status::Success ||
|
|
280
|
+
properties.format == wgpu::TextureFormat::Undefined) {
|
|
281
|
+
throw std::runtime_error(
|
|
282
|
+
prefix(caller) +
|
|
283
|
+
"the import failed - is the 'shared-texture-memory-iosurface' "
|
|
284
|
+
"(Apple) or 'shared-texture-memory-ahardware-buffer' (Android) "
|
|
285
|
+
"feature enabled on the device, and the handle a usable IOSurface / "
|
|
286
|
+
"AHardwareBuffer?");
|
|
287
|
+
}
|
|
288
|
+
return memory;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
void beginAccess(const wgpu::SharedTextureMemory &memory,
|
|
292
|
+
const wgpu::Texture &texture, bool initialized,
|
|
293
|
+
const std::vector<wgpu::SharedFence> &fences,
|
|
294
|
+
const std::vector<uint64_t> &signaledValues,
|
|
295
|
+
const char *caller) {
|
|
296
|
+
wgpu::SharedTextureMemoryBeginAccessDescriptor desc{};
|
|
297
|
+
desc.initialized = initialized;
|
|
298
|
+
desc.concurrentRead = false;
|
|
299
|
+
desc.fenceCount = fences.size();
|
|
300
|
+
desc.fences = fences.empty() ? nullptr : fences.data();
|
|
301
|
+
desc.signaledValueCount = signaledValues.size();
|
|
302
|
+
desc.signaledValues =
|
|
303
|
+
signaledValues.empty() ? nullptr : signaledValues.data();
|
|
304
|
+
#if defined(__ANDROID__)
|
|
305
|
+
// Dawn's Vulkan backend validates that the descriptor chains the layout to
|
|
306
|
+
// acquire the image in. For a buffer with pixels to keep, GENERAL on both
|
|
307
|
+
// ends, as Dawn's own Android layer passes: with UNDEFINED, Dawn's acquire
|
|
308
|
+
// barrier starts from UNDEFINED (see ImportedTextureBase::
|
|
309
|
+
// TransitionUsageForPass in TextureVk.cpp), which Vulkan defines as "the
|
|
310
|
+
// contents may be discarded", while the buffer holds the frame its producer
|
|
311
|
+
// just wrote. For a buffer whose contents the caller declares discardable
|
|
312
|
+
// (initialized: false), UNDEFINED is exactly that, and no prior producer
|
|
313
|
+
// put the image in GENERAL.
|
|
314
|
+
wgpu::SharedTextureMemoryVkImageLayoutBeginState vkLayout{};
|
|
315
|
+
vkLayout.oldLayout =
|
|
316
|
+
initialized ? kVkImageLayoutGeneral : kVkImageLayoutUndefined;
|
|
317
|
+
vkLayout.newLayout =
|
|
318
|
+
initialized ? kVkImageLayoutGeneral : kVkImageLayoutUndefined;
|
|
319
|
+
desc.nextInChain = &vkLayout;
|
|
320
|
+
#endif
|
|
321
|
+
if (memory.BeginAccess(texture, &desc) != wgpu::Status::Success) {
|
|
322
|
+
throw std::runtime_error(prefix(caller) + "BeginAccess failed");
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
void endAccess(const wgpu::SharedTextureMemory &memory,
|
|
327
|
+
const wgpu::Texture &texture,
|
|
328
|
+
wgpu::SharedTextureMemoryEndAccessState *state,
|
|
329
|
+
const char *caller) {
|
|
330
|
+
#if defined(__ANDROID__)
|
|
331
|
+
// Dawn's Vulkan backend writes the released VkImage layouts into a chained
|
|
332
|
+
// SharedTextureMemoryVkImageLayoutEndState; validation requires the chain
|
|
333
|
+
// even when the caller doesn't read the values. The Metal backend only
|
|
334
|
+
// accepts its own optional end state.
|
|
335
|
+
wgpu::SharedTextureMemoryVkImageLayoutEndState vkLayout{};
|
|
336
|
+
state->nextInChain = &vkLayout;
|
|
337
|
+
#endif
|
|
338
|
+
auto status = memory.EndAccess(texture, state);
|
|
339
|
+
#if defined(__ANDROID__)
|
|
340
|
+
// The chained state is this frame's: unchain it before `state` outlives it.
|
|
341
|
+
state->nextInChain = nullptr;
|
|
342
|
+
#endif
|
|
343
|
+
if (status != wgpu::Status::Success) {
|
|
344
|
+
throw std::runtime_error(prefix(caller) + "EndAccess failed");
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
void endAccess(const wgpu::SharedTextureMemory &memory,
|
|
349
|
+
const wgpu::Texture &texture) {
|
|
350
|
+
wgpu::SharedTextureMemoryEndAccessState state{};
|
|
351
|
+
#if defined(__ANDROID__)
|
|
352
|
+
wgpu::SharedTextureMemoryVkImageLayoutEndState vkLayout{};
|
|
353
|
+
state.nextInChain = &vkLayout;
|
|
354
|
+
#endif
|
|
355
|
+
(void)memory.EndAccess(texture, &state);
|
|
356
|
+
#if defined(__ANDROID__)
|
|
357
|
+
state.nextInChain = nullptr;
|
|
358
|
+
#endif
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
wgpu::SharedFence importAcquireFence(const wgpu::Device &device,
|
|
362
|
+
const VideoFrameHandle &frame) {
|
|
363
|
+
#if defined(__ANDROID__)
|
|
364
|
+
// The producer of the frame (the video decoder, the camera) may still be
|
|
365
|
+
// writing it: ImageReader hands a buffer out as soon as it is queued, and
|
|
366
|
+
// only waits on the producer's fence for CPU access. The fence travels with
|
|
367
|
+
// the frame as a sync fd; imported as a shared fence, BeginAccess makes the
|
|
368
|
+
// GPU wait on it before any work that samples the texture. Dawn duplicates
|
|
369
|
+
// the descriptor on import; the frame keeps its own.
|
|
370
|
+
if (frame.acquireFenceFd < 0) {
|
|
371
|
+
return nullptr;
|
|
372
|
+
}
|
|
373
|
+
if (device.HasFeature(wgpu::FeatureName::SharedFenceSyncFD)) {
|
|
374
|
+
wgpu::SharedFenceSyncFDDescriptor fdDesc{};
|
|
375
|
+
fdDesc.handle = frame.acquireFenceFd;
|
|
376
|
+
wgpu::SharedFenceDescriptor fenceDesc{};
|
|
377
|
+
fenceDesc.nextInChain = &fdDesc;
|
|
378
|
+
return device.ImportSharedFence(&fenceDesc);
|
|
379
|
+
}
|
|
380
|
+
// A device without shared-fence-sync-fd (imported from another library that
|
|
381
|
+
// did not request it) cannot wait on the GPU: wait here instead. A sync fd
|
|
382
|
+
// becomes readable once it signals.
|
|
383
|
+
pollfd pfd{};
|
|
384
|
+
pfd.fd = frame.acquireFenceFd;
|
|
385
|
+
pfd.events = POLLIN;
|
|
386
|
+
int polled = poll(&pfd, 1, 3000);
|
|
387
|
+
if (polled <= 0) {
|
|
388
|
+
__android_log_print(ANDROID_LOG_WARN, "GPUExternalTexture",
|
|
389
|
+
"the acquire fence of the frame did not signal "
|
|
390
|
+
"within 3s; sampling it anyway");
|
|
391
|
+
}
|
|
392
|
+
return nullptr;
|
|
393
|
+
#else
|
|
394
|
+
(void)device;
|
|
395
|
+
(void)frame;
|
|
396
|
+
return nullptr;
|
|
397
|
+
#endif
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
wgpu::ExternalTexture createExternalTexture(
|
|
401
|
+
const wgpu::Device &device, const wgpu::SharedTextureMemory &memory,
|
|
402
|
+
const wgpu::Texture &texture, const VideoFrameHandle &frame,
|
|
403
|
+
const GPUExternalTextureDescriptor &descriptor, const std::string &label,
|
|
404
|
+
const char *caller) {
|
|
405
|
+
wgpu::ExternalTextureDescriptor extDesc{};
|
|
406
|
+
if (!label.empty()) {
|
|
407
|
+
extDesc.label = wgpu::StringView(label.c_str(), label.size());
|
|
408
|
+
}
|
|
409
|
+
dawn::ExternalTextureColorSpaceParams params =
|
|
410
|
+
dawn::GetNoopColorSpaceParams();
|
|
411
|
+
// The plane views outlive the synchronous CreateExternalTexture() below.
|
|
412
|
+
wgpu::TextureView plane0;
|
|
413
|
+
wgpu::TextureView plane1;
|
|
414
|
+
#if defined(__APPLE__)
|
|
415
|
+
(void)memory;
|
|
416
|
+
// NV12 needs plane0 = R8 luma and plane1 = RG8 chroma; BGRA only plane0.
|
|
417
|
+
// Dawn converts the YUV planes to RGB in the destination color space while
|
|
418
|
+
// sampling, with the matrix, range, transfer function and gamut of the
|
|
419
|
+
// frame's color space (from the CVPixelBuffer attachments; BT.709 limited
|
|
420
|
+
// range, the norm for HD video, where they are missing). A BGRA frame is
|
|
421
|
+
// RGB already: no-op parameters.
|
|
422
|
+
bool isYuv = frame.pixelFormat == VideoPixelFormat::NV12;
|
|
423
|
+
{
|
|
424
|
+
wgpu::TextureViewDescriptor v{};
|
|
425
|
+
v.aspect =
|
|
426
|
+
isYuv ? wgpu::TextureAspect::Plane0Only : wgpu::TextureAspect::All;
|
|
427
|
+
plane0 = texture.CreateView(&v);
|
|
428
|
+
}
|
|
429
|
+
if (isYuv) {
|
|
430
|
+
wgpu::TextureViewDescriptor v{};
|
|
431
|
+
v.aspect = wgpu::TextureAspect::Plane1Only;
|
|
432
|
+
plane1 = texture.CreateView(&v);
|
|
433
|
+
params = colorSpaceParams(
|
|
434
|
+
resolveColorSpace(frame.colorSpace, descriptor.sourceColorSpace,
|
|
435
|
+
wgpu::ColorSpaceYCbCrMatrixDawn::Rec709,
|
|
436
|
+
wgpu::ColorSpaceYCbCrRangeDawn::Narrow),
|
|
437
|
+
descriptor.colorSpace, caller);
|
|
438
|
+
}
|
|
439
|
+
extDesc.plane0 = plane0;
|
|
440
|
+
if (isYuv) {
|
|
441
|
+
extDesc.plane1 = plane1;
|
|
442
|
+
}
|
|
443
|
+
#elif defined(__ANDROID__)
|
|
444
|
+
// Dawn imports every YUV AHardwareBuffer as an OpaqueYCbCrAndroid texture
|
|
445
|
+
// and samples it through a Vulkan YCbCr sampler whose own conversion is the
|
|
446
|
+
// identity (SamplerVk.cpp), so the sample is raw Y, Cb, Cr; the external
|
|
447
|
+
// texture then applies the descriptor's YCbCr matrix, transfer functions and
|
|
448
|
+
// gamut conversion while sampling (Dawn's YUV AHB specialization), the same
|
|
449
|
+
// way it does for the biplanar textures of the other backends. The frame's
|
|
450
|
+
// color space comes from its producer (Image.getDataSpace on API 33, the
|
|
451
|
+
// container's color info before that); what it leaves unknown falls back to
|
|
452
|
+
// the YCbCr model and range the driver suggests for the buffer, read from
|
|
453
|
+
// the shared memory's properties, and to BT.601 limited range, the camera
|
|
454
|
+
// norm, when the driver has no opinion. A color-format AHB (RGBA8, R8, ...)
|
|
455
|
+
// samples as RGB directly: no-op parameters.
|
|
456
|
+
if (texture.GetFormat() == wgpu::TextureFormat::OpaqueYCbCrAndroid) {
|
|
457
|
+
auto fallbackMatrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec601;
|
|
458
|
+
auto fallbackRange = wgpu::ColorSpaceYCbCrRangeDawn::Narrow;
|
|
459
|
+
wgpu::SharedTextureMemoryAHardwareBufferProperties ahbProps{};
|
|
460
|
+
wgpu::SharedTextureMemoryProperties props{};
|
|
461
|
+
props.nextInChain = &ahbProps;
|
|
462
|
+
if (memory.GetProperties(&props) == wgpu::Status::Success) {
|
|
463
|
+
// VkSamplerYcbcrModelConversion / VkSamplerYcbcrRange values, inlined
|
|
464
|
+
// to keep this cross-platform file free of Vulkan headers.
|
|
465
|
+
switch (ahbProps.yCbCrInfo.vkYCbCrModel) {
|
|
466
|
+
case 0: // VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY: RGB data
|
|
467
|
+
fallbackMatrix = wgpu::ColorSpaceYCbCrMatrixDawn::Identity;
|
|
468
|
+
fallbackRange = wgpu::ColorSpaceYCbCrRangeDawn::Identity;
|
|
469
|
+
break;
|
|
470
|
+
case 2: // VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709
|
|
471
|
+
fallbackMatrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec709;
|
|
472
|
+
break;
|
|
473
|
+
case 4: // VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_2020
|
|
474
|
+
fallbackMatrix = wgpu::ColorSpaceYCbCrMatrixDawn::Rec2020;
|
|
475
|
+
break;
|
|
476
|
+
default: // YCBCR_601, YCBCR_IDENTITY and unknown models
|
|
477
|
+
break;
|
|
478
|
+
}
|
|
479
|
+
if (fallbackMatrix != wgpu::ColorSpaceYCbCrMatrixDawn::Identity) {
|
|
480
|
+
fallbackRange = ahbProps.yCbCrInfo.vkYCbCrRange == 0
|
|
481
|
+
? wgpu::ColorSpaceYCbCrRangeDawn::Full
|
|
482
|
+
: wgpu::ColorSpaceYCbCrRangeDawn::Narrow;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
params = colorSpaceParams(resolveColorSpace(frame.colorSpace,
|
|
486
|
+
descriptor.sourceColorSpace,
|
|
487
|
+
fallbackMatrix, fallbackRange),
|
|
488
|
+
descriptor.colorSpace, caller);
|
|
489
|
+
}
|
|
490
|
+
plane0 = texture.CreateView();
|
|
491
|
+
extDesc.plane0 = plane0;
|
|
492
|
+
#else
|
|
493
|
+
(void)memory;
|
|
494
|
+
(void)frame;
|
|
495
|
+
throw std::runtime_error(prefix(caller) + "unsupported platform");
|
|
496
|
+
#endif
|
|
497
|
+
applyColorSpaceParams(extDesc, params);
|
|
498
|
+
|
|
499
|
+
// The crop, the whole frame by default (the size of the texture: the frame
|
|
500
|
+
// reports it too, but a buffer wrapped from another library may not). It
|
|
501
|
+
// must lie within the frame and not be empty.
|
|
502
|
+
uint32_t width = texture.GetWidth();
|
|
503
|
+
uint32_t height = texture.GetHeight();
|
|
504
|
+
uint32_t cropX = 0;
|
|
505
|
+
uint32_t cropY = 0;
|
|
506
|
+
if (descriptor.cropOrigin.has_value() && *descriptor.cropOrigin) {
|
|
507
|
+
cropX = cropCoordinate((*descriptor.cropOrigin)->x, width, "cropOrigin.x",
|
|
508
|
+
caller);
|
|
509
|
+
cropY = cropCoordinate((*descriptor.cropOrigin)->y, height, "cropOrigin.y",
|
|
510
|
+
caller);
|
|
511
|
+
}
|
|
512
|
+
if (cropX >= width || cropY >= height) {
|
|
513
|
+
throw std::runtime_error(
|
|
514
|
+
prefix(caller) + "cropOrigin (" + std::to_string(cropX) + ", " +
|
|
515
|
+
std::to_string(cropY) + ") is outside the " + std::to_string(width) +
|
|
516
|
+
"x" + std::to_string(height) + " frame");
|
|
517
|
+
}
|
|
518
|
+
uint32_t cropWidth = width - cropX;
|
|
519
|
+
uint32_t cropHeight = height - cropY;
|
|
520
|
+
if (descriptor.cropSize.has_value() && *descriptor.cropSize) {
|
|
521
|
+
cropWidth = cropCoordinate((*descriptor.cropSize)->width, cropWidth,
|
|
522
|
+
"cropSize.width", caller);
|
|
523
|
+
cropHeight = cropCoordinate((*descriptor.cropSize)->height, cropHeight,
|
|
524
|
+
"cropSize.height", caller);
|
|
525
|
+
if (cropWidth == 0 || cropHeight == 0) {
|
|
526
|
+
throw std::runtime_error(prefix(caller) + "cropSize is empty");
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
extDesc.cropOrigin = {cropX, cropY};
|
|
530
|
+
extDesc.cropSize = {cropWidth, cropHeight};
|
|
531
|
+
extDesc.apparentSize = {cropWidth, cropHeight};
|
|
532
|
+
extDesc.mirrored = descriptor.mirrored.value_or(false);
|
|
533
|
+
extDesc.rotation = toExternalTextureRotation(descriptor.rotation.value_or(0));
|
|
534
|
+
|
|
535
|
+
auto external = device.CreateExternalTexture(&extDesc);
|
|
536
|
+
if (external == nullptr) {
|
|
537
|
+
throw std::runtime_error(prefix(caller) +
|
|
538
|
+
"CreateExternalTexture returned null");
|
|
539
|
+
}
|
|
540
|
+
return external;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
bool mergeFences(const wgpu::Device &device,
|
|
544
|
+
const wgpu::SharedTextureMemoryEndAccessState &state,
|
|
545
|
+
wgpu::SharedFence *fence, uint64_t *signaledValue,
|
|
546
|
+
const char *caller) {
|
|
547
|
+
if (state.fenceCount == 0) {
|
|
548
|
+
return false;
|
|
549
|
+
}
|
|
550
|
+
if (state.fenceCount == 1) {
|
|
551
|
+
*fence = state.fences[0];
|
|
552
|
+
*signaledValue = state.signaledValues[0];
|
|
553
|
+
return true;
|
|
554
|
+
}
|
|
555
|
+
#if defined(__ANDROID__)
|
|
556
|
+
// Dawn's ExportInfo lends the fd of each fence (the fence closes it): merge
|
|
557
|
+
// duplicates into one sync fd, then import that as the fence to return.
|
|
558
|
+
// Dawn duplicates the fd on import, so the merged one is closed here.
|
|
559
|
+
UniqueFd merged;
|
|
560
|
+
for (size_t i = 0; i < state.fenceCount; i++) {
|
|
561
|
+
wgpu::SharedFence source = state.fences[i];
|
|
562
|
+
wgpu::SharedFenceExportInfo info{};
|
|
563
|
+
wgpu::SharedFenceSyncFDExportInfo fdInfo{};
|
|
564
|
+
info.nextInChain = &fdInfo;
|
|
565
|
+
source.ExportInfo(&info);
|
|
566
|
+
if (info.type != wgpu::SharedFenceType::SyncFD) {
|
|
567
|
+
// Dropping it would hand back a fence that signals before all the GPU
|
|
568
|
+
// work of the access is done, and the producer would reuse the buffer
|
|
569
|
+
// too early.
|
|
570
|
+
throw std::runtime_error(prefix(caller) +
|
|
571
|
+
"a fence of the access is not a sync fd and "
|
|
572
|
+
"cannot be merged with the others");
|
|
573
|
+
}
|
|
574
|
+
if (fdInfo.handle < 0) {
|
|
575
|
+
// A sync fd of -1 is a fence that has signaled already.
|
|
576
|
+
continue;
|
|
577
|
+
}
|
|
578
|
+
if (merged.fd < 0) {
|
|
579
|
+
merged = UniqueFd(::fcntl(fdInfo.handle, F_DUPFD_CLOEXEC, 0));
|
|
580
|
+
} else {
|
|
581
|
+
UniqueFd next(sync_merge("rnwgpu-end-access", merged.fd, fdInfo.handle));
|
|
582
|
+
merged = std::move(next);
|
|
583
|
+
}
|
|
584
|
+
if (merged.fd < 0) {
|
|
585
|
+
throw std::runtime_error(prefix(caller) +
|
|
586
|
+
"merging the fences of the access failed");
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
if (merged.fd < 0) {
|
|
590
|
+
return false;
|
|
591
|
+
}
|
|
592
|
+
wgpu::SharedFenceSyncFDDescriptor fdDesc{};
|
|
593
|
+
fdDesc.handle = merged.fd;
|
|
594
|
+
wgpu::SharedFenceDescriptor fenceDesc{};
|
|
595
|
+
fenceDesc.nextInChain = &fdDesc;
|
|
596
|
+
auto imported = device.ImportSharedFence(&fenceDesc);
|
|
597
|
+
if (imported == nullptr) {
|
|
598
|
+
throw std::runtime_error(prefix(caller) +
|
|
599
|
+
"importing the merged fence of the access failed");
|
|
600
|
+
}
|
|
601
|
+
*fence = std::move(imported);
|
|
602
|
+
*signaledValue = 1;
|
|
603
|
+
return true;
|
|
604
|
+
#else
|
|
605
|
+
// Metal: the fences of one shared event are covered by its largest value.
|
|
606
|
+
// Dawn's fences are all the device's event, but a fence the caller passed to
|
|
607
|
+
// beginAccess() comes back unconsumed when the access did not use the
|
|
608
|
+
// texture, and it may be on another event: two events can't be merged into
|
|
609
|
+
// one fence, and dropping either would let the producer reuse the buffer
|
|
610
|
+
// too early.
|
|
611
|
+
(void)device;
|
|
612
|
+
void *event = nullptr;
|
|
613
|
+
size_t last = 0;
|
|
614
|
+
for (size_t i = 0; i < state.fenceCount; i++) {
|
|
615
|
+
wgpu::SharedFence source = state.fences[i];
|
|
616
|
+
wgpu::SharedFenceExportInfo info{};
|
|
617
|
+
wgpu::SharedFenceMTLSharedEventExportInfo eventInfo{};
|
|
618
|
+
info.nextInChain = &eventInfo;
|
|
619
|
+
source.ExportInfo(&info);
|
|
620
|
+
if (info.type != wgpu::SharedFenceType::MTLSharedEvent ||
|
|
621
|
+
(i > 0 && eventInfo.sharedEvent != event)) {
|
|
622
|
+
throw std::runtime_error(prefix(caller) +
|
|
623
|
+
"the fences of the access are on different "
|
|
624
|
+
"shared events and cannot be merged into one");
|
|
625
|
+
}
|
|
626
|
+
event = eventInfo.sharedEvent;
|
|
627
|
+
if (state.signaledValues[i] > state.signaledValues[last]) {
|
|
628
|
+
last = i;
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
*fence = state.fences[last];
|
|
632
|
+
*signaledValue = state.signaledValues[last];
|
|
633
|
+
return true;
|
|
634
|
+
#endif
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
} // namespace native_frame
|
|
638
|
+
} // namespace rnwgpu
|