react-native-webgpu 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/cpp/AndroidPlatformContext.h +42 -8
- package/android/cpp/AndroidVideoPlayer.h +307 -0
- package/android/src/main/java/com/webgpu/WebGPUModule.java +10 -0
- package/android/src/main/java/com/webgpu/WebGPUVideoPlayer.java +420 -0
- package/apple/AppleVideoPlayer.mm +157 -47
- package/cpp/rnwgpu/PlatformContext.h +25 -5
- package/cpp/rnwgpu/api/GPUDevice.cpp +2 -1
- package/cpp/rnwgpu/api/GPUDevice.h +6 -0
- package/cpp/rnwgpu/api/GPUQueue.cpp +80 -10
- package/cpp/rnwgpu/api/GPUQueue.h +15 -4
- package/cpp/rnwgpu/api/VideoFrameBlit.h +355 -0
- package/cpp/rnwgpu/api/VideoPlayer.h +50 -3
- package/cpp/rnwgpu/api/descriptors/GPUImageCopyExternalImage.h +30 -3
- package/lib/commonjs/importDevice.js +2 -2
- package/lib/commonjs/index.js.map +1 -1
- package/lib/commonjs/main/index.js +12 -0
- package/lib/commonjs/main/index.js.map +1 -1
- package/lib/commonjs/main/index.web.js +11 -0
- package/lib/commonjs/main/index.web.js.map +1 -1
- package/lib/commonjs/video.js +58 -0
- package/lib/commonjs/video.js.map +1 -0
- package/lib/module/importDevice.js +2 -2
- package/lib/module/index.js.map +1 -1
- package/lib/module/main/index.js +1 -0
- package/lib/module/main/index.js.map +1 -1
- package/lib/module/main/index.web.js +1 -0
- package/lib/module/main/index.web.js.map +1 -1
- package/lib/module/video.js +49 -0
- package/lib/module/video.js.map +1 -0
- package/lib/typescript/lib/commonjs/video.d.ts +30 -0
- package/lib/typescript/lib/commonjs/video.d.ts.map +1 -0
- package/lib/typescript/lib/module/main/index.d.ts +1 -0
- package/lib/typescript/lib/module/main/index.web.d.ts +1 -0
- package/lib/typescript/lib/module/video.d.ts +4 -0
- package/lib/typescript/lib/module/video.d.ts.map +1 -0
- package/lib/typescript/src/importDevice.d.ts +2 -2
- package/lib/typescript/src/index.d.ts +5 -3
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/main/index.d.ts +1 -0
- package/lib/typescript/src/main/index.d.ts.map +1 -1
- package/lib/typescript/src/main/index.web.d.ts +1 -0
- package/lib/typescript/src/main/index.web.d.ts.map +1 -1
- package/lib/typescript/src/types.d.ts +19 -0
- package/lib/typescript/src/types.d.ts.map +1 -1
- package/lib/typescript/src/video.d.ts +41 -0
- package/lib/typescript/src/video.d.ts.map +1 -0
- package/package.json +1 -1
- package/src/importDevice.ts +2 -2
- package/src/index.tsx +28 -3
- package/src/main/index.tsx +1 -0
- package/src/main/index.web.tsx +1 -0
- package/src/types.ts +60 -7
- package/src/video.ts +69 -0
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
package com.webgpu;
|
|
2
|
+
|
|
3
|
+
import android.content.Context;
|
|
4
|
+
import android.graphics.ImageFormat;
|
|
5
|
+
import android.hardware.HardwareBuffer;
|
|
6
|
+
import android.media.AudioAttributes;
|
|
7
|
+
import android.media.Image;
|
|
8
|
+
import android.media.ImageReader;
|
|
9
|
+
import android.media.MediaExtractor;
|
|
10
|
+
import android.media.MediaFormat;
|
|
11
|
+
import android.media.MediaMetadataRetriever;
|
|
12
|
+
import android.media.MediaPlayer;
|
|
13
|
+
import android.net.Uri;
|
|
14
|
+
import android.os.Build;
|
|
15
|
+
import android.util.Log;
|
|
16
|
+
|
|
17
|
+
import com.facebook.proguard.annotations.DoNotStrip;
|
|
18
|
+
|
|
19
|
+
import java.io.File;
|
|
20
|
+
import java.io.IOException;
|
|
21
|
+
import java.util.HashSet;
|
|
22
|
+
import java.util.Set;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Decodes a video with {@link MediaPlayer} into an {@link ImageReader} whose
|
|
26
|
+
* buffers the GPU can sample ({@link ImageFormat#PRIVATE} with
|
|
27
|
+
* {@link HardwareBuffer#USAGE_GPU_SAMPLED_IMAGE}). Each frame is handed to C++
|
|
28
|
+
* (AndroidVideoPlayer) as an {@link Image} that stays acquired until the
|
|
29
|
+
* NativeVideoFrame is released, which hands the buffer back to the decoder.
|
|
30
|
+
*
|
|
31
|
+
* The frames keep the decoder's native YUV layout, so they are sampled through
|
|
32
|
+
* GPUDevice.importExternalTexture. MediaPlayer owns the audio, the clock and
|
|
33
|
+
* the A/V synchronization.
|
|
34
|
+
*
|
|
35
|
+
* MediaPlayer is driven from the caller's thread (its methods are thread
|
|
36
|
+
* safe); its callbacks arrive on the main looper, so the shared state is
|
|
37
|
+
* guarded by {@code lock}.
|
|
38
|
+
*/
|
|
39
|
+
@DoNotStrip
|
|
40
|
+
public class WebGPUVideoPlayer {
|
|
41
|
+
private static final String TAG = "WebGPUVideoPlayer";
|
|
42
|
+
|
|
43
|
+
// Frames the consumer can hold at once: the one being drawn, the next one,
|
|
44
|
+
// and a spare. Holding more makes copyLatestFrame() return null until a
|
|
45
|
+
// frame is released.
|
|
46
|
+
private static final int MAX_IMAGES = 3;
|
|
47
|
+
|
|
48
|
+
private final Object lock = new Object();
|
|
49
|
+
private final Set<Image> acquired = new HashSet<>();
|
|
50
|
+
private MediaPlayer player;
|
|
51
|
+
private ImageReader reader;
|
|
52
|
+
private boolean prepared = false;
|
|
53
|
+
private boolean playing = false;
|
|
54
|
+
private boolean loop = true;
|
|
55
|
+
private float volume = 1f;
|
|
56
|
+
private double pendingSeekSeconds = -1;
|
|
57
|
+
private boolean released = false;
|
|
58
|
+
|
|
59
|
+
private volatile int width = 0;
|
|
60
|
+
private volatile int height = 0;
|
|
61
|
+
private volatile int rotation = 0;
|
|
62
|
+
private volatile double frameRate = 0;
|
|
63
|
+
private volatile double durationSeconds = 0;
|
|
64
|
+
|
|
65
|
+
WebGPUVideoPlayer(Context context, String source) {
|
|
66
|
+
Uri uri = parseSource(source);
|
|
67
|
+
readMetadata(context, uri);
|
|
68
|
+
|
|
69
|
+
MediaPlayer mp = new MediaPlayer();
|
|
70
|
+
mp.setAudioAttributes(
|
|
71
|
+
new AudioAttributes.Builder()
|
|
72
|
+
.setUsage(AudioAttributes.USAGE_MEDIA)
|
|
73
|
+
.setContentType(AudioAttributes.CONTENT_TYPE_MOVIE)
|
|
74
|
+
.build());
|
|
75
|
+
mp.setOnVideoSizeChangedListener((p, w, h) -> onVideoSizeChanged(w, h));
|
|
76
|
+
mp.setOnPreparedListener(this::onPrepared);
|
|
77
|
+
mp.setOnCompletionListener(
|
|
78
|
+
p -> {
|
|
79
|
+
// Not called while looping: MediaPlayer restarts on its own.
|
|
80
|
+
synchronized (lock) {
|
|
81
|
+
if (!loop) {
|
|
82
|
+
playing = false;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
mp.setOnErrorListener(
|
|
87
|
+
(p, what, extra) -> {
|
|
88
|
+
Log.e(TAG, "MediaPlayer error " + what + " (" + extra + ") for " + source);
|
|
89
|
+
synchronized (lock) {
|
|
90
|
+
playing = false;
|
|
91
|
+
}
|
|
92
|
+
return true;
|
|
93
|
+
});
|
|
94
|
+
try {
|
|
95
|
+
mp.setDataSource(context, uri);
|
|
96
|
+
} catch (IOException | IllegalArgumentException | SecurityException e) {
|
|
97
|
+
mp.release();
|
|
98
|
+
throw new RuntimeException("createVideoPlayer: cannot open " + source, e);
|
|
99
|
+
}
|
|
100
|
+
mp.setLooping(loop);
|
|
101
|
+
mp.prepareAsync();
|
|
102
|
+
synchronized (lock) {
|
|
103
|
+
player = mp;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// A URL (http, https, file, content) or a plain file path.
|
|
108
|
+
private static Uri parseSource(String source) {
|
|
109
|
+
Uri uri = Uri.parse(source);
|
|
110
|
+
if (uri.getScheme() == null) {
|
|
111
|
+
return Uri.fromFile(new File(source));
|
|
112
|
+
}
|
|
113
|
+
return uri;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// The frame geometry comes from the container (MediaMetadataRetriever for
|
|
117
|
+
// the size, rotation and duration, MediaExtractor for the frame rate). The
|
|
118
|
+
// metadata of a remote video is fetched off the calling thread; the getters
|
|
119
|
+
// report 0 until it is known.
|
|
120
|
+
private void readMetadata(Context context, Uri uri) {
|
|
121
|
+
Thread thread =
|
|
122
|
+
new Thread(
|
|
123
|
+
() -> {
|
|
124
|
+
MediaMetadataRetriever retriever = new MediaMetadataRetriever();
|
|
125
|
+
try {
|
|
126
|
+
String scheme = uri.getScheme();
|
|
127
|
+
if ("http".equals(scheme) || "https".equals(scheme)) {
|
|
128
|
+
retriever.setDataSource(uri.toString(), new java.util.HashMap<>());
|
|
129
|
+
} else {
|
|
130
|
+
retriever.setDataSource(context, uri);
|
|
131
|
+
}
|
|
132
|
+
int w = parseInt(retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH));
|
|
133
|
+
int h = parseInt(retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT));
|
|
134
|
+
if (w > 0 && h > 0 && width == 0) {
|
|
135
|
+
width = w;
|
|
136
|
+
height = h;
|
|
137
|
+
}
|
|
138
|
+
rotation = parseInt(retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION));
|
|
139
|
+
long durationMs = parseInt(retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION));
|
|
140
|
+
if (durationMs > 0 && durationSeconds == 0) {
|
|
141
|
+
durationSeconds = durationMs / 1000.0;
|
|
142
|
+
}
|
|
143
|
+
} catch (RuntimeException e) {
|
|
144
|
+
Log.w(TAG, "Could not read the metadata of " + uri + ": " + e.getMessage());
|
|
145
|
+
} finally {
|
|
146
|
+
try {
|
|
147
|
+
retriever.release();
|
|
148
|
+
} catch (IOException ignored) {
|
|
149
|
+
// Nothing to recover.
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
MediaExtractor extractor = new MediaExtractor();
|
|
154
|
+
try {
|
|
155
|
+
extractor.setDataSource(context, uri, null);
|
|
156
|
+
for (int i = 0; i < extractor.getTrackCount(); i++) {
|
|
157
|
+
MediaFormat format = extractor.getTrackFormat(i);
|
|
158
|
+
String mime = format.getString(MediaFormat.KEY_MIME);
|
|
159
|
+
if (mime != null && mime.startsWith("video/")) {
|
|
160
|
+
if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
|
|
161
|
+
frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
|
|
162
|
+
}
|
|
163
|
+
break;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
} catch (IOException | RuntimeException e) {
|
|
167
|
+
Log.w(TAG, "Could not read the frame rate of " + uri + ": " + e.getMessage());
|
|
168
|
+
} finally {
|
|
169
|
+
extractor.release();
|
|
170
|
+
}
|
|
171
|
+
},
|
|
172
|
+
"WebGPUVideoPlayer metadata");
|
|
173
|
+
thread.setDaemon(true);
|
|
174
|
+
thread.start();
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
private static int parseInt(String value) {
|
|
178
|
+
if (value == null) {
|
|
179
|
+
return 0;
|
|
180
|
+
}
|
|
181
|
+
try {
|
|
182
|
+
return Integer.parseInt(value);
|
|
183
|
+
} catch (NumberFormatException e) {
|
|
184
|
+
return 0;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// The decoder reports the frame size (possibly more than once for adaptive
|
|
189
|
+
// streams); the ImageReader that receives the frames is sized accordingly.
|
|
190
|
+
private void onVideoSizeChanged(int w, int h) {
|
|
191
|
+
if (w <= 0 || h <= 0) {
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
ImageReader previous;
|
|
195
|
+
synchronized (lock) {
|
|
196
|
+
if (released || player == null) {
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
if (reader != null && reader.getWidth() == w && reader.getHeight() == h) {
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
if (width == 0) {
|
|
203
|
+
width = w;
|
|
204
|
+
height = h;
|
|
205
|
+
}
|
|
206
|
+
previous = reader;
|
|
207
|
+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
|
208
|
+
reader = ImageReader.newInstance(w, h, ImageFormat.PRIVATE, MAX_IMAGES, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
|
|
209
|
+
} else {
|
|
210
|
+
reader = ImageReader.newInstance(w, h, ImageFormat.PRIVATE, MAX_IMAGES);
|
|
211
|
+
}
|
|
212
|
+
player.setSurface(reader.getSurface());
|
|
213
|
+
}
|
|
214
|
+
if (previous != null) {
|
|
215
|
+
// Frames still held by the consumer were acquired from the old reader;
|
|
216
|
+
// closing it closes them, and the native side keeps the buffers alive.
|
|
217
|
+
synchronized (lock) {
|
|
218
|
+
acquired.clear();
|
|
219
|
+
}
|
|
220
|
+
previous.close();
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
private void onPrepared(MediaPlayer mp) {
|
|
225
|
+
synchronized (lock) {
|
|
226
|
+
if (released) {
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
prepared = true;
|
|
230
|
+
int durationMs = mp.getDuration();
|
|
231
|
+
if (durationMs > 0) {
|
|
232
|
+
durationSeconds = durationMs / 1000.0;
|
|
233
|
+
}
|
|
234
|
+
mp.setVolume(volume, volume);
|
|
235
|
+
mp.setLooping(loop);
|
|
236
|
+
if (pendingSeekSeconds >= 0) {
|
|
237
|
+
mp.seekTo((long) (pendingSeekSeconds * 1000), MediaPlayer.SEEK_CLOSEST);
|
|
238
|
+
pendingSeekSeconds = -1;
|
|
239
|
+
}
|
|
240
|
+
if (playing) {
|
|
241
|
+
mp.start();
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* The most recently decoded frame, or null when no new frame was decoded
|
|
248
|
+
* since the last call (or when the consumer holds {@link #MAX_IMAGES}
|
|
249
|
+
* frames). Close it with {@link #closeImage} once the frame is released.
|
|
250
|
+
*/
|
|
251
|
+
@DoNotStrip
|
|
252
|
+
public Image copyLatestFrame() {
|
|
253
|
+
ImageReader current;
|
|
254
|
+
synchronized (lock) {
|
|
255
|
+
if (released || reader == null) {
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
current = reader;
|
|
259
|
+
}
|
|
260
|
+
Image image;
|
|
261
|
+
try {
|
|
262
|
+
image = current.acquireLatestImage();
|
|
263
|
+
} catch (IllegalStateException e) {
|
|
264
|
+
Log.w(TAG, "copyLatestFrame: every frame is held, release one to get a new frame");
|
|
265
|
+
return null;
|
|
266
|
+
}
|
|
267
|
+
if (image == null) {
|
|
268
|
+
return null;
|
|
269
|
+
}
|
|
270
|
+
synchronized (lock) {
|
|
271
|
+
acquired.add(image);
|
|
272
|
+
}
|
|
273
|
+
return image;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** Hands the buffer of a frame returned by {@link #copyLatestFrame} back to the decoder. */
|
|
277
|
+
@DoNotStrip
|
|
278
|
+
public void closeImage(Image image) {
|
|
279
|
+
synchronized (lock) {
|
|
280
|
+
if (!acquired.remove(image)) {
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
image.close();
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
@DoNotStrip
|
|
288
|
+
public void play() {
|
|
289
|
+
synchronized (lock) {
|
|
290
|
+
playing = true;
|
|
291
|
+
if (player != null && prepared) {
|
|
292
|
+
player.start();
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
@DoNotStrip
|
|
298
|
+
public void pause() {
|
|
299
|
+
synchronized (lock) {
|
|
300
|
+
playing = false;
|
|
301
|
+
if (player != null && prepared && player.isPlaying()) {
|
|
302
|
+
player.pause();
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
@DoNotStrip
|
|
308
|
+
public boolean getPaused() {
|
|
309
|
+
synchronized (lock) {
|
|
310
|
+
return !playing;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
@DoNotStrip
|
|
315
|
+
public double getCurrentTime() {
|
|
316
|
+
synchronized (lock) {
|
|
317
|
+
if (player != null && prepared) {
|
|
318
|
+
return player.getCurrentPosition() / 1000.0;
|
|
319
|
+
}
|
|
320
|
+
return pendingSeekSeconds >= 0 ? pendingSeekSeconds : 0;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// An exact seek, also while paused: the decoder then renders the frame at
|
|
325
|
+
// the new position, which the next copyLatestFrame() returns.
|
|
326
|
+
@DoNotStrip
|
|
327
|
+
public void seek(double seconds) {
|
|
328
|
+
synchronized (lock) {
|
|
329
|
+
if (player != null && prepared) {
|
|
330
|
+
player.seekTo((long) (Math.max(0, seconds) * 1000), MediaPlayer.SEEK_CLOSEST);
|
|
331
|
+
} else {
|
|
332
|
+
pendingSeekSeconds = Math.max(0, seconds);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
@DoNotStrip
|
|
338
|
+
public double getDuration() {
|
|
339
|
+
return durationSeconds;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
@DoNotStrip
|
|
343
|
+
public double getVolume() {
|
|
344
|
+
synchronized (lock) {
|
|
345
|
+
return volume;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
@DoNotStrip
|
|
350
|
+
public void setVolume(double value) {
|
|
351
|
+
synchronized (lock) {
|
|
352
|
+
volume = (float) Math.max(0, Math.min(1, value));
|
|
353
|
+
if (player != null) {
|
|
354
|
+
player.setVolume(volume, volume);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
@DoNotStrip
|
|
360
|
+
public boolean getLoop() {
|
|
361
|
+
synchronized (lock) {
|
|
362
|
+
return loop;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
@DoNotStrip
|
|
367
|
+
public void setLoop(boolean value) {
|
|
368
|
+
synchronized (lock) {
|
|
369
|
+
loop = value;
|
|
370
|
+
if (player != null) {
|
|
371
|
+
player.setLooping(value);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
@DoNotStrip
|
|
377
|
+
public int getVideoWidth() {
|
|
378
|
+
return width;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
@DoNotStrip
|
|
382
|
+
public int getVideoHeight() {
|
|
383
|
+
return height;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
@DoNotStrip
|
|
387
|
+
public int getRotation() {
|
|
388
|
+
return rotation;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
@DoNotStrip
|
|
392
|
+
public double getFrameRate() {
|
|
393
|
+
return frameRate;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/** Stops playback and releases the decoder. Frames already copied stay valid until closed. */
|
|
397
|
+
@DoNotStrip
|
|
398
|
+
public void release() {
|
|
399
|
+
MediaPlayer mp;
|
|
400
|
+
ImageReader current;
|
|
401
|
+
synchronized (lock) {
|
|
402
|
+
if (released) {
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
released = true;
|
|
406
|
+
playing = false;
|
|
407
|
+
mp = player;
|
|
408
|
+
player = null;
|
|
409
|
+
current = reader;
|
|
410
|
+
reader = null;
|
|
411
|
+
acquired.clear();
|
|
412
|
+
}
|
|
413
|
+
if (mp != null) {
|
|
414
|
+
mp.release();
|
|
415
|
+
}
|
|
416
|
+
if (current != null) {
|
|
417
|
+
current.close();
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
#import <AVFoundation/AVFoundation.h>
|
|
4
4
|
#import <CoreVideo/CoreVideo.h>
|
|
5
5
|
|
|
6
|
+
#include <algorithm>
|
|
7
|
+
#include <atomic>
|
|
8
|
+
#include <memory>
|
|
6
9
|
#include <stdexcept>
|
|
7
10
|
|
|
8
11
|
namespace rnwgpu {
|
|
@@ -16,27 +19,27 @@ namespace {
|
|
|
16
19
|
// range) means luma is 16..235, chroma is 16..240 (8-bit).
|
|
17
20
|
// Reference: https://en.wikipedia.org/wiki/YCbCr (BT.601 / BT.709).
|
|
18
21
|
static constexpr float kBT709LimitedToRgb[12] = {
|
|
19
|
-
1.164383f, 0.000000f,
|
|
20
|
-
1.164383f, -0.213249f, -0.532909f, 0.301517f,
|
|
21
|
-
1.164383f, 2.112402f,
|
|
22
|
+
1.164383f, 0.000000f, 1.792741f, -0.972945f, //
|
|
23
|
+
1.164383f, -0.213249f, -0.532909f, 0.301517f, //
|
|
24
|
+
1.164383f, 2.112402f, 0.000000f, -1.133402f, //
|
|
22
25
|
};
|
|
23
26
|
static constexpr float kBT601LimitedToRgb[12] = {
|
|
24
|
-
1.164383f, 0.000000f,
|
|
25
|
-
1.164383f, -0.391762f, -0.812968f, 0.531668f,
|
|
26
|
-
1.164383f, 2.017232f,
|
|
27
|
+
1.164383f, 0.000000f, 1.596027f, -0.874202f, //
|
|
28
|
+
1.164383f, -0.391762f, -0.812968f, 0.531668f, //
|
|
29
|
+
1.164383f, 2.017232f, 0.000000f, -1.085631f, //
|
|
27
30
|
};
|
|
28
31
|
static constexpr float kBT2020LimitedToRgb[12] = {
|
|
29
|
-
1.164383f, 0.000000f,
|
|
30
|
-
1.164383f, -0.187326f, -0.650424f, 0.347459f,
|
|
31
|
-
1.164383f, 2.141772f,
|
|
32
|
+
1.164383f, 0.000000f, 1.678674f, -0.915688f, //
|
|
33
|
+
1.164383f, -0.187326f, -0.650424f, 0.347459f, //
|
|
34
|
+
1.164383f, 2.141772f, 0.000000f, -1.148145f, //
|
|
32
35
|
};
|
|
33
36
|
|
|
34
37
|
// Pick the right YUV→RGB matrix from the pixel buffer's color attachments.
|
|
35
38
|
// Falls back to BT.709 limited range (the right call for ≥720p H.264, which
|
|
36
39
|
// is what AVPlayer hands us for Big Buck Bunny and most streamed media).
|
|
37
40
|
static void fillYuvMatrix(CVPixelBufferRef pixelBuffer, float out[12]) {
|
|
38
|
-
CFTypeRef matrixKey =
|
|
39
|
-
pixelBuffer, kCVImageBufferYCbCrMatrixKey, nullptr);
|
|
41
|
+
CFTypeRef matrixKey =
|
|
42
|
+
CVBufferGetAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, nullptr);
|
|
40
43
|
const float *src = kBT709LimitedToRgb;
|
|
41
44
|
if (matrixKey) {
|
|
42
45
|
auto matrix = (CFStringRef)matrixKey;
|
|
@@ -53,28 +56,108 @@ static void fillYuvMatrix(CVPixelBufferRef pixelBuffer, float out[12]) {
|
|
|
53
56
|
}
|
|
54
57
|
|
|
55
58
|
// Map a CVPixelBuffer's pixel format to our VideoPixelFormat enum.
|
|
56
|
-
static VideoPixelFormat
|
|
57
|
-
|
|
59
|
+
static VideoPixelFormat
|
|
60
|
+
pixelFormatFromCVPixelBuffer(CVPixelBufferRef pixelBuffer) {
|
|
58
61
|
OSType type = CVPixelBufferGetPixelFormatType(pixelBuffer);
|
|
59
62
|
switch (type) {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
|
|
64
|
+
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
|
|
65
|
+
return VideoPixelFormat::NV12;
|
|
66
|
+
default:
|
|
67
|
+
return VideoPixelFormat::BGRA8;
|
|
65
68
|
}
|
|
66
69
|
}
|
|
67
70
|
|
|
71
|
+
// The clockwise rotation, in degrees, that a track's preferredTransform
|
|
72
|
+
// applies to its frames: a video recorded in portrait stores landscape frames
|
|
73
|
+
// with a 90 or 270 degree transform.
|
|
74
|
+
static int rotationFromTransform(CGAffineTransform t) {
|
|
75
|
+
if (t.a == 0 && t.b == 1 && t.c == -1 && t.d == 0) {
|
|
76
|
+
return 90;
|
|
77
|
+
}
|
|
78
|
+
if (t.a == 0 && t.b == -1 && t.c == 1 && t.d == 0) {
|
|
79
|
+
return 270;
|
|
80
|
+
}
|
|
81
|
+
if (t.a == -1 && t.b == 0 && t.c == 0 && t.d == -1) {
|
|
82
|
+
return 180;
|
|
83
|
+
}
|
|
84
|
+
return 0;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// The state the AVFoundation callbacks update. It is shared with the blocks
|
|
88
|
+
// observing the asset and the item, so a callback that fires after the player
|
|
89
|
+
// was destroyed only touches memory that outlives it.
|
|
90
|
+
struct AppleVideoPlayerState {
|
|
91
|
+
std::atomic<bool> playing{false};
|
|
92
|
+
std::atomic<bool> loop{true};
|
|
93
|
+
std::atomic<double> duration{0};
|
|
94
|
+
std::atomic<double> frameRate{0};
|
|
95
|
+
std::atomic<uint32_t> width{0};
|
|
96
|
+
std::atomic<uint32_t> height{0};
|
|
97
|
+
std::atomic<int> rotation{0};
|
|
98
|
+
};
|
|
99
|
+
|
|
68
100
|
class AppleVideoPlayer : public IVideoPlayer {
|
|
69
101
|
public:
|
|
70
|
-
AppleVideoPlayer(AVPlayer *player,
|
|
71
|
-
|
|
72
|
-
: _player(player), _output(output),
|
|
102
|
+
AppleVideoPlayer(AVPlayer *player, AVPlayerItem *item,
|
|
103
|
+
AVPlayerItemVideoOutput *output)
|
|
104
|
+
: _player(player), _output(output),
|
|
105
|
+
_state(std::make_shared<AppleVideoPlayerState>()) {
|
|
106
|
+
auto state = _state;
|
|
107
|
+
|
|
108
|
+
// The metadata loads asynchronously, so that a remote asset never blocks
|
|
109
|
+
// the calling thread; the getters report 0 until it has.
|
|
110
|
+
AVAsset *asset = item.asset;
|
|
111
|
+
#pragma clang diagnostic push
|
|
112
|
+
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
|
|
113
|
+
[asset
|
|
114
|
+
loadValuesAsynchronouslyForKeys:@[ @"duration", @"tracks" ]
|
|
115
|
+
completionHandler:^{
|
|
116
|
+
if ([asset statusOfValueForKey:@"duration" error:nil] ==
|
|
117
|
+
AVKeyValueStatusLoaded) {
|
|
118
|
+
CMTime duration = asset.duration;
|
|
119
|
+
if (CMTIME_IS_NUMERIC(duration)) {
|
|
120
|
+
state->duration = CMTimeGetSeconds(duration);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
if ([asset statusOfValueForKey:@"tracks" error:nil] ==
|
|
124
|
+
AVKeyValueStatusLoaded) {
|
|
125
|
+
AVAssetTrack *track =
|
|
126
|
+
[[asset tracksWithMediaType:AVMediaTypeVideo]
|
|
127
|
+
firstObject];
|
|
128
|
+
if (track) {
|
|
129
|
+
CGSize size = track.naturalSize;
|
|
130
|
+
state->width = static_cast<uint32_t>(size.width);
|
|
131
|
+
state->height = static_cast<uint32_t>(size.height);
|
|
132
|
+
state->rotation =
|
|
133
|
+
rotationFromTransform(track.preferredTransform);
|
|
134
|
+
state->frameRate = track.nominalFrameRate;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}];
|
|
138
|
+
#pragma clang diagnostic pop
|
|
139
|
+
|
|
140
|
+
// At the end of the stream, either loop or stop. The player keeps showing
|
|
141
|
+
// the last frame either way (actionAtItemEnd is None).
|
|
142
|
+
__weak AVPlayer *weakPlayer = player;
|
|
143
|
+
_endObserver = [[NSNotificationCenter defaultCenter]
|
|
144
|
+
addObserverForName:AVPlayerItemDidPlayToEndTimeNotification
|
|
145
|
+
object:item
|
|
146
|
+
queue:[NSOperationQueue mainQueue]
|
|
147
|
+
usingBlock:^(NSNotification * /*note*/) {
|
|
148
|
+
if (state->loop) {
|
|
149
|
+
[weakPlayer seekToTime:kCMTimeZero];
|
|
150
|
+
[weakPlayer play];
|
|
151
|
+
} else {
|
|
152
|
+
state->playing = false;
|
|
153
|
+
}
|
|
154
|
+
}];
|
|
155
|
+
}
|
|
73
156
|
|
|
74
157
|
~AppleVideoPlayer() override {
|
|
75
|
-
if (
|
|
76
|
-
[[NSNotificationCenter defaultCenter] removeObserver:
|
|
77
|
-
|
|
158
|
+
if (_endObserver) {
|
|
159
|
+
[[NSNotificationCenter defaultCenter] removeObserver:_endObserver];
|
|
160
|
+
_endObserver = nil;
|
|
78
161
|
}
|
|
79
162
|
[_player pause];
|
|
80
163
|
_player = nil;
|
|
@@ -105,13 +188,51 @@ public:
|
|
|
105
188
|
}
|
|
106
189
|
}
|
|
107
190
|
|
|
108
|
-
void play() override {
|
|
109
|
-
|
|
191
|
+
void play() override {
|
|
192
|
+
_state->playing = true;
|
|
193
|
+
[_player play];
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
void pause() override {
|
|
197
|
+
_state->playing = false;
|
|
198
|
+
[_player pause];
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
bool paused() override { return !_state->playing; }
|
|
202
|
+
|
|
203
|
+
double currentTime() override {
|
|
204
|
+
CMTime time = [_player currentTime];
|
|
205
|
+
return CMTIME_IS_NUMERIC(time) ? CMTimeGetSeconds(time) : 0;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// An exact seek: the output then holds the frame at that position, also
|
|
209
|
+
// while paused, so the next copyLatestFrame() returns it.
|
|
210
|
+
void seek(double seconds) override {
|
|
211
|
+
[_player seekToTime:CMTimeMakeWithSeconds(seconds, NSEC_PER_SEC)
|
|
212
|
+
toleranceBefore:kCMTimeZero
|
|
213
|
+
toleranceAfter:kCMTimeZero];
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
double duration() override { return _state->duration; }
|
|
217
|
+
|
|
218
|
+
double volume() override { return _player.volume; }
|
|
219
|
+
void setVolume(double volume) override {
|
|
220
|
+
_player.volume = static_cast<float>(std::clamp(volume, 0.0, 1.0));
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
bool loop() override { return _state->loop; }
|
|
224
|
+
void setLoop(bool loop) override { _state->loop = loop; }
|
|
225
|
+
|
|
226
|
+
uint32_t videoWidth() override { return _state->width; }
|
|
227
|
+
uint32_t videoHeight() override { return _state->height; }
|
|
228
|
+
int rotation() override { return _state->rotation; }
|
|
229
|
+
double frameRate() override { return _state->frameRate; }
|
|
110
230
|
|
|
111
231
|
private:
|
|
112
232
|
AVPlayer *_player;
|
|
113
233
|
AVPlayerItemVideoOutput *_output;
|
|
114
|
-
|
|
234
|
+
std::shared_ptr<AppleVideoPlayerState> _state;
|
|
235
|
+
id _endObserver = nil;
|
|
115
236
|
};
|
|
116
237
|
|
|
117
238
|
} // namespace
|
|
@@ -142,8 +263,8 @@ VideoFrameHandle wrapCVPixelBuffer(CVPixelBufferRef pixelBuffer) {
|
|
|
142
263
|
return handle;
|
|
143
264
|
}
|
|
144
265
|
|
|
145
|
-
std::unique_ptr<IVideoPlayer>
|
|
146
|
-
|
|
266
|
+
std::unique_ptr<IVideoPlayer> createAppleVideoPlayer(const std::string &path,
|
|
267
|
+
VideoPixelFormat format) {
|
|
147
268
|
NSString *nsPath = [NSString stringWithUTF8String:path.c_str()];
|
|
148
269
|
NSURL *url;
|
|
149
270
|
if ([nsPath hasPrefix:@"http://"] || [nsPath hasPrefix:@"https://"] ||
|
|
@@ -164,8 +285,8 @@ createAppleVideoPlayer(const std::string &path, VideoPixelFormat format) {
|
|
|
164
285
|
// importExternalTexture. BGRA is the "decode + convert" path for the
|
|
165
286
|
// single-plane SharedTextureMemory demo.
|
|
166
287
|
OSType pixelFormat = format == VideoPixelFormat::NV12
|
|
167
|
-
|
|
168
|
-
|
|
288
|
+
? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
|
|
289
|
+
: kCVPixelFormatType_32BGRA;
|
|
169
290
|
NSDictionary *outputSettings = @{
|
|
170
291
|
(NSString *)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
|
|
171
292
|
(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{},
|
|
@@ -176,20 +297,11 @@ createAppleVideoPlayer(const std::string &path, VideoPixelFormat format) {
|
|
|
176
297
|
[item addOutput:output];
|
|
177
298
|
|
|
178
299
|
AVPlayer *player = [AVPlayer playerWithPlayerItem:item];
|
|
300
|
+
// Keep the last frame at the end of the stream; AppleVideoPlayer decides
|
|
301
|
+
// whether to loop.
|
|
179
302
|
player.actionAtItemEnd = AVPlayerActionAtItemEndNone;
|
|
180
303
|
|
|
181
|
-
|
|
182
|
-
__weak AVPlayer *weakPlayer = player;
|
|
183
|
-
id loopObserver = [[NSNotificationCenter defaultCenter]
|
|
184
|
-
addObserverForName:AVPlayerItemDidPlayToEndTimeNotification
|
|
185
|
-
object:item
|
|
186
|
-
queue:[NSOperationQueue mainQueue]
|
|
187
|
-
usingBlock:^(NSNotification * /*note*/) {
|
|
188
|
-
[weakPlayer seekToTime:kCMTimeZero];
|
|
189
|
-
[weakPlayer play];
|
|
190
|
-
}];
|
|
191
|
-
|
|
192
|
-
return std::make_unique<AppleVideoPlayer>(player, output, loopObserver);
|
|
304
|
+
return std::make_unique<AppleVideoPlayer>(player, item, output);
|
|
193
305
|
}
|
|
194
306
|
|
|
195
307
|
std::string writeAppleTestVideoFile() {
|
|
@@ -231,8 +343,7 @@ std::string writeAppleTestVideoFile() {
|
|
|
231
343
|
writerInput.expectsMediaDataInRealTime = NO;
|
|
232
344
|
|
|
233
345
|
NSDictionary *bufferAttrs = @{
|
|
234
|
-
(NSString *)kCVPixelBufferPixelFormatTypeKey :
|
|
235
|
-
@(kCVPixelFormatType_32BGRA),
|
|
346
|
+
(NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA),
|
|
236
347
|
(NSString *)kCVPixelBufferWidthKey : @(kWidth),
|
|
237
348
|
(NSString *)kCVPixelBufferHeightKey : @(kHeight),
|
|
238
349
|
(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{},
|
|
@@ -274,8 +385,7 @@ std::string writeAppleTestVideoFile() {
|
|
|
274
385
|
for (int x = 0; x < kWidth; ++x) {
|
|
275
386
|
uint8_t r = static_cast<uint8_t>((x + phase) & 0xFF);
|
|
276
387
|
uint8_t g = static_cast<uint8_t>((y + phase * 2) & 0xFF);
|
|
277
|
-
uint8_t b =
|
|
278
|
-
static_cast<uint8_t>(((x + y + phase) & 0x40) ? 220 : 30);
|
|
388
|
+
uint8_t b = static_cast<uint8_t>(((x + y + phase) & 0x40) ? 220 : 30);
|
|
279
389
|
row[x * 4 + 0] = b;
|
|
280
390
|
row[x * 4 + 1] = g;
|
|
281
391
|
row[x * 4 + 2] = r;
|