react-native-webgpu 0.11.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/android/cpp/AndroidPlatformContext.h +42 -8
  2. package/android/cpp/AndroidVideoPlayer.h +307 -0
  3. package/android/src/main/java/com/webgpu/WebGPUModule.java +10 -0
  4. package/android/src/main/java/com/webgpu/WebGPUVideoPlayer.java +420 -0
  5. package/apple/AppleVideoPlayer.mm +157 -47
  6. package/cpp/rnwgpu/PlatformContext.h +25 -5
  7. package/cpp/rnwgpu/api/GPUDevice.cpp +2 -1
  8. package/cpp/rnwgpu/api/GPUDevice.h +6 -0
  9. package/cpp/rnwgpu/api/GPUQueue.cpp +80 -10
  10. package/cpp/rnwgpu/api/GPUQueue.h +15 -4
  11. package/cpp/rnwgpu/api/VideoFrameBlit.h +355 -0
  12. package/cpp/rnwgpu/api/VideoPlayer.h +50 -3
  13. package/cpp/rnwgpu/api/descriptors/GPUImageCopyExternalImage.h +30 -3
  14. package/lib/commonjs/importDevice.js +2 -2
  15. package/lib/commonjs/index.js.map +1 -1
  16. package/lib/commonjs/main/index.js +12 -0
  17. package/lib/commonjs/main/index.js.map +1 -1
  18. package/lib/commonjs/main/index.web.js +11 -0
  19. package/lib/commonjs/main/index.web.js.map +1 -1
  20. package/lib/commonjs/video.js +58 -0
  21. package/lib/commonjs/video.js.map +1 -0
  22. package/lib/module/importDevice.js +2 -2
  23. package/lib/module/index.js.map +1 -1
  24. package/lib/module/main/index.js +1 -0
  25. package/lib/module/main/index.js.map +1 -1
  26. package/lib/module/main/index.web.js +1 -0
  27. package/lib/module/main/index.web.js.map +1 -1
  28. package/lib/module/video.js +49 -0
  29. package/lib/module/video.js.map +1 -0
  30. package/lib/typescript/lib/commonjs/video.d.ts +30 -0
  31. package/lib/typescript/lib/commonjs/video.d.ts.map +1 -0
  32. package/lib/typescript/lib/module/main/index.d.ts +1 -0
  33. package/lib/typescript/lib/module/main/index.web.d.ts +1 -0
  34. package/lib/typescript/lib/module/video.d.ts +4 -0
  35. package/lib/typescript/lib/module/video.d.ts.map +1 -0
  36. package/lib/typescript/src/importDevice.d.ts +2 -2
  37. package/lib/typescript/src/index.d.ts +5 -3
  38. package/lib/typescript/src/index.d.ts.map +1 -1
  39. package/lib/typescript/src/main/index.d.ts +1 -0
  40. package/lib/typescript/src/main/index.d.ts.map +1 -1
  41. package/lib/typescript/src/main/index.web.d.ts +1 -0
  42. package/lib/typescript/src/main/index.web.d.ts.map +1 -1
  43. package/lib/typescript/src/types.d.ts +19 -0
  44. package/lib/typescript/src/types.d.ts.map +1 -1
  45. package/lib/typescript/src/video.d.ts +41 -0
  46. package/lib/typescript/src/video.d.ts.map +1 -0
  47. package/package.json +1 -1
  48. package/src/importDevice.ts +2 -2
  49. package/src/index.tsx +28 -3
  50. package/src/main/index.tsx +1 -0
  51. package/src/main/index.web.tsx +1 -0
  52. package/src/types.ts +60 -7
  53. package/src/video.ts +69 -0
@@ -0,0 +1,420 @@
1
+ package com.webgpu;
2
+
3
+ import android.content.Context;
4
+ import android.graphics.ImageFormat;
5
+ import android.hardware.HardwareBuffer;
6
+ import android.media.AudioAttributes;
7
+ import android.media.Image;
8
+ import android.media.ImageReader;
9
+ import android.media.MediaExtractor;
10
+ import android.media.MediaFormat;
11
+ import android.media.MediaMetadataRetriever;
12
+ import android.media.MediaPlayer;
13
+ import android.net.Uri;
14
+ import android.os.Build;
15
+ import android.util.Log;
16
+
17
+ import com.facebook.proguard.annotations.DoNotStrip;
18
+
19
+ import java.io.File;
20
+ import java.io.IOException;
21
+ import java.util.HashSet;
22
+ import java.util.Set;
23
+
24
+ /**
25
+ * Decodes a video with {@link MediaPlayer} into an {@link ImageReader} whose
26
+ * buffers the GPU can sample ({@link ImageFormat#PRIVATE} with
27
+ * {@link HardwareBuffer#USAGE_GPU_SAMPLED_IMAGE}). Each frame is handed to C++
28
+ * (AndroidVideoPlayer) as an {@link Image} that stays acquired until the
29
+ * NativeVideoFrame is released, which hands the buffer back to the decoder.
30
+ *
31
+ * The frames keep the decoder's native YUV layout, so they are sampled through
32
+ * GPUDevice.importExternalTexture. MediaPlayer owns the audio, the clock and
33
+ * the A/V synchronization.
34
+ *
35
+ * MediaPlayer is driven from the caller's thread (its methods are thread
36
+ * safe); its callbacks arrive on the main looper, so the shared state is
37
+ * guarded by {@code lock}.
38
+ */
39
+ @DoNotStrip
40
+ public class WebGPUVideoPlayer {
41
+ private static final String TAG = "WebGPUVideoPlayer";
42
+
43
+ // Frames the consumer can hold at once: the one being drawn, the next one,
44
+ // and a spare. Holding more makes copyLatestFrame() return null until a
45
+ // frame is released.
46
+ private static final int MAX_IMAGES = 3;
47
+
48
+ private final Object lock = new Object();
49
+ private final Set<Image> acquired = new HashSet<>();
50
+ private MediaPlayer player;
51
+ private ImageReader reader;
52
+ private boolean prepared = false;
53
+ private boolean playing = false;
54
+ private boolean loop = true;
55
+ private float volume = 1f;
56
+ private double pendingSeekSeconds = -1;
57
+ private boolean released = false;
58
+
59
+ private volatile int width = 0;
60
+ private volatile int height = 0;
61
+ private volatile int rotation = 0;
62
+ private volatile double frameRate = 0;
63
+ private volatile double durationSeconds = 0;
64
+
65
+ WebGPUVideoPlayer(Context context, String source) {
66
+ Uri uri = parseSource(source);
67
+ readMetadata(context, uri);
68
+
69
+ MediaPlayer mp = new MediaPlayer();
70
+ mp.setAudioAttributes(
71
+ new AudioAttributes.Builder()
72
+ .setUsage(AudioAttributes.USAGE_MEDIA)
73
+ .setContentType(AudioAttributes.CONTENT_TYPE_MOVIE)
74
+ .build());
75
+ mp.setOnVideoSizeChangedListener((p, w, h) -> onVideoSizeChanged(w, h));
76
+ mp.setOnPreparedListener(this::onPrepared);
77
+ mp.setOnCompletionListener(
78
+ p -> {
79
+ // Not called while looping: MediaPlayer restarts on its own.
80
+ synchronized (lock) {
81
+ if (!loop) {
82
+ playing = false;
83
+ }
84
+ }
85
+ });
86
+ mp.setOnErrorListener(
87
+ (p, what, extra) -> {
88
+ Log.e(TAG, "MediaPlayer error " + what + " (" + extra + ") for " + source);
89
+ synchronized (lock) {
90
+ playing = false;
91
+ }
92
+ return true;
93
+ });
94
+ try {
95
+ mp.setDataSource(context, uri);
96
+ } catch (IOException | IllegalArgumentException | SecurityException e) {
97
+ mp.release();
98
+ throw new RuntimeException("createVideoPlayer: cannot open " + source, e);
99
+ }
100
+ mp.setLooping(loop);
101
+ mp.prepareAsync();
102
+ synchronized (lock) {
103
+ player = mp;
104
+ }
105
+ }
106
+
107
+ // A URL (http, https, file, content) or a plain file path.
108
+ private static Uri parseSource(String source) {
109
+ Uri uri = Uri.parse(source);
110
+ if (uri.getScheme() == null) {
111
+ return Uri.fromFile(new File(source));
112
+ }
113
+ return uri;
114
+ }
115
+
116
+ // The frame geometry comes from the container (MediaMetadataRetriever for
117
+ // the size, rotation and duration, MediaExtractor for the frame rate). The
118
+ // metadata of a remote video is fetched off the calling thread; the getters
119
+ // report 0 until it is known.
120
+ private void readMetadata(Context context, Uri uri) {
121
+ Thread thread =
122
+ new Thread(
123
+ () -> {
124
+ MediaMetadataRetriever retriever = new MediaMetadataRetriever();
125
+ try {
126
+ String scheme = uri.getScheme();
127
+ if ("http".equals(scheme) || "https".equals(scheme)) {
128
+ retriever.setDataSource(uri.toString(), new java.util.HashMap<>());
129
+ } else {
130
+ retriever.setDataSource(context, uri);
131
+ }
132
+ int w = parseInt(retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH));
133
+ int h = parseInt(retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT));
134
+ if (w > 0 && h > 0 && width == 0) {
135
+ width = w;
136
+ height = h;
137
+ }
138
+ rotation = parseInt(retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION));
139
+ long durationMs = parseInt(retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION));
140
+ if (durationMs > 0 && durationSeconds == 0) {
141
+ durationSeconds = durationMs / 1000.0;
142
+ }
143
+ } catch (RuntimeException e) {
144
+ Log.w(TAG, "Could not read the metadata of " + uri + ": " + e.getMessage());
145
+ } finally {
146
+ try {
147
+ retriever.release();
148
+ } catch (IOException ignored) {
149
+ // Nothing to recover.
150
+ }
151
+ }
152
+
153
+ MediaExtractor extractor = new MediaExtractor();
154
+ try {
155
+ extractor.setDataSource(context, uri, null);
156
+ for (int i = 0; i < extractor.getTrackCount(); i++) {
157
+ MediaFormat format = extractor.getTrackFormat(i);
158
+ String mime = format.getString(MediaFormat.KEY_MIME);
159
+ if (mime != null && mime.startsWith("video/")) {
160
+ if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
161
+ frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
162
+ }
163
+ break;
164
+ }
165
+ }
166
+ } catch (IOException | RuntimeException e) {
167
+ Log.w(TAG, "Could not read the frame rate of " + uri + ": " + e.getMessage());
168
+ } finally {
169
+ extractor.release();
170
+ }
171
+ },
172
+ "WebGPUVideoPlayer metadata");
173
+ thread.setDaemon(true);
174
+ thread.start();
175
+ }
176
+
177
+ private static int parseInt(String value) {
178
+ if (value == null) {
179
+ return 0;
180
+ }
181
+ try {
182
+ return Integer.parseInt(value);
183
+ } catch (NumberFormatException e) {
184
+ return 0;
185
+ }
186
+ }
187
+
188
+ // The decoder reports the frame size (possibly more than once for adaptive
189
+ // streams); the ImageReader that receives the frames is sized accordingly.
190
+ private void onVideoSizeChanged(int w, int h) {
191
+ if (w <= 0 || h <= 0) {
192
+ return;
193
+ }
194
+ ImageReader previous;
195
+ synchronized (lock) {
196
+ if (released || player == null) {
197
+ return;
198
+ }
199
+ if (reader != null && reader.getWidth() == w && reader.getHeight() == h) {
200
+ return;
201
+ }
202
+ if (width == 0) {
203
+ width = w;
204
+ height = h;
205
+ }
206
+ previous = reader;
207
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
208
+ reader = ImageReader.newInstance(w, h, ImageFormat.PRIVATE, MAX_IMAGES, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
209
+ } else {
210
+ reader = ImageReader.newInstance(w, h, ImageFormat.PRIVATE, MAX_IMAGES);
211
+ }
212
+ player.setSurface(reader.getSurface());
213
+ }
214
+ if (previous != null) {
215
+ // Frames still held by the consumer were acquired from the old reader;
216
+ // closing it closes them, and the native side keeps the buffers alive.
217
+ synchronized (lock) {
218
+ acquired.clear();
219
+ }
220
+ previous.close();
221
+ }
222
+ }
223
+
224
+ private void onPrepared(MediaPlayer mp) {
225
+ synchronized (lock) {
226
+ if (released) {
227
+ return;
228
+ }
229
+ prepared = true;
230
+ int durationMs = mp.getDuration();
231
+ if (durationMs > 0) {
232
+ durationSeconds = durationMs / 1000.0;
233
+ }
234
+ mp.setVolume(volume, volume);
235
+ mp.setLooping(loop);
236
+ if (pendingSeekSeconds >= 0) {
237
+ mp.seekTo((long) (pendingSeekSeconds * 1000), MediaPlayer.SEEK_CLOSEST);
238
+ pendingSeekSeconds = -1;
239
+ }
240
+ if (playing) {
241
+ mp.start();
242
+ }
243
+ }
244
+ }
245
+
246
+ /**
247
+ * The most recently decoded frame, or null when no new frame was decoded
248
+ * since the last call (or when the consumer holds {@link #MAX_IMAGES}
249
+ * frames). Close it with {@link #closeImage} once the frame is released.
250
+ */
251
+ @DoNotStrip
252
+ public Image copyLatestFrame() {
253
+ ImageReader current;
254
+ synchronized (lock) {
255
+ if (released || reader == null) {
256
+ return null;
257
+ }
258
+ current = reader;
259
+ }
260
+ Image image;
261
+ try {
262
+ image = current.acquireLatestImage();
263
+ } catch (IllegalStateException e) {
264
+ Log.w(TAG, "copyLatestFrame: every frame is held, release one to get a new frame");
265
+ return null;
266
+ }
267
+ if (image == null) {
268
+ return null;
269
+ }
270
+ synchronized (lock) {
271
+ acquired.add(image);
272
+ }
273
+ return image;
274
+ }
275
+
276
+ /** Hands the buffer of a frame returned by {@link #copyLatestFrame} back to the decoder. */
277
+ @DoNotStrip
278
+ public void closeImage(Image image) {
279
+ synchronized (lock) {
280
+ if (!acquired.remove(image)) {
281
+ return;
282
+ }
283
+ }
284
+ image.close();
285
+ }
286
+
287
+ @DoNotStrip
288
+ public void play() {
289
+ synchronized (lock) {
290
+ playing = true;
291
+ if (player != null && prepared) {
292
+ player.start();
293
+ }
294
+ }
295
+ }
296
+
297
+ @DoNotStrip
298
+ public void pause() {
299
+ synchronized (lock) {
300
+ playing = false;
301
+ if (player != null && prepared && player.isPlaying()) {
302
+ player.pause();
303
+ }
304
+ }
305
+ }
306
+
307
+ @DoNotStrip
308
+ public boolean getPaused() {
309
+ synchronized (lock) {
310
+ return !playing;
311
+ }
312
+ }
313
+
314
+ @DoNotStrip
315
+ public double getCurrentTime() {
316
+ synchronized (lock) {
317
+ if (player != null && prepared) {
318
+ return player.getCurrentPosition() / 1000.0;
319
+ }
320
+ return pendingSeekSeconds >= 0 ? pendingSeekSeconds : 0;
321
+ }
322
+ }
323
+
324
+ // An exact seek, also while paused: the decoder then renders the frame at
325
+ // the new position, which the next copyLatestFrame() returns.
326
+ @DoNotStrip
327
+ public void seek(double seconds) {
328
+ synchronized (lock) {
329
+ if (player != null && prepared) {
330
+ player.seekTo((long) (Math.max(0, seconds) * 1000), MediaPlayer.SEEK_CLOSEST);
331
+ } else {
332
+ pendingSeekSeconds = Math.max(0, seconds);
333
+ }
334
+ }
335
+ }
336
+
337
+ @DoNotStrip
338
+ public double getDuration() {
339
+ return durationSeconds;
340
+ }
341
+
342
+ @DoNotStrip
343
+ public double getVolume() {
344
+ synchronized (lock) {
345
+ return volume;
346
+ }
347
+ }
348
+
349
+ @DoNotStrip
350
+ public void setVolume(double value) {
351
+ synchronized (lock) {
352
+ volume = (float) Math.max(0, Math.min(1, value));
353
+ if (player != null) {
354
+ player.setVolume(volume, volume);
355
+ }
356
+ }
357
+ }
358
+
359
+ @DoNotStrip
360
+ public boolean getLoop() {
361
+ synchronized (lock) {
362
+ return loop;
363
+ }
364
+ }
365
+
366
+ @DoNotStrip
367
+ public void setLoop(boolean value) {
368
+ synchronized (lock) {
369
+ loop = value;
370
+ if (player != null) {
371
+ player.setLooping(value);
372
+ }
373
+ }
374
+ }
375
+
376
+ @DoNotStrip
377
+ public int getVideoWidth() {
378
+ return width;
379
+ }
380
+
381
+ @DoNotStrip
382
+ public int getVideoHeight() {
383
+ return height;
384
+ }
385
+
386
+ @DoNotStrip
387
+ public int getRotation() {
388
+ return rotation;
389
+ }
390
+
391
+ @DoNotStrip
392
+ public double getFrameRate() {
393
+ return frameRate;
394
+ }
395
+
396
+ /** Stops playback and releases the decoder. Frames already copied stay valid until closed. */
397
+ @DoNotStrip
398
+ public void release() {
399
+ MediaPlayer mp;
400
+ ImageReader current;
401
+ synchronized (lock) {
402
+ if (released) {
403
+ return;
404
+ }
405
+ released = true;
406
+ playing = false;
407
+ mp = player;
408
+ player = null;
409
+ current = reader;
410
+ reader = null;
411
+ acquired.clear();
412
+ }
413
+ if (mp != null) {
414
+ mp.release();
415
+ }
416
+ if (current != null) {
417
+ current.close();
418
+ }
419
+ }
420
+ }
@@ -3,6 +3,9 @@
3
3
  #import <AVFoundation/AVFoundation.h>
4
4
  #import <CoreVideo/CoreVideo.h>
5
5
 
6
+ #include <algorithm>
7
+ #include <atomic>
8
+ #include <memory>
6
9
  #include <stdexcept>
7
10
 
8
11
  namespace rnwgpu {
@@ -16,27 +19,27 @@ namespace {
16
19
  // range) means luma is 16..235, chroma is 16..240 (8-bit).
17
20
  // Reference: https://en.wikipedia.org/wiki/YCbCr (BT.601 / BT.709).
18
21
  static constexpr float kBT709LimitedToRgb[12] = {
19
- 1.164383f, 0.000000f, 1.792741f, -0.972945f, //
20
- 1.164383f, -0.213249f, -0.532909f, 0.301517f, //
21
- 1.164383f, 2.112402f, 0.000000f, -1.133402f, //
22
+ 1.164383f, 0.000000f, 1.792741f, -0.972945f, //
23
+ 1.164383f, -0.213249f, -0.532909f, 0.301517f, //
24
+ 1.164383f, 2.112402f, 0.000000f, -1.133402f, //
22
25
  };
23
26
  static constexpr float kBT601LimitedToRgb[12] = {
24
- 1.164383f, 0.000000f, 1.596027f, -0.874202f, //
25
- 1.164383f, -0.391762f, -0.812968f, 0.531668f, //
26
- 1.164383f, 2.017232f, 0.000000f, -1.085631f, //
27
+ 1.164383f, 0.000000f, 1.596027f, -0.874202f, //
28
+ 1.164383f, -0.391762f, -0.812968f, 0.531668f, //
29
+ 1.164383f, 2.017232f, 0.000000f, -1.085631f, //
27
30
  };
28
31
  static constexpr float kBT2020LimitedToRgb[12] = {
29
- 1.164383f, 0.000000f, 1.678674f, -0.915688f, //
30
- 1.164383f, -0.187326f, -0.650424f, 0.347459f, //
31
- 1.164383f, 2.141772f, 0.000000f, -1.148145f, //
32
+ 1.164383f, 0.000000f, 1.678674f, -0.915688f, //
33
+ 1.164383f, -0.187326f, -0.650424f, 0.347459f, //
34
+ 1.164383f, 2.141772f, 0.000000f, -1.148145f, //
32
35
  };
33
36
 
34
37
  // Pick the right YUV→RGB matrix from the pixel buffer's color attachments.
35
38
  // Falls back to BT.709 limited range (the right call for ≥720p H.264, which
36
39
  // is what AVPlayer hands us for Big Buck Bunny and most streamed media).
37
40
  static void fillYuvMatrix(CVPixelBufferRef pixelBuffer, float out[12]) {
38
- CFTypeRef matrixKey = CVBufferGetAttachment(
39
- pixelBuffer, kCVImageBufferYCbCrMatrixKey, nullptr);
41
+ CFTypeRef matrixKey =
42
+ CVBufferGetAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, nullptr);
40
43
  const float *src = kBT709LimitedToRgb;
41
44
  if (matrixKey) {
42
45
  auto matrix = (CFStringRef)matrixKey;
@@ -53,28 +56,108 @@ static void fillYuvMatrix(CVPixelBufferRef pixelBuffer, float out[12]) {
53
56
  }
54
57
 
55
58
  // Map a CVPixelBuffer's pixel format to our VideoPixelFormat enum.
56
- static VideoPixelFormat pixelFormatFromCVPixelBuffer(
57
- CVPixelBufferRef pixelBuffer) {
59
+ static VideoPixelFormat
60
+ pixelFormatFromCVPixelBuffer(CVPixelBufferRef pixelBuffer) {
58
61
  OSType type = CVPixelBufferGetPixelFormatType(pixelBuffer);
59
62
  switch (type) {
60
- case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
61
- case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
62
- return VideoPixelFormat::NV12;
63
- default:
64
- return VideoPixelFormat::BGRA8;
63
+ case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
64
+ case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
65
+ return VideoPixelFormat::NV12;
66
+ default:
67
+ return VideoPixelFormat::BGRA8;
65
68
  }
66
69
  }
67
70
 
71
+ // The clockwise rotation, in degrees, that a track's preferredTransform
72
+ // applies to its frames: a video recorded in portrait stores landscape frames
73
+ // with a 90 or 270 degree transform.
74
+ static int rotationFromTransform(CGAffineTransform t) {
75
+ if (t.a == 0 && t.b == 1 && t.c == -1 && t.d == 0) {
76
+ return 90;
77
+ }
78
+ if (t.a == 0 && t.b == -1 && t.c == 1 && t.d == 0) {
79
+ return 270;
80
+ }
81
+ if (t.a == -1 && t.b == 0 && t.c == 0 && t.d == -1) {
82
+ return 180;
83
+ }
84
+ return 0;
85
+ }
86
+
87
+ // The state the AVFoundation callbacks update. It is shared with the blocks
88
+ // observing the asset and the item, so a callback that fires after the player
89
+ // was destroyed only touches memory that outlives it.
90
+ struct AppleVideoPlayerState {
91
+ std::atomic<bool> playing{false};
92
+ std::atomic<bool> loop{true};
93
+ std::atomic<double> duration{0};
94
+ std::atomic<double> frameRate{0};
95
+ std::atomic<uint32_t> width{0};
96
+ std::atomic<uint32_t> height{0};
97
+ std::atomic<int> rotation{0};
98
+ };
99
+
68
100
  class AppleVideoPlayer : public IVideoPlayer {
69
101
  public:
70
- AppleVideoPlayer(AVPlayer *player, AVPlayerItemVideoOutput *output,
71
- id loopObserver)
72
- : _player(player), _output(output), _loopObserver(loopObserver) {}
102
+ AppleVideoPlayer(AVPlayer *player, AVPlayerItem *item,
103
+ AVPlayerItemVideoOutput *output)
104
+ : _player(player), _output(output),
105
+ _state(std::make_shared<AppleVideoPlayerState>()) {
106
+ auto state = _state;
107
+
108
+ // The metadata loads asynchronously, so that a remote asset never blocks
109
+ // the calling thread; the getters report 0 until it has.
110
+ AVAsset *asset = item.asset;
111
+ #pragma clang diagnostic push
112
+ #pragma clang diagnostic ignored "-Wdeprecated-declarations"
113
+ [asset
114
+ loadValuesAsynchronouslyForKeys:@[ @"duration", @"tracks" ]
115
+ completionHandler:^{
116
+ if ([asset statusOfValueForKey:@"duration" error:nil] ==
117
+ AVKeyValueStatusLoaded) {
118
+ CMTime duration = asset.duration;
119
+ if (CMTIME_IS_NUMERIC(duration)) {
120
+ state->duration = CMTimeGetSeconds(duration);
121
+ }
122
+ }
123
+ if ([asset statusOfValueForKey:@"tracks" error:nil] ==
124
+ AVKeyValueStatusLoaded) {
125
+ AVAssetTrack *track =
126
+ [[asset tracksWithMediaType:AVMediaTypeVideo]
127
+ firstObject];
128
+ if (track) {
129
+ CGSize size = track.naturalSize;
130
+ state->width = static_cast<uint32_t>(size.width);
131
+ state->height = static_cast<uint32_t>(size.height);
132
+ state->rotation =
133
+ rotationFromTransform(track.preferredTransform);
134
+ state->frameRate = track.nominalFrameRate;
135
+ }
136
+ }
137
+ }];
138
+ #pragma clang diagnostic pop
139
+
140
+ // At the end of the stream, either loop or stop. The player keeps showing
141
+ // the last frame either way (actionAtItemEnd is None).
142
+ __weak AVPlayer *weakPlayer = player;
143
+ _endObserver = [[NSNotificationCenter defaultCenter]
144
+ addObserverForName:AVPlayerItemDidPlayToEndTimeNotification
145
+ object:item
146
+ queue:[NSOperationQueue mainQueue]
147
+ usingBlock:^(NSNotification * /*note*/) {
148
+ if (state->loop) {
149
+ [weakPlayer seekToTime:kCMTimeZero];
150
+ [weakPlayer play];
151
+ } else {
152
+ state->playing = false;
153
+ }
154
+ }];
155
+ }
73
156
 
74
157
  ~AppleVideoPlayer() override {
75
- if (_loopObserver) {
76
- [[NSNotificationCenter defaultCenter] removeObserver:_loopObserver];
77
- _loopObserver = nil;
158
+ if (_endObserver) {
159
+ [[NSNotificationCenter defaultCenter] removeObserver:_endObserver];
160
+ _endObserver = nil;
78
161
  }
79
162
  [_player pause];
80
163
  _player = nil;
@@ -105,13 +188,51 @@ public:
105
188
  }
106
189
  }
107
190
 
108
- void play() override { [_player play]; }
109
- void pause() override { [_player pause]; }
191
+ void play() override {
192
+ _state->playing = true;
193
+ [_player play];
194
+ }
195
+
196
+ void pause() override {
197
+ _state->playing = false;
198
+ [_player pause];
199
+ }
200
+
201
+ bool paused() override { return !_state->playing; }
202
+
203
+ double currentTime() override {
204
+ CMTime time = [_player currentTime];
205
+ return CMTIME_IS_NUMERIC(time) ? CMTimeGetSeconds(time) : 0;
206
+ }
207
+
208
+ // An exact seek: the output then holds the frame at that position, also
209
+ // while paused, so the next copyLatestFrame() returns it.
210
+ void seek(double seconds) override {
211
+ [_player seekToTime:CMTimeMakeWithSeconds(seconds, NSEC_PER_SEC)
212
+ toleranceBefore:kCMTimeZero
213
+ toleranceAfter:kCMTimeZero];
214
+ }
215
+
216
+ double duration() override { return _state->duration; }
217
+
218
+ double volume() override { return _player.volume; }
219
+ void setVolume(double volume) override {
220
+ _player.volume = static_cast<float>(std::clamp(volume, 0.0, 1.0));
221
+ }
222
+
223
+ bool loop() override { return _state->loop; }
224
+ void setLoop(bool loop) override { _state->loop = loop; }
225
+
226
+ uint32_t videoWidth() override { return _state->width; }
227
+ uint32_t videoHeight() override { return _state->height; }
228
+ int rotation() override { return _state->rotation; }
229
+ double frameRate() override { return _state->frameRate; }
110
230
 
111
231
  private:
112
232
  AVPlayer *_player;
113
233
  AVPlayerItemVideoOutput *_output;
114
- id _loopObserver;
234
+ std::shared_ptr<AppleVideoPlayerState> _state;
235
+ id _endObserver = nil;
115
236
  };
116
237
 
117
238
  } // namespace
@@ -142,8 +263,8 @@ VideoFrameHandle wrapCVPixelBuffer(CVPixelBufferRef pixelBuffer) {
142
263
  return handle;
143
264
  }
144
265
 
145
- std::unique_ptr<IVideoPlayer>
146
- createAppleVideoPlayer(const std::string &path, VideoPixelFormat format) {
266
+ std::unique_ptr<IVideoPlayer> createAppleVideoPlayer(const std::string &path,
267
+ VideoPixelFormat format) {
147
268
  NSString *nsPath = [NSString stringWithUTF8String:path.c_str()];
148
269
  NSURL *url;
149
270
  if ([nsPath hasPrefix:@"http://"] || [nsPath hasPrefix:@"https://"] ||
@@ -164,8 +285,8 @@ createAppleVideoPlayer(const std::string &path, VideoPixelFormat format) {
164
285
  // importExternalTexture. BGRA is the "decode + convert" path for the
165
286
  // single-plane SharedTextureMemory demo.
166
287
  OSType pixelFormat = format == VideoPixelFormat::NV12
167
- ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
168
- : kCVPixelFormatType_32BGRA;
288
+ ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
289
+ : kCVPixelFormatType_32BGRA;
169
290
  NSDictionary *outputSettings = @{
170
291
  (NSString *)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
171
292
  (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{},
@@ -176,20 +297,11 @@ createAppleVideoPlayer(const std::string &path, VideoPixelFormat format) {
176
297
  [item addOutput:output];
177
298
 
178
299
  AVPlayer *player = [AVPlayer playerWithPlayerItem:item];
300
+ // Keep the last frame at the end of the stream; AppleVideoPlayer decides
301
+ // whether to loop.
179
302
  player.actionAtItemEnd = AVPlayerActionAtItemEndNone;
180
303
 
181
- // Loop on end-of-stream by seeking back to zero.
182
- __weak AVPlayer *weakPlayer = player;
183
- id loopObserver = [[NSNotificationCenter defaultCenter]
184
- addObserverForName:AVPlayerItemDidPlayToEndTimeNotification
185
- object:item
186
- queue:[NSOperationQueue mainQueue]
187
- usingBlock:^(NSNotification * /*note*/) {
188
- [weakPlayer seekToTime:kCMTimeZero];
189
- [weakPlayer play];
190
- }];
191
-
192
- return std::make_unique<AppleVideoPlayer>(player, output, loopObserver);
304
+ return std::make_unique<AppleVideoPlayer>(player, item, output);
193
305
  }
194
306
 
195
307
  std::string writeAppleTestVideoFile() {
@@ -231,8 +343,7 @@ std::string writeAppleTestVideoFile() {
231
343
  writerInput.expectsMediaDataInRealTime = NO;
232
344
 
233
345
  NSDictionary *bufferAttrs = @{
234
- (NSString *)kCVPixelBufferPixelFormatTypeKey :
235
- @(kCVPixelFormatType_32BGRA),
346
+ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA),
236
347
  (NSString *)kCVPixelBufferWidthKey : @(kWidth),
237
348
  (NSString *)kCVPixelBufferHeightKey : @(kHeight),
238
349
  (NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{},
@@ -274,8 +385,7 @@ std::string writeAppleTestVideoFile() {
274
385
  for (int x = 0; x < kWidth; ++x) {
275
386
  uint8_t r = static_cast<uint8_t>((x + phase) & 0xFF);
276
387
  uint8_t g = static_cast<uint8_t>((y + phase * 2) & 0xFF);
277
- uint8_t b =
278
- static_cast<uint8_t>(((x + y + phase) & 0x40) ? 220 : 30);
388
+ uint8_t b = static_cast<uint8_t>(((x + y + phase) & 0x40) ? 220 : 30);
279
389
  row[x * 4 + 0] = b;
280
390
  row[x * 4 + 1] = g;
281
391
  row[x * 4 + 2] = r;