react-native-skia-webgpu-video 0.0.0-stage → 0.1.0-beta.1

package/android/src/main/java/com/azzapp/rnskv/VideoCompositionFramesExtractor.java

@@ -0,0 +1,463 @@
+package com.azzapp.rnskv;
+
+import android.app.Activity;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.os.Message;
+import android.os.Process;
+import android.os.SystemClock;
+import android.util.Log;
+import android.view.Choreographer;
+
+import androidx.media3.common.util.UnstableApi;
+
+import com.facebook.react.bridge.ReactApplicationContext;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.microedition.khronos.egl.EGLContext;
+import javax.microedition.khronos.egl.EGL10;
+
+/**
+ * A class that previews a video composition.
+ */
+@UnstableApi
+public class VideoCompositionFramesExtractor {
+  private static final String TAG = "VideoCompositionFramesExtractor";
+
+  private static final int PLAYBACK_PREPARE = 1;
+  private static final int PLAYBACK_PLAY = 2;
+  private static final int PLAYBACK_PAUSE = 3;
+  private static final int PLAYBACK_LOOP = 4;
+  private static final int PLAYBACK_SEEK = 5;
+  private static final int PLAYBACK_RELEASE = 6;
+
+  private final VideoComposition composition;
+
+  private final VideoCompositionDecoder decoder;
+
+  private AudioCompositionPlayer audioPlayer;
+
+  private final PlaybackThread playbackThread;
+
+  private final Handler handler;
+
+  private final NativeEventDispatcher eventDispatcher;
+
+  private boolean playWhenReady = false;
+
+  private boolean prepared;
+  private boolean releasing;
+  private boolean looping;
+
+
+  private long pendingSeek = 0;
+  // Read on the thread that collects frames (see releaseForNextVsync), written on the playback thread.
+  private volatile boolean isPlaying = false;
+
+  private volatile long startTime = 0;
+  private volatile long pausePosition = 0;
+  private boolean isEOS = false;
+  private boolean completeDispatched = false;
+
+  // Vsync-aligned release, written and read on the thread that collects frames.
+  private long lastVsyncNanos = 0;
+  private long recordedVsyncNanos = 0;
+  private boolean recordingVsync = false;
+  /** Notes each vsync's timestamp on the collecting thread, for as long as the preview lives. */
+  private final Choreographer.FrameCallback vsyncRecorder = new Choreographer.FrameCallback() {
+    @Override
+    public void doFrame(long frameTimeNanos) {
+      if (releasing) {
+        unwatchDisplay();
+        return;
+      }
+      recordedVsyncNanos = frameTimeNanos;
+      Choreographer.getInstance().postFrameCallback(this);
+    }
+  };
+  // The display the preview is on; null until the first collection, or when unknown.
+  private DisplayVsyncPeriod vsyncPeriod;
+  // When that thread last released frames; the playback loop stands back meanwhile.
+  private volatile long lastVsyncReleaseNanos = 0;
+  // The release itself runs on the playback thread: the decoder's locks are held
+  // there for whole codec flushes on a seek, and the UI thread must never
+  // wait on them (measured: 65 ms frames when it did).
+  private volatile long vsyncReleasePosition = 0;
+  private final AtomicBoolean vsyncReleasePosted = new AtomicBoolean(false);
+  private final Runnable vsyncRelease = this::runVsyncRelease;
+
+  private void runVsyncRelease() {
+    vsyncReleasePosted.set(false);
+    if (!releasing) decoder.render(vsyncReleasePosition);
+  }
+  private static final long VSYNC_RELEASE_TIMEOUT_NANOS = 100_000_000L;
+
+  /**
+   * Create a new VideoCompositionFramesExtractor.
+   *
+   * @param composition the video composition to preview
+   */
+  public VideoCompositionFramesExtractor(VideoComposition composition, NativeEventDispatcher eventDispatcher) {
+    this.eventDispatcher = eventDispatcher;
+    this.composition = composition;
+    decoder = new VideoCompositionDecoder(composition, true);
+    // Only a decoder that cannot open, as on iOS: a codec's passing errors
+    // stay quiet, since the player gives up on the first error it reports.
+    decoder.setOnOpenErrorListener(
+      error -> eventDispatcher.dispatchEvent("error", messageOf(error)));
+    playbackThread = new PlaybackThread();
+    playbackThread.start();
+    handler = new Handler(playbackThread.getLooper(), playbackThread);
+  }
+
+  public void prepare() {
+    if (prepared) {
+      return;
+    }
+    try {
+      decoder.prepare(EGL10.EGL_NO_CONTEXT);
+    } catch (RuntimeException error) {
+      // A decoder that cannot open, as a lazy one reports it: thrown, it
+      // reached the UI thread's worklet uncaught and killed the app.
+      eventDispatcher.dispatchEvent("error", messageOf(error));
+      return;
+    }
+    handler.sendEmptyMessage(PLAYBACK_PREPARE);
+  }
+
+  /**
+   * Start playing the composition.
+   */
+  public void play() {
+    handler.sendEmptyMessage(PLAYBACK_PLAY);
+  }
+
+  /**
+   * Pause the composition.
+   */
+  public void pause() {
+    handler.sendEmptyMessage(PLAYBACK_PAUSE);
+  }
+
+  /**
+   * Seek to given position
+   */
+  public void seekTo(long position) {
+    handler.removeMessages(PLAYBACK_SEEK);
+    handler.obtainMessage(PLAYBACK_SEEK, position).sendToTarget();
+  }
+
+  /**
+   * Decode the next frame of each composition item according to the current position of the player.
+   *
+   * @return an independently retained snapshot map; callers close its frames
+   * when finished, without affecting the producer or other snapshots
+   */
+  public Map<String, VideoFrame> decodeCompositionFrames() {
+    Map<String, VideoFrame> frames = decoder.updateVideosFrames();
+    releaseForNextVsync();
+    return frames;
+  }
+
+  /**
+   * Releases the decoded frames the next vsync is due to show, from the thread
+   * that collects them, right after collecting.
+   *
+   * The playback loop released them on its 10 ms tick, out of phase with the
+   * display, so whether a frame was collected on the vsync after its time or
+   * the one after that was chance: 24 fps on a 60 Hz display came out in
+   * 17/67 ms steps among the 33/50 ms it implies. Released against the vsync's
+   * own timestamp, one vsync ahead, every frame is collected on the first
+   * vsync at or after its time. The position is decided here, against the
+   * vsync; the release is posted to the playback thread, a vsync ahead of the
+   * collection that needs it.
+   */
+  private void releaseForNextVsync() {
+    if (!recordingVsync) {
+      try {
+        // The collecting thread is a Choreographer thread (the UI thread).
+        Choreographer.getInstance().postFrameCallback(vsyncRecorder);
+        recordingVsync = true;
+      } catch (RuntimeException e) {
+        // No looper on this thread: leave the release to the playback loop.
+        return;
+      }
+      watchDisplay();
+      return;
+    }
+    long period = vsyncPeriod != null ? vsyncPeriod.getPeriodNanos() : 0;
+    if (period == 0 || recordedVsyncNanos == 0) return;
+    // This frame's vsync, whether the recorder has run in this frame yet or not.
+    long since = Math.max(0, System.nanoTime() - recordedVsyncNanos);
+    long vsyncNanos = recordedVsyncNanos + (since / period) * period;
+    if (vsyncNanos == lastVsyncNanos) return;
+    lastVsyncNanos = vsyncNanos;
+    long vsyncPeriodNanos = period;
+    long position = isPlaying
+      ? (vsyncNanos + vsyncPeriodNanos) / 1000 - startTime
+      : pausePosition;
+    lastVsyncReleaseNanos = System.nanoTime();
+    vsyncReleasePosition = position;
+    if (vsyncReleasePosted.compareAndSet(false, true)) {
+      handler.post(vsyncRelease);
+    }
+  }
+
+  /**
+   * Follows the refresh rate of the display the preview is on, from the
+   * collecting thread, whose looper then receives its changes. Without a React
+   * context (instrumented tests) the period stays unknown and the playback loop
+   * releases on its tick.
+   */
+  private void watchDisplay() {
+    ReactApplicationContext context = ReactNativeSkiaVideoModule.currentReactApplicationContext();
+    if (context == null) return;
+    Activity activity = context.getCurrentActivity();
+    vsyncPeriod = DisplayVsyncPeriod.watch(context,
+      activity != null ? activity.getWindowManager().getDefaultDisplay() : null);
+  }
+
+  private void unwatchDisplay() {
+    if (vsyncPeriod != null) {
+      vsyncPeriod.release();
+      vsyncPeriod = null;
+    }
+  }
+
+  /**
+   * Makes the decoder's GL context current on the calling thread, the one that
+   * calls {@link #decodeCompositionFrames}.
+   */
+  public void makeGLContextCurrent() {
+    decoder.makeGLContextCurrent();
+  }
+
+  /**
+   * @return the current position of the player in microseconds
+   */
+  public long getCurrentPosition() {
+    return isPlaying ? microTime() - startTime : pausePosition;
+  }
+
+  /**
+   * @return the version of the frames returned by {@link #decodeCompositionFrames},
+   * see {@link VideoCompositionDecoder#getFramesVersion()}
+   */
+  public long getFramesVersion() {
+    return decoder.getFramesVersion();
+  }
+
+  /**
+   * @return whether the player is looping
+   */
+  public boolean getIsLooping() {
+    return looping;
+  }
+
+  /**
+   * Set whether the player should loop.
+   */
+  public void setIsLooping(boolean value) {
+    looping = value;
+    if (prepared && isEOS && value) {
+      play();
+    }
+  }
+
+  /**
+   * @return whether the player is currently playing
+   */
+  public boolean getIsPlaying() {
+    return isPlaying;
+  }
+
+  public void release() {
+    if (!playbackThread.isAlive()) {
+      decoder.release();
+      return;
+    }
+    isPlaying = false;
+    releasing = true;
+    handler.sendEmptyMessage(PLAYBACK_RELEASE);
+  }
+
+  private void prepareInternal() {
+    decoder.start();
+    if (composition.hasAudio()) {
+      audioPlayer = new AudioCompositionPlayer(composition);
+      audioPlayer.setOnErrorListener(
+        message -> eventDispatcher.dispatchEvent("error", message));
+      audioPlayer.prepare();
+    }
+    prepared = true;
+    eventDispatcher.dispatchEvent("ready", null);
+    handler.sendEmptyMessage(PLAYBACK_LOOP);
+    if (pendingSeek != 0) {
+      seekInternal(pendingSeek);
+    }
+    if (playWhenReady) {
+      playInternal();
+    }
+  }
+
+  private void playInternal() {
+    if (!prepared) {
+      playWhenReady = true;
+      return;
+    }
+    if (isEOS) {
+      isEOS = false;
+      pausePosition = 0;
+      seekInternal(0);
+    }
+    startTime = microTime() - pausePosition;
+    isPlaying = true;
+    pausePosition = 0;
+  }
+
+  private void pauseInternal() {
+    if (!isPlaying) {
+      playWhenReady = false;
+      return;
+    }
+    pausePosition = getCurrentPosition();
+    isPlaying = false;
+    if (audioPlayer != null) {
+      audioPlayer.pause();
+    }
+  }
+
+  private void loopInternal() throws IOException, InterruptedException {
+    long loopStartTime = SystemClock.elapsedRealtime();
+
+    long currentPosition = getCurrentPosition();
+
+    isEOS = currentPosition >= TimeHelpers.secToUs(composition.getDuration());
+    if (isEOS) {
+      if (!completeDispatched) {
+        completeDispatched = true;
+        eventDispatcher.dispatchEvent("complete", null);
+      }
+      isPlaying = false;
+      pausePosition = TimeHelpers.secToUs(composition.getDuration());
+      currentPosition = pausePosition;
+    } else {
+      completeDispatched = false;
+    }
+    decoder.updateWindow(currentPosition);
+    if (System.nanoTime() - lastVsyncReleaseNanos > VSYNC_RELEASE_TIMEOUT_NANOS) {
+      // No one collecting at vsync (not yet, or no longer): release on the tick.
+      decoder.render(currentPosition);
+    }
+    if (isEOS && looping) {
+      playInternal();
+    }
+    if (audioPlayer != null) {
+      audioPlayer.update(getCurrentPosition(), isPlaying);
+    }
+    long delay = 10;
+    long duration = (SystemClock.elapsedRealtime() - loopStartTime);
+    delay = delay - duration;
+    if (delay > 0) {
+      handler.sendEmptyMessageDelayed(PLAYBACK_LOOP, delay);
+    } else {
+      handler.sendEmptyMessage(PLAYBACK_LOOP);
+    }
+  }
+
+  private void seekInternal(long position) {
+    if (!prepared) {
+      pendingSeek = position;
+      return;
+    }
+    decoder.seekTo(position);
+    // After the seek, so a decoder opened for this position is not sought again.
+    decoder.updateWindow(position);
+    if (audioPlayer != null) {
+      audioPlayer.seekTo(position);
+    }
+    if (isPlaying) {
+      startTime = microTime() - position;
+    } else {
+      pausePosition = position;
+    }
+  }
+
+  private void releaseInternal() {
+    if (audioPlayer != null) {
+      try {
+        audioPlayer.release();
+      } catch (Exception e) {
+        Log.w(TAG, "Could not release the audio players", e);
+      }
+      audioPlayer = null;
+    }
+    playbackThread.interrupt();
+    playbackThread.quit();
+    decoder.release();
+  }
+
+  private class PlaybackThread extends HandlerThread implements Handler.Callback {
+    public PlaybackThread() {
+      super(TAG + PlaybackThread.class.getSimpleName(), Process.THREAD_PRIORITY_VIDEO);
+    }
+
+    @Override
+    public boolean handleMessage(Message msg) {
+      try {
+        if (releasing) {
+          // When the releasing flag is set, just release without processing any more messages
+          releaseInternal();
+          return true;
+        }
+
+        switch (msg.what) {
+          case PLAYBACK_PREPARE -> {
+            prepareInternal();
+            return true;
+          }
+          case PLAYBACK_PLAY -> {
+            playInternal();
+            return true;
+          }
+          case PLAYBACK_PAUSE -> {
+            pauseInternal();
+            return true;
+          }
+          case PLAYBACK_LOOP -> {
+            loopInternal();
+            return true;
+          }
+          case PLAYBACK_SEEK -> {
+            seekInternal((Long) msg.obj);
+            return true;
+          }
+          case PLAYBACK_RELEASE -> {
+            releaseInternal();
+            return true;
+          }
+          default -> {
+            return false;
+          }
+        }
+      } catch (Exception error) {
+        eventDispatcher.dispatchEvent("error", messageOf(error));
+      }
+
+      // Release after an exception
+      releaseInternal();
+      return true;
+    }
+  }
+
+  // Never null: the native side reads the message as a string.
+  private static String messageOf(Exception error) {
+    return error.getMessage() != null ? error.getMessage() : error.toString();
+  }
+
+  private static long microTime() {
+    return System.nanoTime() / 1000;
+  }
+}

package/android/src/main/java/com/azzapp/rnskv/VideoCompositionFramesExtractorSync.java

@@ -0,0 +1,241 @@
+package com.azzapp.rnskv;
+
+import android.opengl.GLES20;
+import android.os.Handler;
+import android.os.HandlerThread;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import javax.microedition.khronos.egl.EGLContext;
+import javax.microedition.khronos.egl.EGL10;
+
+public class VideoCompositionFramesExtractorSync {
+  private final VideoComposition composition;
+
+  private final VideoCompositionDecoder decoder;
+
+  private boolean decoding = false;
+  private long decodingTimeUs;
+
+  private final Map<VideoComposition.Item, Long> itemsTimes = new HashMap<>();
+
+  private final Set<VideoComposition.Item> itemsEnded = new HashSet<>();
+
+  private final Map<String, Long> renderedTimes = new HashMap<>();
+
+  private HandlerThread exportThread = null;
+
+  private Handler handler;
+
+  private static final long DECODE_TIMEOUT_MS = 30000;
+  private final DecodeRequests<Map<String, VideoFrame>> requests =
+    new DecodeRequests<>(VideoCompositionDecoder::closeFrames);
+  private volatile CompletableFuture<Void> startup;
+
+  public VideoCompositionFramesExtractorSync(VideoComposition composition) {
+    this.composition = composition;
+    this.decoder = new VideoCompositionDecoder(composition, false);
+  }
+
+  public void start() throws Exception {
+    final CompletableFuture<Void> initializing;
+    synchronized (this) {
+      if (requests.isClosed()) throw new IllegalStateException("Video decoder has been released");
+      if (startup == null) {
+        exportThread = new HandlerThread("ReactNativeSkiaVideo-ExportThread");
+        exportThread.start();
+        handler = new Handler(exportThread.getLooper());
+        startup = new CompletableFuture<>();
+        final CompletableFuture<Void> started = startup;
+        if (!handler.post(() -> {
+          try {
+            if (requests.isClosed()) throw new IllegalStateException("Video decoder has been released");
+            decoder.prepare(EGL10.EGL_NO_CONTEXT);
+            decoder.setOnErrorListener(this::handleError);
+            decoder.setOnFrameAvailableListener(this::onFrameAvailable);
+            decoder.setOnItemEndReachedListener(this::onItemEndReached);
+            decoder.setOnItemImageAvailableListener(this::onItemImageAvailable);
+            decoder.start();
+            started.complete(null);
+          } catch (Exception error) {
+            handleError(error);
+            started.completeExceptionally(error);
+          }
+        })) started.completeExceptionally(new IllegalStateException("Export looper rejected startup"));
+      }
+      initializing = startup;
+    }
+    try { initializing.get(DECODE_TIMEOUT_MS, TimeUnit.MILLISECONDS); }
+    catch (Exception error) { release(); throw error; }
+  }
+
+  /**
+   * Decode the next frame of each composition item according to the current position of the player.
+   *
+   * @return a map of item id to video frame
+   */
+  public Map<String, VideoFrame> decodeCompositionFrames(double time) throws Exception {
+    final CompletableFuture<Map<String, VideoFrame>> request;
+    final long targetUs = TimeHelpers.secToUs(time);
+    synchronized (this) {
+      if (handler == null || startup == null || !startup.isDone()) {
+        throw new IllegalStateException("Video decoder is not started");
+      }
+      request = requests.begin();
+      if (!handler.post(() -> {
+        if (requests.isClosed()) return;
+        try {
+          decodingTimeUs = targetUs;
+          decoder.updateWindow(decodingTimeUs);
+          forgetClosedItems();
+          decoding = true;
+          renderedTimes.clear();
+          checkIfFrameDecoded();
+        } catch (Exception error) { handleError(error); }
+      })) requests.fail(new IllegalStateException("Export looper rejected decode request"));
+    }
+    try { return requests.await(request, DECODE_TIMEOUT_MS); }
+    catch (Exception error) { release(); throw error; }
+  }
+
+  public void release() {
+    final Handler ownedHandler;
+    final HandlerThread ownedThread;
+    synchronized (this) {
+      if (requests.isClosed() && handler == null) return;
+      requests.close();
+      if (startup != null) startup.cancel(true);
+      ownedHandler = handler;
+      ownedThread = exportThread;
+      handler = null;
+      exportThread = null;
+    }
+    if (ownedHandler != null) {
+      // The decoder's EGL context and GL resources are bound to the export
+      // thread (where they are current); releasing them from another thread
+      // would silently fail the GL deletes and defer the EGL context
+      // destruction. Release on the export thread, then let the looper
+      // drain and quit.
+      if (!ownedHandler.post(decoder::release)) decoder.release();
+      if (ownedThread != null) ownedThread.quitSafely();
+    } else {
+      decoder.release();
+      if (ownedThread != null) ownedThread.quit();
+    }
+  }
+
+  /**
+   * With lazy decoders, an item reopened later must wait for its own frames.
+   */
+  private void forgetClosedItems() {
+    for (VideoComposition.Item item : composition.getItems()) {
+      if (item.isVideo() && !decoder.isOpen(item)) {
+        itemsTimes.remove(item);
+        itemsEnded.remove(item);
+      }
+    }
+  }
+
+  private void onFrameAvailable(VideoComposition.Item item, long presentationTimeUs) {
+    if (requests.isClosed()) return;
+    try {
+      itemsTimes.put(item, presentationTimeUs);
+      if (decoding) checkIfFrameDecoded();
+    } catch (Exception error) { handleError(error); }
+  }
+
+  private void onItemEndReached(VideoComposition.Item item) {
+    if (requests.isClosed()) return;
+    try {
+      itemsEnded.add(item);
+      if (decoding) checkIfFrameDecoded();
+    } catch (Exception error) { handleError(error); }
+  }
+
+  private void checkIfFrameDecoded() {
+    boolean allItemsReady = true;
+    for (VideoComposition.Item item : composition.getItems()) {
+      if (!item.isVideo() || !decoder.isOpen(item)) {
+        continue;
+      }
+      // Before the frame check: an item that ends without a single frame in
+      // its range (a start past the file's end) would otherwise hold the
+      // export forever.
+      if (itemsEnded.contains(item)) {
+        continue;
+      }
+      if (!itemsTimes.containsKey(item)) {
+        allItemsReady = false;
+        continue;
+      }
+      Long itemTime = itemsTimes.get(item);
+      if (itemTime == null) {
+        allItemsReady = false;
+        continue;
+      }
+      long itemCurrentTimeUs = itemTime;
+      long startTimeUs = TimeHelpers.secToUs(item.getStartTime());
+      long compositionStartTimeUs = TimeHelpers.secToUs(item.getCompositionStartTime());
+      if (itemCurrentTimeUs - startTimeUs < decodingTimeUs - compositionStartTimeUs) {
+        allItemsReady = false;
+      }
+    }
+    Map<String, Long> renderedTimes = decoder.render(decodingTimeUs);
+    renderedTimes.forEach((itemId, time) -> {
+      if (time != null) {
+        this.renderedTimes.put(itemId, time);
+      }
+    });
+    if (allItemsReady) {
+      decoding = false;
+      resolveIfReady();
+    }
+  }
+
+  private void onItemImageAvailable(VideoComposition.Item item) {
+    if (requests.isClosed()) return;
+    try {
+      if (!decoding) resolveIfReady();
+    } catch (Exception error) { handleError(error); }
+  }
+
+  private void handleError(Exception e) {
+    requests.fail(e);
+    CompletableFuture<Void> initializing = startup;
+    if (initializing != null) initializing.completeExceptionally(e);
+  }
+
+  private void resolveIfReady() {
+    if (!requests.isPending()) return;
+    Map<String, VideoFrame> videoFrames = decoder.updateInternalFrames();
+    for (VideoComposition.Item item : composition.getItems()) {
+      if (!item.isVideo() || !decoder.isOpen(item)) {
+        continue;
+      }
+      VideoFrame videoFrame = videoFrames.getOrDefault(item.getId(), null);
+      if (videoFrame == null) {
+        if (itemsEnded.contains(item) && !itemsTimes.containsKey(item)) {
+          // Ended without a frame: there will never be one to wait for.
+          continue;
+        }
+        return;
+      }
+      Long itemFrameTime = renderedTimes.getOrDefault(item.getId(), null);
+      if (itemFrameTime == null) {
+        continue;
+      }
+      long videoFrameTime = TimeHelpers.nsecToUs(videoFrame.getTimestampNs());
+      if (Math.abs(itemFrameTime - videoFrameTime) > 1000) {
+        return;
+      }
+    }
+    // Never publish the producer's owning wrappers. The completed request owns
+    // these aliases until await hands them over, or cancellation discards them.
+    requests.complete(VideoCompositionDecoder.retainFrames(videoFrames));
+  }
+}