react-native-skia-webgpu-video 0.0.0-stage → 0.1.0-beta.1

package/android/src/main/java/com/azzapp/rnskv/AudioCompositionPlayer.java

@@ -0,0 +1,321 @@
+package com.azzapp.rnskv;
+
+import android.net.Uri;
+import android.os.Looper;
+import android.os.SystemClock;
+
+import androidx.media3.common.C;
+import androidx.media3.common.MediaItem;
+import androidx.media3.common.PlaybackException;
+import androidx.media3.common.PlaybackParameters;
+import androidx.media3.common.Player;
+import androidx.media3.common.Tracks;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.exoplayer.ExoPlayer;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Plays the audio of the audio-enabled items of a video composition during
+ * playback. One audio only ExoPlayer is created per audio-enabled item and
+ * slaved to the composition clock through the {@link #update} method.
+ * Every method must be called on the thread that called {@link #prepare}.
+ */
+@UnstableApi
+public class AudioCompositionPlayer {
+
+  public interface OnErrorListener {
+    void onError(String message);
+  }
+
+  // Drift above which a player is hard-seeked (audible); smaller drifts are
+  // caught up by adjusting the playback speed (pitch preserving, inaudible).
+  private static final long HARD_RESYNC_US = 500000;
+
+  // Speed adjustment hysteresis: starts above 60ms of drift, stops under 20ms.
+  private static final long DRIFT_ADJUST_START_US = 60000;
+  private static final long DRIFT_ADJUST_STOP_US = 20000;
+
+  private static final float CATCH_UP_SPEED = 1.05f;
+  private static final float SLOW_DOWN_SPEED = 0.95f;
+
+  // Minimum delay between two hard resyncs of the same item; without it a
+  // player whose seek latency exceeds the threshold re-seeks on every tick.
+  private static final long RESYNC_COOLDOWN_MS = 1000;
+
+  // Window after a start during which the observed drift is attributed to
+  // the audio pipeline startup latency and learned as the seek lead.
+  private static final long START_LEARN_WINDOW_MS = 2000;
+
+  private static final long MAX_SEEK_LEAD_US = 1000000;
+
+  private final VideoComposition composition;
+
+  private final List<ItemPlayer> itemPlayers = new ArrayList<>();
+
+  private OnErrorListener onErrorListener;
+
+  private boolean released = false;
+
+  public AudioCompositionPlayer(VideoComposition composition) {
+    this.composition = composition;
+  }
+
+  public void setOnErrorListener(OnErrorListener onErrorListener) {
+    this.onErrorListener = onErrorListener;
+  }
+
+  public void prepare() {
+    if (released || !itemPlayers.isEmpty()) return;
+    try {
+      for (VideoComposition.Item item : composition.getItems()) {
+        if (!item.isAudioEnabled()) continue;
+        ItemPlayer player = new ItemPlayer(item, this::dispatchError, composition.isLazyDecoders());
+        // Register ownership before open(): an ExoPlayer created before a later
+        // prepare failure must still be released.
+        itemPlayers.add(player);
+        if (!composition.isLazyDecoders()) player.open();
+      }
+    } catch (RuntimeException | Error error) {
+      release();
+      throw error;
+    }
+  }
+
+  private void dispatchError(String message) {
+    if (!released && onErrorListener != null) {
+      onErrorListener.onError(message);
+    }
+  }
+
+  /**
+   * Synchronizes the item players with the composition clock.
+   */
+  public void update(long positionUs, boolean isPlaying) {
+    if (released) {
+      return;
+    }
+    for (ItemPlayer itemPlayer : itemPlayers) {
+      itemPlayer.update(positionUs, isPlaying);
+    }
+  }
+
+  public void seekTo(long positionUs) {
+    if (released) {
+      return;
+    }
+    for (ItemPlayer itemPlayer : itemPlayers) {
+      itemPlayer.seekTo(positionUs);
+    }
+  }
+
+  public void pause() {
+    if (released) {
+      return;
+    }
+    for (ItemPlayer itemPlayer : itemPlayers) {
+      itemPlayer.pause();
+    }
+  }
+
+  public void release() {
+    if (released) {
+      return;
+    }
+    released = true;
+    for (ItemPlayer itemPlayer : itemPlayers) {
+      itemPlayer.release();
+    }
+    itemPlayers.clear();
+  }
+
+  private static class ItemPlayer {
+    private ExoPlayer player;
+    private final VideoComposition.Item item;
+    private final OnErrorListener onErrorListener;
+    private final DecoderWindow window;
+    private long generation;
+    private final long compositionStartTimeUs;
+    private final long durationUs;
+
+    // Learned seek/startup latency: seeking lands the player behind the
+    // composition clock by the time the audio pipeline takes to resume,
+    // leading every seek by the observed drift compensates it.
+    private long seekLeadUs = 0;
+
+    private long lastSyncElapsedMs = -1;
+
+    private long lastStartElapsedMs = -1;
+
+    private float currentSpeed = 1f;
+
+    ItemPlayer(VideoComposition.Item item, OnErrorListener onErrorListener, boolean lazy) {
+      this.item = item;
+      this.onErrorListener = onErrorListener;
+      window = DecoderWindow.of(lazy, true);
+      compositionStartTimeUs = TimeHelpers.secToUs(item.getCompositionStartTime());
+      durationUs = TimeHelpers.secToUs(item.getDuration());
+    }
+
+    private void open() {
+      if (player != null) return;
+      final long openedGeneration = ++generation;
+      player = new ExoPlayer.Builder(
+        ReactNativeSkiaVideoModule.currentReactApplicationContext()
+      ).setLooper(Looper.myLooper()).build();
+
+      player.addListener(new Player.Listener() {
+        private boolean audioTrackChecked = false;
+
+        @Override
+        public void onPlayerError(PlaybackException error) {
+          if (openedGeneration != generation || player == null) return;
+          onErrorListener.onError(error.getMessage());
+        }
+
+        @Override
+        public void onTracksChanged(Tracks tracks) {
+          if (openedGeneration != generation || player == null) return;
+          if (item.isVideo() || audioTrackChecked || tracks.getGroups().isEmpty()) {
+            return;
+          }
+          audioTrackChecked = true;
+          for (Tracks.Group group : tracks.getGroups()) {
+            if (group.getType() == C.TRACK_TYPE_AUDIO) {
+              return;
+            }
+          }
+          onErrorListener.onError("No audio track for path: " + item.getPath());
+        }
+      });
+
+      String path = item.getPath();
+      Uri uri = path.startsWith("/") ? Uri.fromFile(new File(path)) : Uri.parse(path);
+      long startTimeMs = Math.round(item.getStartTime() * 1000);
+      long endTimeMs = startTimeMs + Math.round(item.getDuration() * 1000);
+      MediaItem mediaItem = new MediaItem.Builder()
+        .setUri(uri)
+        .setClippingConfiguration(
+          new MediaItem.ClippingConfiguration.Builder()
+            .setStartPositionMs(startTimeMs)
+            .setEndPositionMs(endTimeMs)
+            .build()
+        )
+        .build();
+      player.setMediaItem(mediaItem);
+      // Disabling video ensures a video file used as an audio source never
+      // instantiates a video decoder.
+      player.setTrackSelectionParameters(
+        player.getTrackSelectionParameters().buildUpon()
+          .setTrackTypeDisabled(C.TRACK_TYPE_VIDEO, true)
+          .build()
+      );
+      player.setVolume((float) Math.max(0, Math.min(1, item.getAudioVolume())));
+      player.setPlayWhenReady(false);
+      player.prepare();
+    }
+
+    void update(long positionUs, boolean isPlaying) {
+      if (player == null && window.opens(compositionStartTimeUs,
+          compositionStartTimeUs + durationUs, positionUs)) {
+        open();
+        syncTo(Math.max(0, positionUs - compositionStartTimeUs));
+      } else if (player != null && !window.keeps(compositionStartTimeUs,
+          compositionStartTimeUs + durationUs, positionUs)) {
+        release();
+      }
+      if (player == null) return;
+      long localPositionUs = positionUs - compositionStartTimeUs;
+      boolean active =
+        isPlaying && localPositionUs >= 0 && localPositionUs < durationUs;
+      if (!active) {
+        if (player.getPlayWhenReady()) {
+          player.setPlayWhenReady(false);
+        }
+        return;
+      }
+      if (!player.getPlayWhenReady()) {
+        // The player is usually already primed at the target position
+        // (prepare, user seek, loop restart): starting it without seeking
+        // avoids the seek + rebuffering latency at the start of playback.
+        long playerPositionUs = player.getCurrentPosition() * 1000;
+        long targetUs = localPositionUs + seekLeadUs;
+        if (Math.abs(playerPositionUs - targetUs) > DRIFT_ADJUST_START_US) {
+          syncTo(localPositionUs);
+        }
+        setSpeed(1f);
+        player.setPlayWhenReady(true);
+        lastStartElapsedMs = SystemClock.elapsedRealtime();
+        lastSyncElapsedMs = lastStartElapsedMs;
+        return;
+      }
+      if (!player.isPlaying()) {
+        // Still buffering after a seek/start.
+        return;
+      }
+      long playerPositionUs = player.getCurrentPosition() * 1000;
+      long driftUs = localPositionUs - playerPositionUs;
+      long absDriftUs = Math.abs(driftUs);
+      long nowMs = SystemClock.elapsedRealtime();
+
+      if (
+        driftUs > 0
+          && lastStartElapsedMs >= 0
+          && nowMs - lastStartElapsedMs < START_LEARN_WINDOW_MS
+      ) {
+        seekLeadUs = Math.min(Math.max(seekLeadUs, driftUs), MAX_SEEK_LEAD_US);
+      }
+
+      if (absDriftUs > HARD_RESYNC_US) {
+        if (lastSyncElapsedMs < 0 || nowMs - lastSyncElapsedMs >= RESYNC_COOLDOWN_MS) {
+          syncTo(localPositionUs);
+          setSpeed(1f);
+        }
+      } else if (absDriftUs > DRIFT_ADJUST_START_US) {
+        setSpeed(driftUs > 0 ? CATCH_UP_SPEED : SLOW_DOWN_SPEED);
+      } else if (absDriftUs < DRIFT_ADJUST_STOP_US) {
+        setSpeed(1f);
+      }
+    }
+
+    private void setSpeed(float speed) {
+      if (currentSpeed != speed) {
+        currentSpeed = speed;
+        player.setPlaybackParameters(new PlaybackParameters(speed));
+      }
+    }
+
+    void seekTo(long positionUs) {
+      if (player == null && window.opens(compositionStartTimeUs,
+          compositionStartTimeUs + durationUs, positionUs)) open();
+      if (player != null && !window.keeps(compositionStartTimeUs,
+          compositionStartTimeUs + durationUs, positionUs)) release();
+      if (player == null) return;
+      long localPositionUs =
+        Math.max(0, Math.min(positionUs - compositionStartTimeUs, durationUs));
+      syncTo(localPositionUs);
+    }
+
+    private void syncTo(long localPositionUs) {
+      player.seekTo((localPositionUs + seekLeadUs) / 1000);
+      lastSyncElapsedMs = SystemClock.elapsedRealtime();
+    }
+
+    void pause() {
+      if (player != null) player.setPlayWhenReady(false);
+    }
+
+    void release() {
+      if (player != null) {
+        generation++;
+        player.release();
+        player = null;
+        currentSpeed = 1f;
+        seekLeadUs = 0;
+        lastStartElapsedMs = lastSyncElapsedMs = -1;
+      }
+    }
+  }
+}

package/android/src/main/java/com/azzapp/rnskv/BufferLifetime.java

@@ -0,0 +1,55 @@
+package com.azzapp.rnskv;
+
+import java.lang.ref.PhantomReference;
+import java.lang.ref.ReferenceQueue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.LongConsumer;
+
+/** Pre-registered backing owner and exposed JNI root: no cleanup allocation is needed during GC. */
+final class BufferLifetime {
+  private final ReferenceQueue<Object> queue = new ReferenceQueue<>();
+  private final Set<Tracked> references = ConcurrentHashMap.newKeySet();
+  private final LongConsumer release;
+
+  BufferLifetime(LongConsumer release) { this.release = release; }
+
+  Tracked track(Object exposed, Object backing, long token) {
+    Tracked tracked = new Tracked(exposed, backing, token);
+    references.add(tracked);
+    return tracked;
+  }
+
+  void drain() {
+    Tracked reference;
+    while ((reference = (Tracked) queue.poll()) != null) process(reference);
+  }
+
+  void run() {
+    for (;;) {
+      try { process((Tracked) queue.remove()); }
+      catch (InterruptedException ignored) { /* Process-wide daemon has no application owner. */ }
+    }
+  }
+
+  private void process(Tracked reference) {
+    if (!references.remove(reference)) return;
+    reference.backing = null;
+    reference.clear();
+    if (reference.token != 0) release.accept(reference.token);
+  }
+
+  int trackedCount() { return references.size(); }
+  // Only package tests use this to enqueue a second-phase reference without relying on GC timing.
+  Tracked onlyTracked() { return references.iterator().next(); }
+
+  final class Tracked extends PhantomReference<Object> {
+    private Object backing;
+    private final long token;
+    Tracked(Object referent, Object backing, long token) {
+      super(referent, queue);
+      this.backing = backing;
+      this.token = token;
+    }
+  }
+}

package/android/src/main/java/com/azzapp/rnskv/DecodeRequests.java

@@ -0,0 +1,100 @@
+package com.azzapp.rnskv;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
+
+/** One synchronous decoder request at a time; errors also persist between requests. */
+final class DecodeRequests<T> {
+  private CompletableFuture<T> pending;
+  private T completedResult;
+  private boolean hasCompletedResult;
+  private final Consumer<T> discard;
+  private Exception failure;
+  private boolean closed;
+
+  DecodeRequests() { this(result -> {}); }
+
+  DecodeRequests(Consumer<T> discard) { this.discard = discard; }
+
+  synchronized CompletableFuture<T> begin() throws Exception {
+    if (closed) throw new IllegalStateException("Video decoder has been released");
+    if (failure != null) throw failure;
+    if (pending != null) {
+      throw new IllegalStateException("A video decode request is already pending");
+    }
+    pending = new CompletableFuture<>();
+    return pending;
+  }
+
+  synchronized void fail(Exception error) {
+    if (closed) return;
+    if (failure == null) failure = error;
+    if (pending != null) pending.completeExceptionally(failure);
+    discardCompleted();
+  }
+
+  synchronized void complete(T result) {
+    if (!closed && failure == null && pending != null && !pending.isDone()) {
+      completedResult = result;
+      hasCompletedResult = true;
+      // Set ownership before waking waiters, including a synchronous completion
+      // callback which may re-enter await on this same monitor.
+      if (!pending.complete(result)) discardCompleted();
+    } else {
+      // Results racing cancellation, duplicate completions, or an absent
+      // request never reach the consumer and must release their own aliases.
+      discard.accept(result);
+    }
+  }
+
+  synchronized boolean isPending() {
+    return !closed && failure == null && pending != null && !pending.isDone();
+  }
+
+  synchronized boolean isClosed() { return closed; }
+
+  synchronized void close() {
+    closed = true;
+    if (pending != null) pending.cancel(true);
+    discardCompleted();
+    pending = null;
+  }
+
+  private void discardCompleted() {
+    if (!hasCompletedResult) return;
+    T discarded = completedResult;
+    completedResult = null;
+    hasCompletedResult = false;
+    discard.accept(discarded);
+  }
+
+  T await(CompletableFuture<T> request, long timeoutMs) throws Exception {
+    try {
+      T result = request.get(timeoutMs, TimeUnit.MILLISECONDS);
+      synchronized (this) {
+        // close/fail may run between Future.get and this ownership transfer.
+        // Such a result has already been discarded and must not escape.
+        if (closed) throw new CancellationException("Video decoder has been released");
+        if (failure != null) throw new ExecutionException(failure);
+        if (pending != request || !hasCompletedResult)
+          throw new IllegalStateException("Video decode result was already consumed");
+        completedResult = null;
+        hasCompletedResult = false;
+        pending = null;
+        return result;
+      }
+    } catch (TimeoutException error) {
+      fail(error);
+      close();
+      throw error;
+    } catch (InterruptedException error) {
+      Thread.currentThread().interrupt();
+      close();
+      throw error;
+    }
+  }
+}

package/android/src/main/java/com/azzapp/rnskv/DecoderWindow.java

@@ -0,0 +1,55 @@
+package com.azzapp.rnskv;
+
+/**
+ * When an item's decoder is open, for a composition with lazy decoders: from a
+ * little before the item starts to a little after it ends. Without lazy
+ * decoders every decoder is open for the life of the player.
+ *
+ * Unlike iOS, no read-ahead over a loop's wrap: the wrap seeks every decoder
+ * back to 0, so one opened early would only race through its item meanwhile.
+ */
+final class DecoderWindow {
+
+  // The preview has to open a codec and decode to the first frame before the
+  // item is on screen; the export waits for it anyway. Chosen, not measured.
+  private static final long PREVIEW_LEAD_US = 1_500_000;
+  private static final long EXPORT_LEAD_US = 500_000;
+
+  // Kept a little past both ends, so a scrub across a cut does not reopen it.
+  private static final long MARGIN_US = 500_000;
+
+  private final boolean lazy;
+
+  private final long leadUs;
+
+  private DecoderWindow(boolean lazy, long leadUs) {
+    this.lazy = lazy;
+    this.leadUs = leadUs;
+  }
+
+  static DecoderWindow of(boolean lazy, boolean realTime) {
+    return new DecoderWindow(lazy, realTime ? PREVIEW_LEAD_US : EXPORT_LEAD_US);
+  }
+
+  boolean isLazy() {
+    return lazy;
+  }
+
+  /**
+   * @param startUs    the item's start in the composition
+   * @param endUs      the item's end in the composition
+   * @param positionUs the composition time
+   */
+  boolean opens(long startUs, long endUs, long positionUs) {
+    return !lazy || (positionUs >= startUs - leadUs && positionUs < endUs);
+  }
+
+  /**
+   * Wider than {@link #opens}, so an item is never closed and reopened on
+   * either side of the same edge.
+   */
+  boolean keeps(long startUs, long endUs, long positionUs) {
+    return !lazy
+      || (positionUs >= startUs - leadUs - MARGIN_US && positionUs < endUs + MARGIN_US);
+  }
+}

package/android/src/main/java/com/azzapp/rnskv/DisplayVsyncPeriod.java

@@ -0,0 +1,72 @@
+package com.azzapp.rnskv;
+
+import android.content.Context;
+import android.hardware.display.DisplayManager;
+import android.view.Display;
+
+/**
+ * The vsync period of a display, kept current as its refresh rate changes.
+ *
+ * Taken from the display rather than from the gaps between vsyncs: those
+ * cannot tell a slower refresh rate from missed vsyncs, and an adaptive
+ * display dropping from 90 or 120 Hz to 60 Hz left such an estimate at the
+ * old rate.
+ */
+final class DisplayVsyncPeriod implements DisplayManager.DisplayListener {
+  private final DisplayManager displayManager;
+  private final Display display;
+  private volatile long periodNanos;
+
+  private DisplayVsyncPeriod(DisplayManager displayManager, Display display) {
+    this.displayManager = displayManager;
+    this.display = display;
+    update();
+  }
+
+  /**
+   * Follows the display's refresh rate changes on the calling thread's looper.
+   *
+   * @param display the display to follow, null for the default one
+   * @return null when the display cannot be followed
+   */
+  static DisplayVsyncPeriod watch(Context context, Display display) {
+    DisplayManager displayManager = context.getSystemService(DisplayManager.class);
+    if (displayManager == null) return null;
+    if (display == null) display = displayManager.getDisplay(Display.DEFAULT_DISPLAY);
+    if (display == null) return null;
+    DisplayVsyncPeriod period = new DisplayVsyncPeriod(displayManager, display);
+    displayManager.registerDisplayListener(period, null);
+    return period;
+  }
+
+  /**
+   * @return the display's vsync period in nanoseconds, 0 when unknown
+   */
+  long getPeriodNanos() {
+    return periodNanos;
+  }
+
+  void release() {
+    displayManager.unregisterDisplayListener(this);
+  }
+
+  private void update() {
+    float rate = display.getRefreshRate();
+    periodNanos = rate > 0 ? (long) (1_000_000_000L / rate) : 0;
+  }
+
+  @Override
+  public void onDisplayAdded(int displayId) {
+  }
+
+  @Override
+  public void onDisplayRemoved(int displayId) {
+  }
+
+  @Override
+  public void onDisplayChanged(int displayId) {
+    if (displayId == display.getDisplayId()) {
+      update();
+    }
+  }
+}