react-native-skia-webgpu-video 0.1.0-beta.2 → 0.1.0-beta.4

package/android/cpp/VideoEncoderHostObject.cpp

@@ -1,49 +1,36 @@
 #include "RNSVCheckedSizes.h"
 #include "VideoEncoderHostObject.h"
-#include "NativeHardwareBuffer.h"
-#include <EGL/egl.h>
+#include <android/data_space.h>
+#include <android/native_window_jni.h>
+#include <dlfcn.h>
 #include <fbjni/ByteBuffer.h>
 #include <cmath>
 #include <limits>
-#include <unistd.h>
 
 namespace RNSkiaVideo {
 using namespace facebook::jni;
 
 namespace {
-/** Makes the encoder's EGL context current; restores the caller's (or none). */
-class ScopedEncoderContext {
-public:
-  explicit ScopedEncoderContext(const global_ref<VideoEncoder>& encoder)
-      : display(eglGetCurrentDisplay()), context(eglGetCurrentContext()),
-        draw(eglGetCurrentSurface(EGL_DRAW)), read(eglGetCurrentSurface(EGL_READ)) {
-    encoder->makeGLContextCurrent();
-  }
-  ~ScopedEncoderContext() {
-    if (context != EGL_NO_CONTEXT) {
-      eglMakeCurrent(display, draw, read, context);
-    } else if (auto current = eglGetCurrentDisplay(); current != EGL_NO_DISPLAY) {
-      eglMakeCurrent(current, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
-    }
-  }
-  ScopedEncoderContext(const ScopedEncoderContext&) = delete;
-  ScopedEncoderContext& operator=(const ScopedEncoderContext&) = delete;
-
-private:
-  EGLDisplay display;
-  EGLContext context;
-  EGLSurface draw;
-  EGLSurface read;
+struct JSurface : JavaClass<JSurface> {
+  static constexpr auto kJavaDescriptor = "Landroid/view/Surface;";
 };
 
-/** Sync-fds handed over by JS; each is closed exactly once. */
-struct OwnedFences {
-  std::vector<int> fds;
-  ~OwnedFences() {
-    for (int fd : fds)
-      if (fd >= 0) close(fd);
-  }
-};
+/**
+ * Sets the presentation time of the next buffer queued to a window, as
+ * eglPresentationTimeANDROID does: the codec reads it as the frame's time.
+ * libnativewindow exports it on Android 10 and later, though the NDK does not
+ * declare it (it is in the LL-NDK); null where it is missing.
+ */
+using SetBuffersTimestamp = int32_t (*)(ANativeWindow*, int64_t);
+SetBuffersTimestamp setBuffersTimestamp() {
+  static const auto function = []() -> SetBuffersTimestamp {
+    void* library = dlopen("libnativewindow.so", RTLD_NOW);
+    return library == nullptr ? nullptr
+                              : reinterpret_cast<SetBuffersTimestamp>(dlsym(
+                                    library, "ANativeWindow_setBuffersTimestamp"));
+  }();
+  return function;
+}
 } // namespace
 
 local_ref<VideoEncoder>
@@ -79,12 +66,6 @@ void VideoEncoder::prepare() const {
   prepareMethod(self());
 }
 
-void VideoEncoder::makeGLContextCurrent() const {
-  static const auto makeGLContextCurrentMethod =
-      getClass()->getMethod<void()>("makeGLContextCurrent");
-  makeGLContextCurrentMethod(self());
-}
-
 void VideoEncoder::encodeFrameRgba(alias_ref<JByteBuffer> pixels, jint width,
                                    jint height, jint bytesPerRow, jdouble time) const {
   static const auto method =
@@ -92,9 +73,21 @@ void VideoEncoder::encodeFrameRgba(alias_ref<JByteBuffer> pixels, jint width,
   method(self(), pixels, width, height, bytesPerRow, time);
 }
 
-void VideoEncoder::encodeTexture(jint texture, jdouble time) const {
-  static const auto method = getClass()->getMethod<void(jint, jdouble)>("encodeTexture");
-  method(self(), texture, time);
+local_ref<jobject> VideoEncoder::getInputSurface() const {
+  static const auto method =
+      getClass()->getMethod<JSurface::javaobject()>("getInputSurface");
+  auto surface = method(self());
+  return make_local(static_cast<jobject>(surface.get()));
+}
+
+jlong VideoEncoder::beginWindowFrame(jdouble time) const {
+  static const auto method = getClass()->getMethod<jlong(jdouble)>("beginWindowFrame");
+  return method(self(), time);
+}
+
+void VideoEncoder::endWindowFrame(jlong timeUs) const {
+  static const auto method = getClass()->getMethod<void(jlong)>("endWindowFrame");
+  method(self(), timeUs);
 }
 
 void VideoEncoder::release() const {
@@ -112,11 +105,11 @@ VideoEncoderHostObject::VideoEncoderHostObject(
     std::string& outPath, int width, int height, int frameRate, int bitRate,
     std::optional<std::string> encoderName, std::optional<std::string> codec,
     alias_ref<VideoComposition> composition, int audioSampleRate,
-    int audioChannelCount, int audioBitRate)
-    : width(width), height(height) {
+    int audioChannelCount, int audioBitRate) {
   framesExtractor = make_global(VideoEncoder::create(
       outPath, width, height, frameRate, bitRate, encoderName, codec,
       composition, audioSampleRate, audioChannelCount, audioBitRate));
+  hints = std::make_unique<ExportHintSession>(frameRate);
 }
 
 VideoEncoderHostObject::~VideoEncoderHostObject() {
@@ -128,8 +121,7 @@ VideoEncoderHostObject::getPropertyNames(jsi::Runtime& rt) {
   std::vector<jsi::PropNameID> result;
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("prepare")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("encodeFrame")));
-  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("acquireFrameBuffer")));
-  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("releaseFrameBuffer")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("acquireInputWindow")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("finishWriting")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("dispose")));
   return result;
@@ -155,39 +147,31 @@ jsi::Value VideoEncoderHostObject::get(jsi::Runtime& runtime,
           auto frame = arguments[0].asObject(runtime);
           const auto kindValue = frame.getProperty(runtime, "kind");
           const auto kind = kindValue.isString() ? kindValue.asString(runtime).utf8(runtime) : "";
-          if (kind == "native-buffer") {
-            // A lent buffer Skia rendered into; its sync-fds are ours to close.
-            OwnedFences fences;
-            const auto fenceValues = frame.getProperty(runtime, "fences");
-            if (fenceValues.isObject()) {
-              auto array = fenceValues.asObject(runtime).asArray(runtime);
-              for (size_t i = 0; i < array.size(runtime); ++i) {
-                auto value = array.getValueAtIndex(runtime, i);
-                if (!value.isBigInt()) throw jsi::JSError(runtime, "Encoder fences must be sync-fd BigInts");
-                // WebGPU reports an already signaled fence as -1, as a uint32.
-                fences.fds.push_back(static_cast<int>(
-                    static_cast<uint32_t>(value.asBigInt(runtime).getUint64(runtime))));
-              }
-            }
-            const auto pointerValue = frame.getProperty(runtime, "nativeBuffer");
-            if (!pointerValue.isBigInt() || !targets)
-              throw jsi::JSError(runtime, "Encoder frame buffer is not the one lent");
-            const auto* buffer = reinterpret_cast<const AHardwareBuffer*>(
-                static_cast<uintptr_t>(pointerValue.asBigInt(runtime).getUint64(runtime)));
+          if (kind == "window") {
+            // The GPU export rendered the frame into the canvas context of the
+            // input window: `present` queues it, with the time set here.
+            if (window == nullptr)
+              throw jsi::JSError(runtime, "Encoder input window was not acquired");
+            auto present = frame.getProperty(runtime, "present");
+            if (!present.isObject() || !present.asObject(runtime).isFunction(runtime))
+              throw jsi::JSError(runtime, "A window frame needs its present function");
             const auto time = checkedMediaSeconds(arguments[1].asNumber());
-            ScopedEncoderContext scope(framesExtractor);
-            const auto texture = targets->beginEncode(buffer, fences.fds);
-            try {
-              framesExtractor->encodeTexture(static_cast<jint>(texture), time);
-            } catch (...) {
-              targets->finishEncode();
-              throw;
-            }
-            targets->finishEncode();
+            const auto timeUs = framesExtractor->beginWindowFrame(time);
+            // The swapchain tags its buffers sRGB, full range, and the codec
+            // would encode and label the file so. Unknown, as EGL leaves it,
+            // lets the codec use its default: Rec.709, limited range. Set for
+            // every frame, as a recreated swapchain tags the window again.
+            if (ANativeWindow_setBuffersDataSpace(window, ADATASPACE_UNKNOWN) != 0)
+              throw jsi::JSError(runtime, "Cannot set the encoder frame dataspace");
+            if (setBuffersTimestamp()(window, static_cast<int64_t>(timeUs) * 1000) != 0)
+              throw jsi::JSError(runtime, "Cannot set the encoder frame time");
+            present.asObject(runtime).asFunction(runtime).call(runtime);
+            framesExtractor->endWindowFrame(timeUs);
+            hints->frameDone();
             return jsi::Value::undefined();
           }
           if (kind != "rgba") {
-            throw jsi::JSError(runtime, "Android encoder accepts RGBA pixels or a lent hardware buffer");
+            throw jsi::JSError(runtime, "Android encoder accepts RGBA pixels or a window frame");
           }
           const auto checkedInteger = [&](const char* name) -> int {
             const auto value = frame.getProperty(runtime, name).asNumber();
@@ -234,7 +218,7 @@ jsi::Value VideoEncoderHostObject::get(jsi::Runtime& runtime,
           framesExtractor->encodeFrameRgba(javaPixels, width, height, stride, checkedMediaSeconds(arguments[1].asNumber()));
           return jsi::Value::undefined();
         });
-  } else if (propName == "acquireFrameBuffer") {
+  } else if (propName == "acquireInputWindow") {
     return getFunction(
         runtime, propName, 0,
         [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
@@ -243,22 +227,17 @@ jsi::Value VideoEncoderHostObject::get(jsi::Runtime& runtime,
           if (released.test() || !prepared || finished) {
             throw jsi::JSError(runtime, "Encoder not prepared");
           }
-          // The buffers are imported by the same Dawn device as decoded AHBs.
-          if (!NativeHardwareBuffer::isEnabled()) {
-            throw jsi::JSError(runtime, "Android GPU export requires AHardwareBuffer interop");
+          // Without it, every frame would carry the time it was queued at.
+          if (setBuffersTimestamp() == nullptr) {
+            throw jsi::JSError(runtime, "Encoder frame times need Android 10 or later");
           }
-          if (!targets) targets = std::make_unique<EncoderTargets>(width, height);
-          auto* buffer = targets->lend();
-          return jsi::BigInt::fromUint64(runtime, reinterpret_cast<uintptr_t>(buffer));
-        });
-  } else if (propName == "releaseFrameBuffer") {
-    return getFunction(
-        runtime, propName, 0,
-        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
-               const jsi::Value* arguments, size_t count) -> jsi::Value {
-          std::lock_guard<std::recursive_mutex> lock(encoderMutex);
-          if (targets) targets->giveBack();
-          return jsi::Value::undefined();
+          if (window == nullptr) {
+            auto surface = framesExtractor->getInputSurface();
+            window = ANativeWindow_fromSurface(Environment::current(), surface.get());
+            if (window == nullptr)
+              throw jsi::JSError(runtime, "Cannot get the encoder input window");
+          }
+          return jsi::BigInt::fromUint64(runtime, reinterpret_cast<uintptr_t>(window));
         });
   } else if (propName == "prepare") {
     return getFunction(
@@ -300,16 +279,12 @@ jsi::Value VideoEncoderHostObject::get(jsi::Runtime& runtime,
 void VideoEncoderHostObject::release() {
   std::lock_guard<std::recursive_mutex> lock(encoderMutex);
   if (!released.test_and_set()) {
-    if (targets && !targets->empty()) {
-      // GL names belong to the encoder context; free them before it goes.
-      // Otherwise the destructor still releases buffers and reservations.
-      try {
-        ScopedEncoderContext scope(framesExtractor);
-        targets->clear();
-      } catch (...) {
-      }
+    // A canvas context presenting to the window holds its own reference.
+    if (window != nullptr) {
+      ANativeWindow_release(window);
+      window = nullptr;
     }
-    targets.reset();
+    hints.reset();
     framesExtractor->release();
     framesExtractor = nullptr;
   }

package/android/cpp/VideoEncoderHostObject.h

@@ -1,8 +1,9 @@
 #pragma once
 
-#include "EncoderTargets.h"
+#include "ExportHintSession.h"
 #include "RNSVHostObject.h"
 #include "VideoComposition.h"
+#include <android/native_window.h>
 #include <memory>
 #include <mutex>
 #include <fbjni/ByteBuffer.h>
@@ -34,13 +35,17 @@ public:
 
   void prepare() const;
 
-  void makeGLContextCurrent() const;
-
   void encodeFrameRgba(alias_ref<JByteBuffer> pixels, jint width, jint height,
                        jint bytesPerRow, jdouble time) const;
 
-  /** A GL_TEXTURE_2D of the encoder's context, e.g. an imported buffer. */
-  void encodeTexture(jint texture, jdouble time) const;
+  /** The codec's input Surface (an android.view.Surface). */
+  local_ref<jobject> getInputSurface() const;
+
+  /** Before a frame is presented to the input Surface: its time in µs. */
+  jlong beginWindowFrame(jdouble time) const;
+
+  /** Once the frame of beginWindowFrame was presented. */
+  void endWindowFrame(jlong timeUs) const;
 
   void finishWriting() const;
 
@@ -62,10 +67,10 @@ public:
 
 private:
   global_ref<VideoEncoder> framesExtractor;
-  // GPU export: buffers Skia renders into, sampled by the encoder's context.
-  std::unique_ptr<EncoderTargets> targets;
-  int width;
-  int height;
+  // GPU export: the codec's input Surface, which WebGPU presents to.
+  ANativeWindow* window = nullptr;
+  // GPU export: the system's hints, per presented frame.
+  std::unique_ptr<ExportHintSession> hints;
   std::recursive_mutex encoderMutex;
   bool prepared = false;
   bool finished = false;

package/android/cpp/VideoFrame.cpp

@@ -138,13 +138,11 @@ jsi::Value VideoFrame::toJSInternal(jsi::Runtime& runtime) {
   if (isHardwareBuffer()) {
     auto owner = acquireHardwareFrame(getNativeHardwareBufferHandle());
     // An ImageReader frame is the codec's YUV buffer: its picture is the crop
-    // rectangle, turned for display when WebGPU samples it.
+    // rectangle, turned for display and scaled to the frame's size when
+    // WebGPU converts it.
     const bool decodedImage = owner->image != nullptr;
-    const bool swapped = owner->rotation == 90 || owner->rotation == 270;
     const bool valid = decodedImage
-        ? owner->description.format != AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM &&
-          (swapped ? owner->cropHeight : owner->cropWidth) == width &&
-          (swapped ? owner->cropWidth : owner->cropHeight) == height
+        ? owner->description.format != AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM
         : owner->description.format == AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM &&
           owner->description.width == static_cast<uint32_t>(width) &&
           owner->description.height == static_cast<uint32_t>(height);

package/android/cpp/cpp-adapter.cpp

@@ -38,12 +38,16 @@ void install(jsi::Runtime& jsiRuntime) {
         const bool hardware = NativeHardwareBuffer::isEnabled();
         info.setProperty(runtime, "decodeTransport", jsi::String::createFromAscii(runtime,
             hardware ? "ahardwarebuffer-rgba-egl" : "cpu-rgba-readback"));
-        // Export renders into encoder-lent AHardwareBuffers, sampled by the
-        // encoder's EGL context; CPU readback stays the fallback.
+        // Export renders straight into the codec's input Surface through a
+        // WebGPU canvas context; the JS export probes it at setup, and CPU
+        // readback stays the fallback. Offered with AHardwareBuffer interop,
+        // which also samples decoded frames in place: turning interop off
+        // selects the readback export, the reference of the pixel checks.
         info.setProperty(runtime, "encodeTransport", jsi::String::createFromAscii(runtime,
-            hardware ? "ahardwarebuffer-egl-sample" : "cpu-rgba-upload"));
+            hardware ? "webgpu-native-window" : "cpu-rgba-upload"));
         info.setProperty(runtime, "exportFallback", jsi::String::createFromAscii(runtime, "cpu-rgba-upload"));
         info.setProperty(runtime, "gpuDirectExport", hardware);
+        info.setProperty(runtime, "zeroCopyEncode", hardware);
         info.setProperty(runtime, "decodeCpuCopiesBeforeSkia", hardware ? 0 : 1);
         info.setProperty(runtime, "cpuReadbackDecode", !hardware);
         info.setProperty(runtime, "ownedFrameTransport", jsi::String::createFromAscii(runtime,
@@ -51,7 +55,6 @@ void install(jsi::Runtime& jsiRuntime) {
         info.setProperty(runtime, "skiaImportTransport", jsi::String::createFromAscii(runtime,
             hardware ? "native-buffer-webgpu-blit-snapshot" : "cpu-rgba-scoped-copy-webgpu-upload-snapshot"));
         info.setProperty(runtime, "zeroCopyDecode", false);
-        info.setProperty(runtime, "zeroCopyEncode", false);
         return info;
       }));
   auto createVideoPlayer = jsi::Function::createFromHostFunction(

package/android/src/main/java/com/azzapp/rnskv/GLFrameExtractor.java

@@ -126,8 +126,8 @@ public class GLFrameExtractor implements SurfaceTexture.OnFrameAvailableListener
   /**
    * The Surface the codec renders into. With {@code nativeImages}, on API 29+
    * with hardware buffers, it is an ImageReader's: frames then keep the codec's
-   * YUV buffer, cropped and turned by {@code rotation} when WebGPU samples it,
-   * so they must be wanted at the encoded size.
+   * YUV buffer, cropped, turned by {@code rotation} and scaled to the frame's
+   * size when WebGPU converts it.
    */
   @android.annotation.TargetApi(Build.VERSION_CODES.Q)
   public Surface surfaceFor(int videoWidth, int videoHeight, int rotation, boolean nativeImages) {
@@ -163,13 +163,7 @@ public class GLFrameExtractor implements SurfaceTexture.OnFrameAvailableListener
     try {
       if (Build.VERSION.SDK_INT >= 33) awaitProducer(image);
       Rect crop = image.getCropRect();
-      boolean swapped = imageRotation == 90 || imageRotation == 270;
-      int pictureWidth = swapped ? crop.height() : crop.width();
-      int pictureHeight = swapped ? crop.width() : crop.height();
-      if (pictureWidth != width || pictureHeight != height) {
-        throw new IllegalStateException("Decoded image " + pictureWidth + "x" + pictureHeight
-          + " does not match the requested " + width + "x" + height);
-      }
+      if (crop.isEmpty()) throw new IllegalStateException("Decoded image has an empty crop");
       HardwareBuffer buffer = image.getHardwareBuffer();
       if (buffer == null) throw new IllegalStateException("Decoded image has no hardware buffer");
       long lease;

package/android/src/main/java/com/azzapp/rnskv/PreviewFrameRateVote.java

@@ -0,0 +1,104 @@
+package com.azzapp.rnskv;
+
+import android.app.Activity;
+import android.os.Build;
+import android.os.SystemClock;
+import android.util.Log;
+import android.view.Display;
+import android.view.View;
+import android.view.ViewGroup;
+
+import com.facebook.react.bridge.ReactApplicationContext;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Asks the display for its highest refresh rate while a preview plays, through
+ * the React Native Skia canvases of the activity (Android 15+). Main thread.
+ *
+ * Android picks the refresh rate from what the app presents. A preview that
+ * presents at 30 fps for a while (30 fps frames drawn as they come) brings a
+ * 60 Hz display down to 30 Hz, and it stays there: the next clip, at 60 or
+ * 24 fps, can only present at 30 fps and never shows the display it needs
+ * more. On a Pixel 8a, one 60 fps frame in two was never shown, and 24 fps
+ * came out irregular on a fifth of its frames. A vote tells the display.
+ *
+ * Only views that draw cast their vote: one on the window's root view is never
+ * counted. The player does not know which canvas shows it, so all of the Skia
+ * TextureView canvases vote; those that do not draw cast nothing. SurfaceView
+ * canvases vote for themselves, and the vote asks what theirs asks: the highest
+ * rate a seamless switch reaches, 120 Hz where the user allows it, so that
+ * what drawFrame animates over the video is not held to 60 Hz. Not the clips'
+ * rate: a 24 fps vote brought the display to 30 Hz, where 24 fps judders.
+ */
+final class PreviewFrameRateVote {
+  private static final String TAG = "PreviewFrameRateVote";
+
+  private static final String SKIA_TEXTURE_VIEW = "com.reactnative.skia.SkiaTextureView";
+
+  // Canvases mounted while playing get the vote at the next refresh.
+  private static final long REFRESH_MILLIS = 1000;
+
+  private static final Set<PreviewFrameRateVote> voting = new HashSet<>();
+
+  private static long lastAppliedMillis = 0;
+
+  /** Whether the player is playing, at each vsync it collects frames on. */
+  void update(boolean playing) {
+    if (Build.VERSION.SDK_INT < Build.VERSION_CODES.VANILLA_ICE_CREAM) {
+      return;
+    }
+    if (playing ? voting.add(this) : voting.remove(this)) {
+      apply();
+    } else if (playing && SystemClock.uptimeMillis() - lastAppliedMillis >= REFRESH_MILLIS) {
+      apply();
+    }
+  }
+
+  /** The player is released. */
+  void withdraw() {
+    update(false);
+  }
+
+  private static void apply() {
+    lastAppliedMillis = SystemClock.uptimeMillis();
+    ReactApplicationContext context = ReactNativeSkiaVideoModule.currentReactApplicationContext();
+    Activity activity = context != null ? context.getCurrentActivity() : null;
+    if (activity == null || activity.getWindow() == null) {
+      return;
+    }
+    float rate = voting.isEmpty() ? View.REQUESTED_FRAME_RATE_CATEGORY_DEFAULT
+      : maxRefreshRate(activity.getWindow().getDecorView().getDisplay());
+    try {
+      vote(activity.getWindow().getDecorView(), rate);
+    } catch (RuntimeException e) {
+      Log.w(TAG, "Could not vote the refresh rate, leaving it to the display", e);
+    }
+  }
+
+  // The highest rate a seamless switch from the current mode reaches.
+  private static float maxRefreshRate(Display display) {
+    if (display == null) {
+      return View.REQUESTED_FRAME_RATE_CATEGORY_HIGH;
+    }
+    Display.Mode mode = display.getMode();
+    float rate = mode.getRefreshRate();
+    for (float alternative : mode.getAlternativeRefreshRates()) {
+      rate = Math.max(rate, alternative);
+    }
+    return rate;
+  }
+
+  private static void vote(View view, float rate) {
+    if (SKIA_TEXTURE_VIEW.equals(view.getClass().getName())) {
+      view.setRequestedFrameRate(rate);
+    }
+    if (view instanceof ViewGroup) {
+      ViewGroup group = (ViewGroup) view;
+      for (int i = 0; i < group.getChildCount(); i++) {
+        vote(group.getChildAt(i), rate);
+      }
+    }
+  }
+}

package/android/src/main/java/com/azzapp/rnskv/VideoComposition.java

@@ -14,6 +14,10 @@ public class VideoComposition {
   private boolean lazyDecoders = false;
   private boolean hardwareBufferEnabled = false;
   public boolean isHardwareBufferEnabled() { return hardwareBufferEnabled; }
+  // framePixelFormat: 'rgba': every frame through the GL path, none through
+  // ImageReader's YUV buffers. Set from the C++ side.
+  private boolean rgbaFrames = false;
+  public boolean isRgbaFrames() { return rgbaFrames; }
 
   public VideoComposition(
     double duration,

package/android/src/main/java/com/azzapp/rnskv/VideoCompositionDecoder.java

@@ -676,7 +676,7 @@ public class VideoCompositionDecoder {
     try {
       decoder.prepare();
       decoder.setSurface(slot.extractor.surfaceFor(decoder.getVideoWidth(),
-        decoder.getVideoHeight(), decoder.getRotation(), wantsNativeImages(item, decoder)));
+        decoder.getVideoHeight(), decoder.getRotation(), wantsNativeImages(decoder)));
     } catch (IOException | RuntimeException e) {
       decoder.release();
       throw e;
@@ -685,15 +685,14 @@ public class VideoCompositionDecoder {
   }
 
   /**
-   * Hardware frames at the encoded size from an 8-bit SDR track skip the GL
-   * render pass: the codec writes into ImageReader buffers WebGPU samples.
-   * A capped or explicit size keeps the GL path, which scales while drawing.
+   * Hardware frames from an 8-bit SDR track skip the GL render pass: the
+   * codec writes into ImageReader buffers, which WebGPU converts at the
+   * frame's size, a capped or explicit one included. `framePixelFormat:
+   * 'rgba'` keeps the GL path.
    */
-  private boolean wantsNativeImages(VideoComposition.Item item, VideoCompositionItemDecoder decoder) {
-    if (!composition.isHardwareBufferEnabled() || !decoder.isEightBitSdr()) return false;
-    int[] size = FrameSize.of(item.getWidth(), item.getHeight(), item.getMaxLongSide(),
-      decoder.getVideoWidth(), decoder.getVideoHeight(), 0);
-    return size[0] == decoder.getVideoWidth() && size[1] == decoder.getVideoHeight();
+  private boolean wantsNativeImages(VideoCompositionItemDecoder decoder) {
+    return !composition.isRgbaFrames() && composition.isHardwareBufferEnabled()
+      && decoder.isEightBitSdr();
   }
 
   private void reportOpenError(Exception e) {

package/android/src/main/java/com/azzapp/rnskv/VideoCompositionFramesExtractor.java

@@ -58,6 +58,8 @@ public class VideoCompositionFramesExtractor {
   private volatile boolean isPlaying = false;
 
   private volatile long startTime = 0;
+  // The position playback last started or was sought to while playing.
+  private volatile long playStartPosition = 0;
   private volatile long pausePosition = 0;
   private boolean isEOS = false;
   private boolean completeDispatched = false;
@@ -72,6 +74,7 @@ public class VideoCompositionFramesExtractor {
     public void doFrame(long frameTimeNanos) {
       if (releasing) {
         unwatchDisplay();
+        frameRateVote.withdraw();
         return;
       }
       recordedVsyncNanos = frameTimeNanos;
@@ -80,8 +83,12 @@ public class VideoCompositionFramesExtractor {
   };
   // The display the preview is on; null until the first collection, or when unknown.
   private DisplayVsyncPeriod vsyncPeriod;
+  // The display's highest refresh rate while playing.
+  private final PreviewFrameRateVote frameRateVote = new PreviewFrameRateVote();
   // When that thread last released frames; the playback loop stands back meanwhile.
   private volatile long lastVsyncReleaseNanos = 0;
+  // The vsync the frames collected last are shown at; 0 until the first.
+  private volatile long collectedVsyncNanos = 0;
   // The release itself runs on the playback thread: the decoder's locks are held
   // there for whole codec flushes on a seek, and the UI thread must never
   // wait on them (measured: 65 ms frames when it did).
@@ -187,11 +194,13 @@ public class VideoCompositionFramesExtractor {
       watchDisplay();
       return;
     }
+    frameRateVote.update(isPlaying);
     long period = vsyncPeriod != null ? vsyncPeriod.getPeriodNanos() : 0;
     if (period == 0 || recordedVsyncNanos == 0) return;
     // This frame's vsync, whether the recorder has run in this frame yet or not.
     long since = Math.max(0, System.nanoTime() - recordedVsyncNanos);
     long vsyncNanos = recordedVsyncNanos + (since / period) * period;
+    collectedVsyncNanos = vsyncNanos;
     if (vsyncNanos == lastVsyncNanos) return;
     lastVsyncNanos = vsyncNanos;
     long vsyncPeriodNanos = period;
@@ -243,6 +252,27 @@ public class VideoCompositionFramesExtractor {
     return isPlaying ? microTime() - startTime : pausePosition;
   }
 
+  /**
+   * The position the frames collected in this vsync were released for: the
+   * clock at the vsync rather than at the call, a few milliseconds later.
+   * drawFrame gets it as its time, so that the time it draws at and the frames
+   * it draws agree: read at the call, the time ran up to a frame ahead of the
+   * frames, at 60 fps on most vsyncs. Without vsync release (no display, or
+   * a stalled collector), the clock at the call.
+   *
+   * @return the position in microseconds
+   */
+  public long getPresentedPosition() {
+    long vsyncNanos = collectedVsyncNanos;
+    if (!isPlaying || vsyncNanos == 0
+      || System.nanoTime() - lastVsyncReleaseNanos > VSYNC_RELEASE_TIMEOUT_NANOS) {
+      return getCurrentPosition();
+    }
+    // Never before the play or seek that started this run: the vsync can
+    // precede it by up to a period.
+    return Math.max(vsyncNanos / 1000 - startTime, playStartPosition);
+  }
+
   /**
    * @return the version of the frames returned by {@link #decodeCompositionFrames},
    * see {@link VideoCompositionDecoder#getFramesVersion()}
@@ -314,6 +344,7 @@ public class VideoCompositionFramesExtractor {
       pausePosition = 0;
       seekInternal(0);
     }
+    playStartPosition = pausePosition;
     startTime = microTime() - pausePosition;
     isPlaying = true;
     pausePosition = 0;
@@ -381,6 +412,7 @@ public class VideoCompositionFramesExtractor {
       audioPlayer.seekTo(position);
     }
     if (isPlaying) {
+      playStartPosition = position;
       startTime = microTime() - position;
     } else {
       pausePosition = position;