react-native-skia-webgpu-video 0.0.0-stage → 0.1.0-beta.1

package/android/cpp/VideoCompositionFramesExtractorSyncHostObject.cpp

@@ -0,0 +1,133 @@
+#include "VideoCompositionFramesExtractorSyncHostObject.h"
+#include "ReturnedVideoFrames.h"
+#include <cmath>
+
+namespace RNSkiaVideo {
+using namespace facebook::jni;
+
+local_ref<VideoCompositionFramesExtractorSync>
+VideoCompositionFramesExtractorSync::create(
+    alias_ref<VideoComposition> composition) {
+  return newInstance(composition);
+}
+
+void VideoCompositionFramesExtractorSync::start() const {
+  static const auto startMethod = getClass()->getMethod<void()>("start");
+  startMethod(self());
+}
+
+local_ref<JMap<JString, VideoFrame>>
+VideoCompositionFramesExtractorSync::decodeCompositionFrames(jdouble time) {
+  static const auto decodeCompositionFramesMethod =
+      getClass()->getMethod<JMap<JString, VideoFrame>(jdouble)>(
+          "decodeCompositionFrames");
+  return decodeCompositionFramesMethod(self(), time);
+}
+
+void VideoCompositionFramesExtractorSync::release() const {
+  static const auto releaseMethod = getClass()->getMethod<void()>("release");
+  releaseMethod(self());
+}
+
+VideoCompositionFramesExtractorSyncHostObject::
+    VideoCompositionFramesExtractorSyncHostObject(jsi::Runtime& runtime,
+                                                  jsi::Object jsComposition) {
+  auto composition = VideoComposition::fromJSIObject(runtime, jsComposition);
+  framesExtractor =
+      make_global(VideoCompositionFramesExtractorSync::create(composition));
+}
+
+VideoCompositionFramesExtractorSyncHostObject::
+    ~VideoCompositionFramesExtractorSyncHostObject() {
+  this->release();
+}
+
+std::vector<jsi::PropNameID>
+VideoCompositionFramesExtractorSyncHostObject::getPropertyNames(
+    jsi::Runtime& rt) {
+  std::vector<jsi::PropNameID> result;
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("start")));
+  result.push_back(
+      jsi::PropNameID::forUtf8(rt, std::string("decodeCompositionFrames")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("dispose")));
+  return result;
+}
+
+// The methods are created once per runtime (see RNSVHostObject):
+// `decodeCompositionFrames` is read for every exported frame.
+jsi::Value VideoCompositionFramesExtractorSyncHostObject::get(
+    jsi::Runtime& runtime, const jsi::PropNameID& propNameId) {
+  auto propName = propNameId.utf8(runtime);
+  if (propName == "decodeCompositionFrames") {
+    return getFunction(
+        runtime, propName, 1,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          auto result = jsi::Object(runtime);
+          global_ref<VideoCompositionFramesExtractorSync> current;
+          {
+            std::lock_guard<std::mutex> lock(extractorMutex);
+            if (released.test()) return result;
+            current = make_global(framesExtractor);
+          }
+          if (count < 1 || !arguments[0].isNumber() || !std::isfinite(arguments[0].asNumber()))
+            throw jsi::JSError(runtime, "Expected a finite video decode time");
+          auto time = arguments[0].asNumber();
+          // Do not hold the C++ mutex while Java waits: dispose must be able
+          // to cancel that request on another runtime.
+          auto frames = current->decodeCompositionFrames(time);
+          ReturnedVideoFrames returned(frames);
+          for (auto& entry : *frames) {
+            auto id = entry.first->toStdString();
+            auto frame = entry.second;
+            if (frame->isHardwareBuffer()) {
+              result.setProperty(runtime, id.c_str(), frame->toJS(runtime));
+              continue;
+            }
+            auto cached = getVersionedObject(runtime, "frame:" + id,
+              static_cast<double>(frame->getId()), [&](jsi::Object& holder) {
+                holder.setProperty(runtime, "value", frame->toJS(runtime));
+              });
+            result.setProperty(runtime, id.c_str(), cached.asObject(runtime).getProperty(runtime, "value"));
+          }
+          returned.close();
+          return result;
+        });
+  } else if (propName == "start") {
+    return getFunction(
+        runtime, propName, 0,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          global_ref<VideoCompositionFramesExtractorSync> current;
+          {
+            std::lock_guard<std::mutex> lock(extractorMutex);
+            if (released.test()) return jsi::Value::undefined();
+            current = make_global(framesExtractor);
+          }
+          current->start();
+          return jsi::Value::undefined();
+        });
+  }
+  if (propName == "dispose") {
+    return getFunction(
+        runtime, propName, 0,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          this->release();
+          return jsi::Value::undefined();
+        });
+  }
+  return jsi::Value::undefined();
+}
+
+void VideoCompositionFramesExtractorSyncHostObject::release() {
+  global_ref<VideoCompositionFramesExtractorSync> current;
+  {
+    std::lock_guard<std::mutex> lock(extractorMutex);
+    if (released.test_and_set()) return;
+    current = std::move(framesExtractor);
+  }
+  if (current) current->release();
+}
+
+} // namespace RNSkiaVideo

package/android/cpp/VideoCompositionFramesExtractorSyncHostObject.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include "RNSVHostObject.h"
+#include "VideoComposition.h"
+#include "VideoFrame.h"
+#include <fbjni/fbjni.h>
+#include <jsi/jsi.h>
+
+namespace RNSkiaVideo {
+using namespace facebook;
+using namespace jni;
+
+struct VideoCompositionFramesExtractorSync
+    : public jni::JavaClass<VideoCompositionFramesExtractorSync> {
+
+public:
+  static constexpr auto kJavaDescriptor =
+      "Lcom/azzapp/rnskv/VideoCompositionFramesExtractorSync;";
+
+  local_ref<VideoCompositionFramesExtractorSync> static create(
+      alias_ref<VideoComposition> composition);
+
+  void start() const;
+
+  local_ref<JMap<JString, VideoFrame>> decodeCompositionFrames(jdouble time);
+
+  void release() const;
+};
+
+class JSI_EXPORT VideoCompositionFramesExtractorSyncHostObject
+    : public RNSVHostObject {
+public:
+  VideoCompositionFramesExtractorSyncHostObject(jsi::Runtime& runtime,
+                                                jsi::Object composition);
+  ~VideoCompositionFramesExtractorSyncHostObject();
+  jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
+  std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
+
+private:
+  global_ref<VideoCompositionFramesExtractorSync> framesExtractor;
+  std::mutex extractorMutex;
+  std::atomic_flag released = ATOMIC_FLAG_INIT;
+  void release();
+};
+
+} // namespace RNSkiaVideo

package/android/cpp/VideoEncoderHostObject.cpp

@@ -0,0 +1,318 @@
+#include "RNSVCheckedSizes.h"
+#include "VideoEncoderHostObject.h"
+#include "NativeHardwareBuffer.h"
+#include <EGL/egl.h>
+#include <fbjni/ByteBuffer.h>
+#include <cmath>
+#include <limits>
+#include <unistd.h>
+
+namespace RNSkiaVideo {
+using namespace facebook::jni;
+
+namespace {
+/** Makes the encoder's EGL context current; restores the caller's (or none). */
+class ScopedEncoderContext {
+public:
+  explicit ScopedEncoderContext(const global_ref<VideoEncoder>& encoder)
+      : display(eglGetCurrentDisplay()), context(eglGetCurrentContext()),
+        draw(eglGetCurrentSurface(EGL_DRAW)), read(eglGetCurrentSurface(EGL_READ)) {
+    encoder->makeGLContextCurrent();
+  }
+  ~ScopedEncoderContext() {
+    if (context != EGL_NO_CONTEXT) {
+      eglMakeCurrent(display, draw, read, context);
+    } else if (auto current = eglGetCurrentDisplay(); current != EGL_NO_DISPLAY) {
+      eglMakeCurrent(current, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
+    }
+  }
+  ScopedEncoderContext(const ScopedEncoderContext&) = delete;
+  ScopedEncoderContext& operator=(const ScopedEncoderContext&) = delete;
+
+private:
+  EGLDisplay display;
+  EGLContext context;
+  EGLSurface draw;
+  EGLSurface read;
+};
+
+/** Sync-fds handed over by JS; each is closed exactly once. */
+struct OwnedFences {
+  std::vector<int> fds;
+  ~OwnedFences() {
+    for (int fd : fds)
+      if (fd >= 0) close(fd);
+  }
+};
+} // namespace
+
+local_ref<VideoEncoder>
+VideoEncoder::create(std::string& outPath, int width, int height, int frameRate,
+                     int bitRate, std::optional<std::string> encoderName,
+                     std::optional<std::string> codec,
+                     alias_ref<VideoComposition> composition,
+                     int audioSampleRate, int audioChannelCount,
+                     int audioBitRate) {
+  // Both strings are nullable on the Java side (a null codec means H.264), so
+  // they are built as jstring refs rather than handed over as std::string: a
+  // ternary mixing std::string and nullptr collapses to std::string, and
+  // constructing one from nullptr calls strlen(nullptr).
+  auto encoderNameRef = encoderName.has_value()
+                            ? make_jstring(encoderName.value())
+                            : local_ref<JString>(nullptr);
+  auto codecRef = codec.has_value() ? make_jstring(codec.value())
+                                    : local_ref<JString>(nullptr);
+  return newInstance(outPath, width, height, frameRate, bitRate,
+                     encoderNameRef, codecRef, composition, audioSampleRate,
+                     audioChannelCount, audioBitRate);
+}
+
+bool VideoEncoder::isCodecSupported(const std::string& codec) {
+  static const auto method =
+      javaClassStatic()->getStaticMethod<jboolean(std::string)>(
+          "isCodecSupported");
+  return method(javaClassStatic(), codec);
+}
+
+void VideoEncoder::prepare() const {
+  static const auto prepareMethod = getClass()->getMethod<void()>("prepare");
+  prepareMethod(self());
+}
+
+void VideoEncoder::makeGLContextCurrent() const {
+  static const auto makeGLContextCurrentMethod =
+      getClass()->getMethod<void()>("makeGLContextCurrent");
+  makeGLContextCurrentMethod(self());
+}
+
+void VideoEncoder::encodeFrameRgba(alias_ref<JByteBuffer> pixels, jint width,
+                                   jint height, jint bytesPerRow, jdouble time) const {
+  static const auto method =
+      getClass()->getMethod<void(alias_ref<JByteBuffer>, jint, jint, jint, jdouble)>("encodeFrameRgba");
+  method(self(), pixels, width, height, bytesPerRow, time);
+}
+
+void VideoEncoder::encodeTexture(jint texture, jdouble time) const {
+  static const auto method = getClass()->getMethod<void(jint, jdouble)>("encodeTexture");
+  method(self(), texture, time);
+}
+
+void VideoEncoder::release() const {
+  static const auto releaseMethod = getClass()->getMethod<void()>("release");
+  releaseMethod(self());
+}
+
+void VideoEncoder::finishWriting() const {
+  static const auto finishWritingMethod =
+      getClass()->getMethod<void()>("finishWriting");
+  finishWritingMethod(self());
+}
+
+VideoEncoderHostObject::VideoEncoderHostObject(
+    std::string& outPath, int width, int height, int frameRate, int bitRate,
+    std::optional<std::string> encoderName, std::optional<std::string> codec,
+    alias_ref<VideoComposition> composition, int audioSampleRate,
+    int audioChannelCount, int audioBitRate)
+    : width(width), height(height) {
+  framesExtractor = make_global(VideoEncoder::create(
+      outPath, width, height, frameRate, bitRate, encoderName, codec,
+      composition, audioSampleRate, audioChannelCount, audioBitRate));
+}
+
+VideoEncoderHostObject::~VideoEncoderHostObject() {
+  this->release();
+}
+
+std::vector<jsi::PropNameID>
+VideoEncoderHostObject::getPropertyNames(jsi::Runtime& rt) {
+  std::vector<jsi::PropNameID> result;
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("prepare")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("encodeFrame")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("acquireFrameBuffer")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("releaseFrameBuffer")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("finishWriting")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("dispose")));
+  return result;
+}
+
+// The methods are created once per runtime (see RNSVHostObject):
+// `encodeFrame` is read for every exported frame.
+jsi::Value VideoEncoderHostObject::get(jsi::Runtime& runtime,
+                                       const jsi::PropNameID& propNameId) {
+  auto propName = propNameId.utf8(runtime);
+  if (propName == "encodeFrame") {
+    return getFunction(
+        runtime, propName, 2,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          std::lock_guard<std::recursive_mutex> lock(encoderMutex);
+          if (released.test() || !prepared || finished) {
+            throw jsi::JSError(runtime, "Video encoder is not ready to accept frames");
+          }
+          if (count != 2 || !arguments[0].isObject() || !arguments[1].isNumber()) {
+            throw jsi::JSError(runtime, "encodeFrame expects an owned RGBA descriptor and a time");
+          }
+          auto frame = arguments[0].asObject(runtime);
+          const auto kindValue = frame.getProperty(runtime, "kind");
+          const auto kind = kindValue.isString() ? kindValue.asString(runtime).utf8(runtime) : "";
+          if (kind == "native-buffer") {
+            // A lent buffer Skia rendered into; its sync-fds are ours to close.
+            OwnedFences fences;
+            const auto fenceValues = frame.getProperty(runtime, "fences");
+            if (fenceValues.isObject()) {
+              auto array = fenceValues.asObject(runtime).asArray(runtime);
+              for (size_t i = 0; i < array.size(runtime); ++i) {
+                auto value = array.getValueAtIndex(runtime, i);
+                if (!value.isBigInt()) throw jsi::JSError(runtime, "Encoder fences must be sync-fd BigInts");
+                // WebGPU reports an already signaled fence as -1, as a uint32.
+                fences.fds.push_back(static_cast<int>(
+                    static_cast<uint32_t>(value.asBigInt(runtime).getUint64(runtime))));
+              }
+            }
+            const auto pointerValue = frame.getProperty(runtime, "nativeBuffer");
+            if (!pointerValue.isBigInt() || !targets)
+              throw jsi::JSError(runtime, "Encoder frame buffer is not the one lent");
+            const auto* buffer = reinterpret_cast<const AHardwareBuffer*>(
+                static_cast<uintptr_t>(pointerValue.asBigInt(runtime).getUint64(runtime)));
+            const auto time = checkedMediaSeconds(arguments[1].asNumber());
+            ScopedEncoderContext scope(framesExtractor);
+            const auto texture = targets->beginEncode(buffer, fences.fds);
+            try {
+              framesExtractor->encodeTexture(static_cast<jint>(texture), time);
+            } catch (...) {
+              targets->finishEncode();
+              throw;
+            }
+            targets->finishEncode();
+            return jsi::Value::undefined();
+          }
+          if (kind != "rgba") {
+            throw jsi::JSError(runtime, "Android encoder accepts RGBA pixels or a lent hardware buffer");
+          }
+          const auto checkedInteger = [&](const char* name) -> int {
+            const auto value = frame.getProperty(runtime, name).asNumber();
+            if (!std::isfinite(value) || value <= 0 || std::floor(value) != value ||
+                value > std::numeric_limits<int>::max()) {
+              throw jsi::JSError(runtime, "Invalid encoder RGBA dimensions or stride");
+            }
+            return static_cast<int>(value);
+          };
+          const int width = checkedInteger("width");
+          const int height = checkedInteger("height");
+          const int stride = checkedInteger("bytesPerRow");
+          const uint64_t packedStride = static_cast<uint64_t>(width) * 4;
+          const uint64_t required = static_cast<uint64_t>(stride) * (height - 1) + packedStride;
+          if (static_cast<uint64_t>(stride) < packedStride || required > std::numeric_limits<int>::max()) {
+            throw jsi::JSError(runtime, "Invalid encoder RGBA byte layout");
+          }
+          auto data = frame.getProperty(runtime, "data").asObject(runtime);
+          size_t offset = 0;
+          size_t length = 0;
+          jsi::ArrayBuffer buffer = [&]() {
+            if (data.isArrayBuffer(runtime)) {
+              auto result = data.getArrayBuffer(runtime);
+              length = result.size(runtime);
+              return result;
+            }
+            auto raw = data.getProperty(runtime, "buffer").asObject(runtime).getArrayBuffer(runtime);
+            const auto offsetValue = data.getProperty(runtime, "byteOffset").asNumber();
+            const auto lengthValue = data.getProperty(runtime, "byteLength").asNumber();
+            if (!std::isfinite(offsetValue) || !std::isfinite(lengthValue) || offsetValue < 0 ||
+                lengthValue < 0 || std::floor(offsetValue) != offsetValue ||
+                std::floor(lengthValue) != lengthValue || offsetValue > raw.size(runtime) ||
+                lengthValue > raw.size(runtime) - offsetValue) {
+              throw jsi::JSError(runtime, "Invalid encoder RGBA byte view");
+            }
+            offset = static_cast<size_t>(offsetValue);
+            length = static_cast<size_t>(lengthValue);
+            return raw;
+          }();
+          if (length < required) {
+            throw jsi::JSError(runtime, "Encoder RGBA data is too short");
+          }
+          auto javaPixels = JByteBuffer::wrapBytes(buffer.data(runtime) + offset, required);
+          framesExtractor->encodeFrameRgba(javaPixels, width, height, stride, checkedMediaSeconds(arguments[1].asNumber()));
+          return jsi::Value::undefined();
+        });
+  } else if (propName == "acquireFrameBuffer") {
+    return getFunction(
+        runtime, propName, 0,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          std::lock_guard<std::recursive_mutex> lock(encoderMutex);
+          if (released.test() || !prepared || finished) {
+            throw jsi::JSError(runtime, "Encoder not prepared");
+          }
+          // The buffers are imported by the same Dawn device as decoded AHBs.
+          if (!NativeHardwareBuffer::isEnabled()) {
+            throw jsi::JSError(runtime, "Android GPU export requires AHardwareBuffer interop");
+          }
+          if (!targets) targets = std::make_unique<EncoderTargets>(width, height);
+          auto* buffer = targets->lend();
+          return jsi::BigInt::fromUint64(runtime, reinterpret_cast<uintptr_t>(buffer));
+        });
+  } else if (propName == "releaseFrameBuffer") {
+    return getFunction(
+        runtime, propName, 0,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          std::lock_guard<std::recursive_mutex> lock(encoderMutex);
+          if (targets) targets->giveBack();
+          return jsi::Value::undefined();
+        });
+  } else if (propName == "prepare") {
+    return getFunction(
+        runtime, propName, 0,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          std::lock_guard<std::recursive_mutex> lock(encoderMutex);
+          if (!released.test() && !prepared) {
+            framesExtractor->prepare();
+            prepared = true;
+          }
+          return jsi::Value::undefined();
+        });
+  } else if (propName == "finishWriting") {
+    return getFunction(
+        runtime, propName, 0,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          std::lock_guard<std::recursive_mutex> lock(encoderMutex);
+          if (!released.test() && prepared && !finished) {
+            framesExtractor->finishWriting();
+            finished = true;
+          }
+          return jsi::Value::undefined();
+        });
+  }
+  if (propName == "dispose") {
+    return getFunction(
+        runtime, propName, 0,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          this->release();
+          return jsi::Value::undefined();
+        });
+  }
+  return jsi::Value::undefined();
+}
+
+void VideoEncoderHostObject::release() {
+  std::lock_guard<std::recursive_mutex> lock(encoderMutex);
+  if (!released.test_and_set()) {
+    if (targets && !targets->empty()) {
+      // GL names belong to the encoder context; free them before it goes.
+      // Otherwise the destructor still releases buffers and reservations.
+      try {
+        ScopedEncoderContext scope(framesExtractor);
+        targets->clear();
+      } catch (...) {
+      }
+    }
+    targets.reset();
+    framesExtractor->release();
+    framesExtractor = nullptr;
+  }
+}
+
+} // namespace RNSkiaVideo

package/android/cpp/VideoEncoderHostObject.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#include "EncoderTargets.h"
+#include "RNSVHostObject.h"
+#include "VideoComposition.h"
+#include <memory>
+#include <mutex>
+#include <fbjni/ByteBuffer.h>
+#include <fbjni/fbjni.h>
+#include <jsi/jsi.h>
+
+namespace RNSkiaVideo {
+using namespace facebook;
+using namespace jni;
+
+struct VideoEncoder : public jni::JavaClass<VideoEncoder> {
+
+public:
+  static constexpr auto kJavaDescriptor = "Lcom/azzapp/rnskv/VideoEncoder;";
+
+  local_ref<VideoEncoder> static create(std::string& outPath, int width,
+                                        int height, int frameRate, int bitRate,
+                                        std::optional<std::string> encoderName,
+                                        std::optional<std::string> codec,
+                                        alias_ref<VideoComposition> composition,
+                                        int audioSampleRate,
+                                        int audioChannelCount,
+                                        int audioBitRate);
+
+  /**
+   * Whether the device has an encoder for the given codec ("h264" or "hevc").
+   */
+  static bool isCodecSupported(const std::string& codec);
+
+  void prepare() const;
+
+  void makeGLContextCurrent() const;
+
+  void encodeFrameRgba(alias_ref<JByteBuffer> pixels, jint width, jint height,
+                       jint bytesPerRow, jdouble time) const;
+
+  /** A GL_TEXTURE_2D of the encoder's context, e.g. an imported buffer. */
+  void encodeTexture(jint texture, jdouble time) const;
+
+  void finishWriting() const;
+
+  void release() const;
+};
+
+class JSI_EXPORT VideoEncoderHostObject : public RNSVHostObject {
+public:
+  VideoEncoderHostObject(std::string& outPath, int width, int height,
+                         int frameRate, int bitRate,
+                         std::optional<std::string> encoderName,
+                         std::optional<std::string> codec,
+                         alias_ref<VideoComposition> composition,
+                         int audioSampleRate, int audioChannelCount,
+                         int audioBitRate);
+  ~VideoEncoderHostObject() override;
+  jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
+  std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
+
+private:
+  global_ref<VideoEncoder> framesExtractor;
+  // GPU export: buffers Skia renders into, sampled by the encoder's context.
+  std::unique_ptr<EncoderTargets> targets;
+  int width;
+  int height;
+  std::recursive_mutex encoderMutex;
+  bool prepared = false;
+  bool finished = false;
+  std::atomic_flag released = ATOMIC_FLAG_INIT;
+  void release();
+};
+
+} // namespace RNSkiaVideo