react-native-skia-webgpu-video 0.0.0-stage → 0.1.0-beta.0

package/ios/VideoCompositionFramesExtractorSyncHostObject.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#import "DecoderWindow.h"
+#import "RNSVHostObject.h"
+#import "VideoComposition.h"
+#import "VideoCompositionItemDecoder.h"
+#import "VideoFrame.h"
+#import <AVFoundation/AVFoundation.h>
+#import <jsi/jsi.h>
+#import <map>
+
+namespace RNSkiaVideo {
+using namespace facebook;
+
+class JSI_EXPORT VideoCompositionFramesExtractorSyncHostObject
+    : public RNSVHostObject {
+public:
+  VideoCompositionFramesExtractorSyncHostObject(
+      std::shared_ptr<VideoComposition> composition);
+  ~VideoCompositionFramesExtractorSyncHostObject();
+  jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
+  std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
+
+private:
+  NSObject* lock;
+  bool disposed = false;
+  bool started = false;
+  std::shared_ptr<VideoComposition> composition;
+  DecoderWindow window;
+  std::map<std::string, std::shared_ptr<VideoCompositionItemDecoder>>
+      itemDecoders;
+  std::map<std::string, std::shared_ptr<VideoFrame>> currentFrames;
+  void release();
+  void updateWindow(CMTime time);
+};
+
+} // namespace RNSkiaVideo

package/ios/VideoCompositionFramesExtractorSyncHostObject.mm

@@ -0,0 +1,168 @@
+#include "VideoCompositionFramesExtractorSyncHostObject.h"
+
+#import "AVAssetTrackUtils.h"
+#import "RNSVJSIUtils.h"
+#include "RNSVCheckedSizes.h"
+#import <AVFoundation/AVFoundation.h>
+#import <Foundation/Foundation.h>
+#import <future>
+
+namespace RNSkiaVideo {
+
+VideoCompositionFramesExtractorSyncHostObject::
+    VideoCompositionFramesExtractorSyncHostObject(
+        std::shared_ptr<VideoComposition> composition)
+    : composition(composition), window(composition->lazyDecoders, false) {
+  lock = [[NSObject alloc] init];
+}
+
+VideoCompositionFramesExtractorSyncHostObject::
+    ~VideoCompositionFramesExtractorSyncHostObject() {
+  this->release();
+}
+
+std::vector<jsi::PropNameID>
+VideoCompositionFramesExtractorSyncHostObject::getPropertyNames(
+    jsi::Runtime& rt) {
+  std::vector<jsi::PropNameID> result;
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("start")));
+  result.push_back(
+      jsi::PropNameID::forUtf8(rt, std::string("decodeCompositionFrames")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("dispose")));
+  return result;
+}
+
+// The methods are created once per runtime (see RNSVHostObject):
+// `decodeCompositionFrames` is read for every exported frame.
+jsi::Value VideoCompositionFramesExtractorSyncHostObject::get(
+    jsi::Runtime& runtime, const jsi::PropNameID& propNameId) {
+  auto propName = propNameId.utf8(runtime);
+  if (propName == "start") {
+    return getFunction(
+        runtime, propName, 0,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          return runPooled([&] {
+            @synchronized(lock) {
+            if (disposed) throw std::runtime_error("Extractor is disposed");
+            if (started) return;
+            try {
+              for (const auto& item : composition->items) {
+                if (!item->isVideo ||
+                    !window.opens(item->compositionStartTime,
+                                  item->compositionStartTime + item->duration,
+                                  0, composition->duration, false)) {
+                  continue;
+                }
+                itemDecoders[item->id] =
+                    std::make_shared<VideoCompositionItemDecoder>(item, false);
+              }
+              started = true;
+            } catch (...) {
+              itemDecoders.clear();
+              throw;
+            }
+            }
+          });
+        });
+  } else if (propName == "decodeCompositionFrames") {
+    return getFunction(
+        runtime, propName, 1,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          if (count < 1 || !arguments[0].isNumber())
+            throw jsi::JSError(runtime, "decodeCompositionFrames expects media seconds");
+          auto currentTime = CMTimeMakeWithSeconds(
+              checkedMediaSeconds(arguments[0].asNumber()), NSEC_PER_SEC);
+          return runPooled([&]() -> jsi::Value {
+            @synchronized(lock) {
+            if (disposed) throw std::runtime_error("Extractor is disposed");
+            auto frames = jsi::Object(runtime);
+            updateWindow(currentTime);
+            for (const auto& entry : itemDecoders) {
+              auto itemId = entry.first;
+              auto decoder = entry.second;
+
+              decoder->advanceDecoder(currentTime);
+
+              auto previousFrame = currentFrames[itemId];
+              auto frame =
+                  decoder->acquireFrameForTime(currentTime, !previousFrame);
+              if (frame) {
+                currentFrames[itemId] = frame;
+              } else {
+                frame = previousFrame;
+              }
+              if (frame) {
+                frames.setProperty(
+                    runtime, entry.first.c_str(),
+                    jsi::Object::createFromHostObject(runtime, frame));
+              }
+            }
+            return frames;
+            }
+          });
+        });
+  } else if (propName == "dispose") {
+    return getFunction(
+        runtime, propName, 0,
+        [this](jsi::Runtime& runtime, const jsi::Value& thisValue,
+               const jsi::Value* arguments, size_t count) -> jsi::Value {
+          return runPooled([&] { this->release(); });
+        });
+  }
+  return jsi::Value::undefined();
+}
+
+void VideoCompositionFramesExtractorSyncHostObject::release() {
+  @synchronized(lock) {
+  if (disposed) return;
+  disposed = true;
+  try {
+    for (const auto& entry : itemDecoders) {
+      auto decoder = entry.second;
+      if (decoder) {
+        entry.second->release();
+      }
+    }
+  } catch (...) {
+  }
+  itemDecoders.clear();
+  currentFrames.clear();
+  }
+}
+
+// Opens the decoders of the items coming up and closes those left behind.
+void VideoCompositionFramesExtractorSyncHostObject::updateWindow(CMTime time) {
+  if (!window.isLazy()) {
+    return;
+  }
+  double position = CMTimeGetSeconds(time);
+  for (const auto& item : composition->items) {
+    if (!item->isVideo) {
+      continue;
+    }
+    double start = item->compositionStartTime;
+    double end = start + item->duration;
+    auto it = itemDecoders.find(item->id);
+    if (it == itemDecoders.end()) {
+      if (window.opens(start, end, position, composition->duration, false)) {
+#if defined(RNSV_TRACE_LIFECYCLE) && RNSV_TRACE_LIFECYCLE
+        NSLog(@"[rnskv] export open %s at %.3fs", item->id.c_str(), position);
+#endif
+        itemDecoders[item->id] = std::make_shared<VideoCompositionItemDecoder>(
+            item, false, nil, time);
+      }
+    } else if (!window.keeps(start, end, position, composition->duration,
+                             false)) {
+#if defined(RNSV_TRACE_LIFECYCLE) && RNSV_TRACE_LIFECYCLE
+      NSLog(@"[rnskv] export close %s at %.3fs", item->id.c_str(), position);
+#endif
+      it->second->release();
+      itemDecoders.erase(it);
+      currentFrames.erase(item->id);
+    }
+  }
+}
+
+} // namespace RNSkiaVideo

package/ios/VideoCompositionItemDecoder.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#import "SeekPolicy.h"
+#import "VideoComposition.h"
+#import "VideoFrame.h"
+#import <AVFoundation/AVFoundation.h>
+#import <list>
+#include <atomic>
+#include <mutex>
+
+using namespace facebook;
+
+namespace RNSkiaVideo {
+
+class VideoCompositionItemDecoder {
+public:
+  VideoCompositionItemDecoder(std::shared_ptr<VideoCompositionItem> item,
+                              bool realTime, AVURLAsset* sharedAsset = nil,
+                              CMTime initialTime = kCMTimeZero);
+  ~VideoCompositionItemDecoder();
+  void advanceDecoder(CMTime currentTime);
+  void seekTo(CMTime currentTime);
+  std::shared_ptr<VideoFrame> acquireFrameForTime(CMTime currentTime,
+                                                  bool force);
+  void release();
+
+private:
+  NSObject* lock;
+  // AVAssetReader cancelReading tears down its internal decode context.
+  // It must never race a copyNextSampleBuffer call on that same reader.
+  std::mutex readerMutex;
+  bool realTime = false;
+  bool hasLooped = false;
+  // Bumped when the reader is replaced/released; reading a forward seek on
+  // keeps its queued samples and outstanding batch in the same generation.
+  uint64_t readerGeneration = 0;
+  std::shared_ptr<VideoCompositionItem> item;
+  double width;
+  double height;
+  int rotation;
+  AVURLAsset* asset;
+  AVAssetTrack* videoTrack;
+  AVAssetReader* assetReader;
+  std::list<std::pair<double, CMSampleBufferRef>> decodedFrames;
+  std::list<std::pair<double, CMSampleBufferRef>> nextLoopFrames;
+  CMTime lastRequestedTime = kCMTimeInvalid;
+  std::shared_ptr<VideoFrame> currentFrame;
+  VideoFrameRing frameRing;
+  uint64_t producerId = allocateVideoProducerId();
+  bool counted = false;
+  std::atomic<bool> released{false};
+
+  void setupReader(CMTime initialTime);
+  /** Drops the reader and its decoded frames, keeping the issued ones. */
+  void discardReader();
+  std::shared_ptr<VideoFrame> makeFrame(CVPixelBufferRef buffer,
+                                        uint64_t memoryToken);
+};
+
+} // namespace RNSkiaVideo