react-native-skia-webgpu-video 0.1.0-beta.3 → 0.1.0-beta.4

package/src/gpuDevice.ts

@@ -8,7 +8,7 @@ import {
   importDevice,
   installWebGPU,
 } from "react-native-webgpu";
-import type { NativeVideoFrame } from "react-native-webgpu";
+import type { NativeVideoFrame, RNCanvasContext } from "react-native-webgpu";
 import RNSkiaVideoModule from "./RNSkiaVideoModule";
 
 // Evaluate on the RN runtime. NativeObject serializers carry these objects to
@@ -61,11 +61,36 @@ export const createNativeVideoGpuFrame = (
   return nativeWebGPU.createVideoFrameFromNativeBuffer(pointer);
 };
 
+/** Android: a canvas context presenting to a native window that is not a
+ * view, such as an encoder's input Surface. */
+export const createNativeWindowGpuContext = (
+  window: bigint,
+  width: number,
+  height: number,
+): RNCanvasContext => {
+  "worklet";
+  if (
+    typeof nativeWebGPU?.MakeWebGPUCanvasContextFromNativeWindow !== "function"
+  )
+    throw new Error(
+      "React Native WebGPU cannot present to a native window; 0.19.0 or later is required",
+    );
+  return nativeWebGPU.MakeWebGPUCanvasContextFromNativeWindow(
+    window,
+    width,
+    height,
+  );
+};
+
 export const nativeVideoTextureUsage =
   GPUTextureUsage.RENDER_ATTACHMENT |
   GPUTextureUsage.TEXTURE_BINDING |
   GPUTextureUsage.COPY_SRC;
 
+// An encoder's input window only receives copies of the frames Skia drew.
+export const encoderWindowTextureUsage =
+  GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.COPY_DST;
+
 // CPU RGBA uploads additionally need COPY_DST. Keep the native-buffer blit's
 // descriptor unchanged: that path renders into its destination attachment.
 export const rgbaVideoTextureUsage =

package/src/previewLifecycle.ts

@@ -33,7 +33,6 @@ export const disposePreviewResources = (resources: PreviewResources) => {
       frame.dispose?.();
       resources.frames.shift();
     }
-    resources.canvas?.dispose();
     resources.canvas = null;
     resources.surface?.dispose();
     resources.surface = null;

package/src/types.ts

@@ -421,14 +421,19 @@ export type VideoEncoder = {
    */
   encodeFrame(texture: unknown, time: number): void;
   /**
-   * Lends the next pool buffer to draw into on the GPU: a CVPixelBuffer on
-   * iOS, an RGBA AHardwareBuffer on Android. Pass it back as a native-buffer
-   * frame (on Android with the sync-fds ending its GPU access, in `fences`),
-   * or release it. One at a time.
+   * iOS: lends the next pool buffer (a CVPixelBuffer) to draw into on the
+   * GPU. Pass it back as a native-buffer frame, or release it. One at a time.
    */
   acquireFrameBuffer?(): bigint;
-  /** Returns a lent buffer that will not be encoded. */
+  /** iOS: returns a lent buffer that will not be encoded. */
   releaseFrameBuffer?(): void;
+  /**
+   * Android: the codec's input Surface, as an ANativeWindow pointer for a
+   * WebGPU canvas context to present to. Each frame is then encoded as
+   * `{ kind: "window", present }`: the encoder sets the frame's time on the
+   * window and calls `present`, which presents the context.
+   */
+  acquireInputWindow?(): bigint;
   /*
    * Finish writing the video to the output file.
    */

package/src/yuvBlit.ts

@@ -68,40 +68,52 @@ struct VsOut { @builtin(position) position: vec4f, @location(0) uv: vec2f };
 // primaries to sRGB, and every export brightens the midtones again.
 const sourceColorSpace = { primaries: "srgb", transfer: "srgb" } as const;
 
-/** The corner of a decoder buffer's crop rectangle once the buffer is turned
- * clockwise for display: the blit takes its source origin in that turned
- * space. Throws unless the rectangle lies in the buffer and turns into the
- * frame's display size. */
-export const yuvFrameOrigin = (
+/** A decoder buffer's crop rectangle once the buffer is turned clockwise for
+ * display: the blit samples that upright region, and draws it at the size of
+ * its target, which a capped or explicit frame size makes smaller. Throws
+ * unless the rectangle lies in the buffer. */
+export const yuvFrameRegion = (
   region: { x: number; y: number; width: number; height: number },
   rotation: number,
   buffer: { width: number; height: number },
-  frame: { width: number; height: number },
-): { x: number; y: number } => {
+): { x: number; y: number; width: number; height: number } => {
   "worklet";
   const turned = rotation === 90 || rotation === 270;
   if (
     (rotation !== 0 && !turned && rotation !== 180) ||
     region.x < 0 ||
     region.y < 0 ||
+    region.width <= 0 ||
+    region.height <= 0 ||
     region.x + region.width > buffer.width ||
-    region.y + region.height > buffer.height ||
-    (turned ? region.height : region.width) !== frame.width ||
-    (turned ? region.width : region.height) !== frame.height
+    region.y + region.height > buffer.height
   ) {
     throw new Error("Native video buffer region or YUV layout mismatch");
   }
-  const { width, height } = buffer;
+  const width = turned ? region.height : region.width;
+  const height = turned ? region.width : region.height;
   if (rotation === 90)
-    return { x: height - region.y - region.height, y: region.x };
+    return {
+      x: buffer.height - region.y - region.height,
+      y: region.x,
+      width,
+      height,
+    };
   if (rotation === 180)
     return {
-      x: width - region.x - region.width,
-      y: height - region.y - region.height,
+      x: buffer.width - region.x - region.width,
+      y: buffer.height - region.y - region.height,
+      width,
+      height,
     };
   if (rotation === 270)
-    return { x: region.y, y: width - region.x - region.width };
-  return { x: region.x, y: region.y };
+    return {
+      x: region.y,
+      y: buffer.width - region.x - region.width,
+      width,
+      height,
+    };
+  return { x: region.x, y: region.y, width, height };
 };
 
 const getYuvBlit = (device: GPUDevice): YuvBlit => {
@@ -251,10 +263,11 @@ const importYuvBuffer = (
 };
 
 /**
- * Draws the upright `width`×`height` region at `origin` of a decoder buffer
- * into `target`, converting YUV to RGB; the buffer is imported on its first
- * frame. Submits the draw: the caller waits for it before releasing the
- * frame's lease or `source`, as after copyExternalImageToTexture.
+ * Draws the upright `region` of a decoder buffer (see yuvFrameRegion) into
+ * the whole of `target`, converting YUV to RGB and scaling it to the target's
+ * size; the buffer is imported on its first frame. Submits the draw: the
+ * caller waits for it before releasing the frame's lease or `source`, as
+ * after copyExternalImageToTexture.
  */
 export const blitYuvFrame = (
   device: GPUDevice,
@@ -262,9 +275,7 @@ export const blitYuvFrame = (
   producer: number,
   source: NativeVideoFrame,
   rotation: number,
-  origin: { x: number; y: number },
-  width: number,
-  height: number,
+  region: { x: number; y: number; width: number; height: number },
   target: GPUTexture,
 ): void => {
   "worklet";
@@ -297,10 +308,10 @@ export const blitYuvFrame = (
   const frameWidth = turned ? source.height : source.width;
   const frameHeight = turned ? source.width : source.height;
   const mapping = [
-    width / frameWidth,
-    height / frameHeight,
-    origin.x / frameWidth,
-    origin.y / frameHeight,
+    region.width / frameWidth,
+    region.height / frameHeight,
+    region.x / frameWidth,
+    region.y / frameHeight,
   ];
   if (mapping.some((value, i) => value !== entry.mapping[i])) {
     device.queue.writeBuffer(entry.uniforms, 0, new Float32Array(mapping));
@@ -355,7 +366,11 @@ export const blitNv12Frame = (
       size: 16,
       usage: videoUniformBufferUsage,
     });
-    device.queue.writeBuffer(blit.wholeFrame, 0, new Float32Array([1, 1, 0, 0]));
+    device.queue.writeBuffer(
+      blit.wholeFrame,
+      0,
+      new Float32Array([1, 1, 0, 0]),
+    );
   }
   const external = device.importExternalTexture({
     label: "video decoder buffer",

package/android/cpp/EncoderTargets.cpp

@@ -1,226 +0,0 @@
-#include "EncoderTargets.h"
-#include "RNSVMemoryBudget.h"
-#include <GLES2/gl2ext.h>
-#include <cerrno>
-#include <poll.h>
-#include <stdexcept>
-#include <string>
-#include <unistd.h>
-
-namespace RNSkiaVideo {
-namespace {
-constexpr EGLTimeKHR kWaitNanoseconds = 5'000'000'000ULL;
-constexpr int kFenceTimeoutMs = 5000;
-
-struct EglFunctions {
-  PFNEGLGETNATIVECLIENTBUFFERANDROIDPROC clientBuffer;
-  PFNEGLCREATEIMAGEKHRPROC createImage;
-  PFNEGLDESTROYIMAGEKHRPROC destroyImage;
-  PFNGLEGLIMAGETARGETTEXTURE2DOESPROC imageTarget;
-  PFNEGLCREATESYNCKHRPROC createSync;
-  PFNEGLCLIENTWAITSYNCKHRPROC clientWaitSync;
-  PFNEGLDESTROYSYNCKHRPROC destroySync;
-};
-
-const EglFunctions& egl() {
-  static const EglFunctions functions = [] {
-    EglFunctions f{
-        reinterpret_cast<PFNEGLGETNATIVECLIENTBUFFERANDROIDPROC>(
-            eglGetProcAddress("eglGetNativeClientBufferANDROID")),
-        reinterpret_cast<PFNEGLCREATEIMAGEKHRPROC>(eglGetProcAddress("eglCreateImageKHR")),
-        reinterpret_cast<PFNEGLDESTROYIMAGEKHRPROC>(eglGetProcAddress("eglDestroyImageKHR")),
-        reinterpret_cast<PFNGLEGLIMAGETARGETTEXTURE2DOESPROC>(
-            eglGetProcAddress("glEGLImageTargetTexture2DOES")),
-        reinterpret_cast<PFNEGLCREATESYNCKHRPROC>(eglGetProcAddress("eglCreateSyncKHR")),
-        reinterpret_cast<PFNEGLCLIENTWAITSYNCKHRPROC>(eglGetProcAddress("eglClientWaitSyncKHR")),
-        reinterpret_cast<PFNEGLDESTROYSYNCKHRPROC>(eglGetProcAddress("eglDestroySyncKHR")),
-    };
-    if (!f.clientBuffer || !f.createImage || !f.destroyImage || !f.imageTarget ||
-        !f.createSync || !f.clientWaitSync || !f.destroySync)
-      throw std::runtime_error("EGL hardware buffer or fence entry points are unavailable");
-    return f;
-  }();
-  return functions;
-}
-
-/** Waits for a sync-fd to signal, then closes it. */
-void waitFence(int& fd) {
-  if (fd < 0) return;  // Already signaled.
-  pollfd entry{fd, POLLIN, 0};
-  int result;
-  do {
-    result = poll(&entry, 1, kFenceTimeoutMs);
-  } while (result < 0 && errno == EINTR);
-  const int error = errno;
-  close(fd);
-  fd = -1;
-  if (result == 0) throw std::runtime_error("GPU export frame did not complete within 5 seconds");
-  if (result < 0) throw std::runtime_error("Cannot wait on GPU export fence: errno " + std::to_string(error));
-  if (entry.revents & (POLLERR | POLLNVAL))
-    throw std::runtime_error("Invalid GPU export fence");
-}
-} // namespace
-
-EncoderTargets::EncoderTargets(int width, int height) : width(width), height(height) {
-  if (width <= 0 || height <= 0 || width > 16384 || height > 16384)
-    throw std::invalid_argument("Invalid encoder buffer dimensions");
-}
-
-EncoderTargets::~EncoderTargets() {
-  // clear() could not run with the encoder context current: its textures went
-  // with that context. Images, syncs and buffers need no current context.
-  for (auto& slot : slots) {
-    if (slot.reads != EGL_NO_SYNC_KHR) egl().destroySync(slot.display, slot.reads);
-    if (slot.image != EGL_NO_IMAGE_KHR) egl().destroyImage(slot.display, slot.image);
-    if (slot.buffer) AHardwareBuffer_release(slot.buffer);
-    if (slot.reservation) MemoryBudget::instance().release(slot.reservation);
-  }
-}
-
-bool EncoderTargets::waitReads(Slot& slot, EGLTimeKHR timeout) {
-  if (slot.reads == EGL_NO_SYNC_KHR) return true;
-  const auto status = egl().clientWaitSync(slot.display, slot.reads, 0, timeout);
-  if (status == EGL_TIMEOUT_EXPIRED_KHR) return false;
-  if (status != EGL_CONDITION_SATISFIED_KHR)
-    throw std::runtime_error("Cannot wait on encoder buffer reads: EGL error " + std::to_string(eglGetError()));
-  egl().destroySync(slot.display, slot.reads);
-  slot.reads = EGL_NO_SYNC_KHR;
-  return true;
-}
-
-void EncoderTargets::allocate(Slot& slot) {
-  slot.reservation = MemoryBudget::instance().reserve(
-      static_cast<uint64_t>(width) * height * 4, "android encoder ahardwarebuffer");
-  AHardwareBuffer_Desc descriptor{};
-  descriptor.width = width;
-  descriptor.height = height;
-  descriptor.layers = 1;
-  descriptor.format = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
-  descriptor.usage = AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE | AHARDWAREBUFFER_USAGE_GPU_COLOR_OUTPUT;
-  const int result = AHardwareBuffer_allocate(&descriptor, &slot.buffer);
-  if (result != 0 || !slot.buffer)
-    throw std::runtime_error("Encoder AHardwareBuffer allocation failed: " + std::to_string(result));
-  AHardwareBuffer_Desc actual{};
-  AHardwareBuffer_describe(slot.buffer, &actual);
-  if (actual.width != descriptor.width || actual.height != descriptor.height ||
-      actual.format != descriptor.format || actual.stride < descriptor.width ||
-      (actual.usage & descriptor.usage) != descriptor.usage)
-    throw std::runtime_error("Invalid encoder AHardwareBuffer layout");
-  MemoryBudget::instance().resize(slot.reservation, static_cast<uint64_t>(actual.stride) * height * 4,
-                                  "android encoder ahardwarebuffer padded stride");
-}
-
-void EncoderTargets::bindTexture(Slot& slot) {
-  if (slot.texture) return;
-  const auto display = eglGetCurrentDisplay();
-  if (display == EGL_NO_DISPLAY || eglGetCurrentContext() == EGL_NO_CONTEXT)
-    throw std::runtime_error("Encoder buffer import needs the encoder EGL context");
-  const EGLint attributes[] = {EGL_IMAGE_PRESERVED_KHR, EGL_TRUE, EGL_NONE};
-  slot.display = display;
-  slot.image = egl().createImage(display, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID,
-                                 egl().clientBuffer(slot.buffer), attributes);
-  if (slot.image == EGL_NO_IMAGE_KHR)
-    throw std::runtime_error("Cannot import encoder AHardwareBuffer: EGL error " + std::to_string(eglGetError()));
-  GLint previous = 0;
-  glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous);
-  glGenTextures(1, &slot.texture);
-  glBindTexture(GL_TEXTURE_2D, slot.texture);
-  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
-  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-  egl().imageTarget(GL_TEXTURE_2D, slot.image);
-  glBindTexture(GL_TEXTURE_2D, previous);
-  const auto error = glGetError();
-  if (error != GL_NO_ERROR)
-    throw std::runtime_error("Encoder AHardwareBuffer texture binding failed: GL error " + std::to_string(error));
-}
-
-AHardwareBuffer* EncoderTargets::lend() {
-  if (lent >= 0 || encoding >= 0)
-    throw std::runtime_error("Previous encoder frame buffer was neither encoded nor released");
-  // A buffer whose last frame the encoder already sampled.
-  for (size_t i = 0; i < slots.size(); ++i) {
-    if (waitReads(slots[i], 0)) {
-      lent = static_cast<int>(i);
-      return slots[i].buffer;
-    }
-  }
-  if (slots.size() < kCapacity) {
-    slots.emplace_back();
-    try {
-      allocate(slots.back());
-    } catch (...) {
-      auto& slot = slots.back();
-      if (slot.buffer) AHardwareBuffer_release(slot.buffer);
-      if (slot.reservation) MemoryBudget::instance().release(slot.reservation);
-      slots.pop_back();
-      throw;
-    }
-    lent = static_cast<int>(slots.size() - 1);
-    return slots.back().buffer;
-  }
-  for (size_t i = 0; i < slots.size(); ++i) {
-    if (waitReads(slots[i], kWaitNanoseconds)) {
-      lent = static_cast<int>(i);
-      return slots[i].buffer;
-    }
-  }
-  throw std::runtime_error("Encoder buffers stayed in use for 5 seconds");
-}
-
-void EncoderTargets::giveBack() { lent = -1; }
-
-bool EncoderTargets::isLent(const AHardwareBuffer* buffer) const {
-  return lent >= 0 && buffer && slots[lent].buffer == buffer;
-}
-
-GLuint EncoderTargets::beginEncode(const AHardwareBuffer* buffer, std::vector<int>& fences) {
-  if (!isLent(buffer)) throw std::runtime_error("Encoder frame buffer is not the one lent");
-  // The renderer's writes, and its release of the buffer, end before GL reads.
-  for (auto& fence : fences) waitFence(fence);
-  auto& slot = slots[lent];
-  bindTexture(slot);
-  encoding = lent;
-  lent = -1;
-  return slot.texture;
-}
-
-void EncoderTargets::finishEncode() {
-  if (encoding < 0) return;
-  auto& slot = slots[encoding];
-  encoding = -1;
-  const auto display = eglGetCurrentDisplay();
-  slot.reads = display == EGL_NO_DISPLAY ? EGL_NO_SYNC_KHR
-                                         : egl().createSync(display, EGL_SYNC_FENCE_KHR, nullptr);
-  if (slot.reads == EGL_NO_SYNC_KHR) {
-    // No fence to poll later: complete the reads now.
-    glFinish();
-    return;
-  }
-  slot.display = display;
-  glFlush();
-}
-
-void EncoderTargets::clear() {
-  for (auto& slot : slots) {
-    if (!waitReads(slot, kWaitNanoseconds)) glFinish();
-    if (slot.reads != EGL_NO_SYNC_KHR) {
-      egl().destroySync(slot.display, slot.reads);
-      slot.reads = EGL_NO_SYNC_KHR;
-    }
-    if (slot.texture) glDeleteTextures(1, &slot.texture);
-    slot.texture = 0;
-    if (slot.image != EGL_NO_IMAGE_KHR) egl().destroyImage(slot.display, slot.image);
-    slot.image = EGL_NO_IMAGE_KHR;
-    if (slot.buffer) AHardwareBuffer_release(slot.buffer);
-    slot.buffer = nullptr;
-    if (slot.reservation) MemoryBudget::instance().release(slot.reservation);
-    slot.reservation = 0;
-  }
-  slots.clear();
-  lent = -1;
-  encoding = -1;
-}
-
-} // namespace RNSkiaVideo

package/android/cpp/EncoderTargets.h

@@ -1,70 +0,0 @@
-#pragma once
-
-#include <EGL/egl.h>
-#include <EGL/eglext.h>
-#include <GLES2/gl2.h>
-#include <android/hardware_buffer.h>
-#include <array>
-#include <cstdint>
-#include <vector>
-
-namespace RNSkiaVideo {
-
-/**
- * RGBA AHardwareBuffers lent to JS for the GPU export: Skia renders into one
- * through WebGPU, then the encoder's EGL context samples it into the codec's
- * input Surface. No CPU readback or upload. Bounded, one lent at a time.
- *
- * Methods that touch GL objects require the encoder's EGL context current.
- * A buffer is lent again only once the GL reads of its previous frame are
- * complete; the renderer's own completion is the sync-fd it hands back.
- */
-class EncoderTargets {
-public:
-  static constexpr size_t kCapacity = 3;
-
-  EncoderTargets(int width, int height);
-  ~EncoderTargets();
-  EncoderTargets(const EncoderTargets&) = delete;
-  EncoderTargets& operator=(const EncoderTargets&) = delete;
-
-  /** A buffer without pending GL reads; allocates up to kCapacity. */
-  AHardwareBuffer* lend();
-  /** Returns the lent buffer unencoded. Its renderer must have finished. */
-  void giveBack();
-  bool isLent(const AHardwareBuffer* buffer) const;
-  /**
-   * Waits on the renderer's sync-fds (taking ownership of each, -1 meaning
-   * already signaled) and returns the lent buffer's texture, bound to its
-   * EGLImage in the current context. The buffer stays reserved until
-   * finishEncode().
-   */
-  GLuint beginEncode(const AHardwareBuffer* buffer, std::vector<int>& fences);
-  /** After the draw sampling the texture, with the encoder context current. */
-  void finishEncode();
-  /** Encoder context current: waits for pending reads and frees everything. */
-  void clear();
-  bool empty() const { return slots.empty(); }
-
-private:
-  struct Slot {
-    AHardwareBuffer* buffer = nullptr;
-    uint64_t reservation = 0;
-    EGLDisplay display = EGL_NO_DISPLAY;
-    EGLImageKHR image = EGL_NO_IMAGE_KHR;
-    GLuint texture = 0;
-    EGLSyncKHR reads = EGL_NO_SYNC_KHR;
-  };
-  int width;
-  int height;
-  std::vector<Slot> slots;
-  // The lent slot, then the slot being encoded; -1 when none.
-  int lent = -1;
-  int encoding = -1;
-
-  bool waitReads(Slot& slot, EGLTimeKHR timeout);
-  void allocate(Slot& slot);
-  void bindTexture(Slot& slot);
-};
-
-} // namespace RNSkiaVideo