react-native-skia-webgpu-video 0.0.0-stage → 0.1.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +303 -2
  3. package/android/BUILD_VALIDATION.md +62 -0
  4. package/android/CMakeLists.txt +99 -0
  5. package/android/build-validation.json +262 -0
  6. package/android/build.gradle +185 -0
  7. package/android/cpp/EncoderTargets.cpp +226 -0
  8. package/android/cpp/EncoderTargets.h +70 -0
  9. package/android/cpp/JNIHelpers.cpp +15 -0
  10. package/android/cpp/JNIHelpers.h +20 -0
  11. package/android/cpp/NativeEventDispatcher.cpp +59 -0
  12. package/android/cpp/NativeEventDispatcher.h +31 -0
  13. package/android/cpp/NativeHardwareBuffer.cpp +397 -0
  14. package/android/cpp/NativeHardwareBuffer.h +34 -0
  15. package/android/cpp/NativeRgbaBuffer.cpp +45 -0
  16. package/android/cpp/NativeRgbaBuffer.h +9 -0
  17. package/android/cpp/ReturnedVideoFrames.h +37 -0
  18. package/android/cpp/RgbaFrameLease.h +53 -0
  19. package/android/cpp/VideoCapabilities.cpp +87 -0
  20. package/android/cpp/VideoCapabilities.h +44 -0
  21. package/android/cpp/VideoComposition.cpp +170 -0
  22. package/android/cpp/VideoComposition.h +36 -0
  23. package/android/cpp/VideoCompositionFramesExtractor.cpp +82 -0
  24. package/android/cpp/VideoCompositionFramesExtractor.h +48 -0
  25. package/android/cpp/VideoCompositionFramesExtractorHostObject.cpp +230 -0
  26. package/android/cpp/VideoCompositionFramesExtractorHostObject.h +42 -0
  27. package/android/cpp/VideoCompositionFramesExtractorSyncHostObject.cpp +133 -0
  28. package/android/cpp/VideoCompositionFramesExtractorSyncHostObject.h +46 -0
  29. package/android/cpp/VideoEncoderHostObject.cpp +318 -0
  30. package/android/cpp/VideoEncoderHostObject.h +76 -0
  31. package/android/cpp/VideoFrame.cpp +293 -0
  32. package/android/cpp/VideoFrame.h +27 -0
  33. package/android/cpp/VideoPlayer.cpp +96 -0
  34. package/android/cpp/VideoPlayer.h +52 -0
  35. package/android/cpp/VideoPlayerHostObject.cpp +225 -0
  36. package/android/cpp/VideoPlayerHostObject.h +43 -0
  37. package/android/cpp/cpp-adapter.cpp +316 -0
  38. package/android/gradle.properties +5 -0
  39. package/android/instrumentation-inventory.json +178 -0
  40. package/android/proguard-rules.pro +4 -0
  41. package/android/src/main/AndroidManifest.xml +2 -0
  42. package/android/src/main/java/com/azzapp/rnskv/AudioCompositionExporter.java +537 -0
  43. package/android/src/main/java/com/azzapp/rnskv/AudioCompositionPlayer.java +321 -0
  44. package/android/src/main/java/com/azzapp/rnskv/BufferLifetime.java +55 -0
  45. package/android/src/main/java/com/azzapp/rnskv/DecodeRequests.java +100 -0
  46. package/android/src/main/java/com/azzapp/rnskv/DecoderWindow.java +55 -0
  47. package/android/src/main/java/com/azzapp/rnskv/EGLResourcesHolder.java +132 -0
  48. package/android/src/main/java/com/azzapp/rnskv/EGLUtils.java +225 -0
  49. package/android/src/main/java/com/azzapp/rnskv/FrameSize.java +50 -0
  50. package/android/src/main/java/com/azzapp/rnskv/GLFrameExtractor.java +510 -0
  51. package/android/src/main/java/com/azzapp/rnskv/JNIHelpers.java +12 -0
  52. package/android/src/main/java/com/azzapp/rnskv/NativeEventDispatcher.java +25 -0
  53. package/android/src/main/java/com/azzapp/rnskv/NativeHardwareBuffer.java +62 -0
  54. package/android/src/main/java/com/azzapp/rnskv/NativeRgbaBuffer.java +35 -0
  55. package/android/src/main/java/com/azzapp/rnskv/ReactNativeSkiaVideoModule.java +45 -0
  56. package/android/src/main/java/com/azzapp/rnskv/ReactNativeSkiaVideoPackage.java +43 -0
  57. package/android/src/main/java/com/azzapp/rnskv/ReactNativeSkiaVideoSpec.java +9 -0
  58. package/android/src/main/java/com/azzapp/rnskv/RgbaBufferAllocator.java +83 -0
  59. package/android/src/main/java/com/azzapp/rnskv/RgbaLayout.java +33 -0
  60. package/android/src/main/java/com/azzapp/rnskv/SurfaceOutputPolicy.java +13 -0
  61. package/android/src/main/java/com/azzapp/rnskv/TextureRenderer.java +150 -0
  62. package/android/src/main/java/com/azzapp/rnskv/TimeHelpers.java +11 -0
  63. package/android/src/main/java/com/azzapp/rnskv/VideoCapabilities.java +354 -0
  64. package/android/src/main/java/com/azzapp/rnskv/VideoComposition.java +147 -0
  65. package/android/src/main/java/com/azzapp/rnskv/VideoCompositionDecoder.java +772 -0
  66. package/android/src/main/java/com/azzapp/rnskv/VideoCompositionFramesExtractor.java +354 -0
  67. package/android/src/main/java/com/azzapp/rnskv/VideoCompositionFramesExtractorSync.java +241 -0
  68. package/android/src/main/java/com/azzapp/rnskv/VideoCompositionItemDecoder.java +492 -0
  69. package/android/src/main/java/com/azzapp/rnskv/VideoEncoder.java +639 -0
  70. package/android/src/main/java/com/azzapp/rnskv/VideoFrame.java +74 -0
  71. package/android/src/main/java/com/azzapp/rnskv/VideoPlayer.java +361 -0
  72. package/cpp/RNSVCheckedSizes.h +54 -0
  73. package/cpp/RNSVEventEmitter.cpp +199 -0
  74. package/cpp/RNSVEventEmitter.h +31 -0
  75. package/cpp/RNSVHostObject.cpp +79 -0
  76. package/cpp/RNSVHostObject.h +67 -0
  77. package/cpp/RNSVMemoryBudget.cpp +68 -0
  78. package/cpp/RNSVMemoryBudget.h +100 -0
  79. package/cpp/RNSVRuntimeLifecycleMonitor.cpp +88 -0
  80. package/cpp/RNSVRuntimeLifecycleMonitor.h +33 -0
  81. package/docs/ANDROID_DEVICE_306.md +95 -0
  82. package/docs/ANDROID_EXPORT_CACHE_306.md +48 -0
  83. package/docs/ANDROID_MEMORY_RETENTION_306.md +67 -0
  84. package/docs/BENCHMARKS.md +368 -0
  85. package/docs/IOS_DEVICE_TESTS.md +115 -0
  86. package/docs/IOS_SIMULATOR_TESTS.md +74 -0
  87. package/docs/MIGRATION.md +82 -0
  88. package/docs/NATIVE_MEMORY_INVESTIGATION_306.md +88 -0
  89. package/docs/PERFORMANCE_AB_IPHONE_303.md +196 -0
  90. package/docs/PERFORMANCE_AB_SIMULATOR.md +135 -0
  91. package/docs/PERFORMANCE_AB_SOAK_SIMULATOR.md +84 -0
  92. package/docs/PERFORMANCE_IPHONE_306_OPTIMIZATION.md +76 -0
  93. package/docs/PERFORMANCE_SIMULATOR.md +89 -0
  94. package/docs/PERFORMANCE_SIMULATOR_INITIAL.md +96 -0
  95. package/docs/PERFORMANCE_SOAK_IPHONE_303.md +41 -0
  96. package/docs/UPSTREAM_NOTES.md +188 -0
  97. package/docs/assets/banner.svg +26 -0
  98. package/examples/webgpu-overlay.ts +147 -0
  99. package/ios/AVAssetTrackUtils.h +24 -0
  100. package/ios/AudioCompositionUtils.h +27 -0
  101. package/ios/AudioCompositionUtils.mm +113 -0
  102. package/ios/DecoderWindow.h +52 -0
  103. package/ios/NativeResourceStats.h +32 -0
  104. package/ios/RNSVColorSpace.h +50 -0
  105. package/ios/RNSVJSIUtils.h +46 -0
  106. package/ios/RNSVScopedPixelBuffer.h +17 -0
  107. package/ios/RNSVVideoPlayer.h +46 -0
  108. package/ios/RNSVVideoPlayer.mm +410 -0
  109. package/ios/ReactNativeSkiaVideo.h +8 -0
  110. package/ios/ReactNativeSkiaVideo.mm +310 -0
  111. package/ios/SeekPolicy.h +27 -0
  112. package/ios/VideoComposition.h +148 -0
  113. package/ios/VideoCompositionFramesExtractorHostObject.h +105 -0
  114. package/ios/VideoCompositionFramesExtractorHostObject.mm +686 -0
  115. package/ios/VideoCompositionFramesExtractorSyncHostObject.h +37 -0
  116. package/ios/VideoCompositionFramesExtractorSyncHostObject.mm +168 -0
  117. package/ios/VideoCompositionItemDecoder.h +60 -0
  118. package/ios/VideoCompositionItemDecoder.mm +480 -0
  119. package/ios/VideoEncoderHostObject.h +83 -0
  120. package/ios/VideoEncoderHostObject.mm +706 -0
  121. package/ios/VideoFrame.h +66 -0
  122. package/ios/VideoFrame.mm +108 -0
  123. package/ios/VideoPlayerHostObject.h +52 -0
  124. package/ios/VideoPlayerHostObject.mm +332 -0
  125. package/lib/module/NativeReactNativeSkiaVideo.js +5 -0
  126. package/lib/module/NativeReactNativeSkiaVideo.js.map +1 -0
  127. package/lib/module/RNSkiaVideoModule.js +20 -0
  128. package/lib/module/RNSkiaVideoModule.js.map +1 -0
  129. package/lib/module/canvas.js +18 -0
  130. package/lib/module/canvas.js.map +1 -0
  131. package/lib/module/exportCache.js +22 -0
  132. package/lib/module/exportCache.js.map +1 -0
  133. package/lib/module/exportFrames.js +338 -0
  134. package/lib/module/exportFrames.js.map +1 -0
  135. package/lib/module/exportQueue.js +52 -0
  136. package/lib/module/exportQueue.js.map +1 -0
  137. package/lib/module/exportTarget.js +137 -0
  138. package/lib/module/exportTarget.js.map +1 -0
  139. package/lib/module/exportVideoComposition.js +492 -0
  140. package/lib/module/exportVideoComposition.js.map +1 -0
  141. package/lib/module/frameInterop.js +532 -0
  142. package/lib/module/frameInterop.js.map +1 -0
  143. package/lib/module/gpu.js +222 -0
  144. package/lib/module/gpu.js.map +1 -0
  145. package/lib/module/gpuDevice.js +54 -0
  146. package/lib/module/gpuDevice.js.map +1 -0
  147. package/lib/module/index.js +36 -0
  148. package/lib/module/index.js.map +1 -0
  149. package/lib/module/memory.js +38 -0
  150. package/lib/module/memory.js.map +1 -0
  151. package/lib/module/package.json +1 -0
  152. package/lib/module/previewLifecycle.js +52 -0
  153. package/lib/module/previewLifecycle.js.map +1 -0
  154. package/lib/module/runtimeYield.js +10 -0
  155. package/lib/module/runtimeYield.js.map +1 -0
  156. package/lib/module/types.js +4 -0
  157. package/lib/module/types.js.map +1 -0
  158. package/lib/module/utils/useEventListener.js +18 -0
  159. package/lib/module/utils/useEventListener.js.map +1 -0
  160. package/lib/module/videoCompositionPlayer.js +348 -0
  161. package/lib/module/videoCompositionPlayer.js.map +1 -0
  162. package/lib/module/videoFrameImage.js +185 -0
  163. package/lib/module/videoFrameImage.js.map +1 -0
  164. package/lib/module/videoPlayer.js +172 -0
  165. package/lib/module/videoPlayer.js.map +1 -0
  166. package/lib/module/yuvBlit.js +299 -0
  167. package/lib/module/yuvBlit.js.map +1 -0
  168. package/lib/typescript/package.json +1 -0
  169. package/lib/typescript/src/NativeReactNativeSkiaVideo.d.ts +7 -0
  170. package/lib/typescript/src/NativeReactNativeSkiaVideo.d.ts.map +1 -0
  171. package/lib/typescript/src/RNSkiaVideoModule.d.ts +4 -0
  172. package/lib/typescript/src/RNSkiaVideoModule.d.ts.map +1 -0
  173. package/lib/typescript/src/canvas.d.ts +4 -0
  174. package/lib/typescript/src/canvas.d.ts.map +1 -0
  175. package/lib/typescript/src/exportCache.d.ts +6 -0
  176. package/lib/typescript/src/exportCache.d.ts.map +1 -0
  177. package/lib/typescript/src/exportFrames.d.ts +57 -0
  178. package/lib/typescript/src/exportFrames.d.ts.map +1 -0
  179. package/lib/typescript/src/exportQueue.d.ts +5 -0
  180. package/lib/typescript/src/exportQueue.d.ts.map +1 -0
  181. package/lib/typescript/src/exportTarget.d.ts +28 -0
  182. package/lib/typescript/src/exportTarget.d.ts.map +1 -0
  183. package/lib/typescript/src/exportVideoComposition.d.ts +68 -0
  184. package/lib/typescript/src/exportVideoComposition.d.ts.map +1 -0
  185. package/lib/typescript/src/frameInterop.d.ts +17 -0
  186. package/lib/typescript/src/frameInterop.d.ts.map +1 -0
  187. package/lib/typescript/src/gpu.d.ts +39 -0
  188. package/lib/typescript/src/gpu.d.ts.map +1 -0
  189. package/lib/typescript/src/gpuDevice.d.ts +11 -0
  190. package/lib/typescript/src/gpuDevice.d.ts.map +1 -0
  191. package/lib/typescript/src/index.d.ts +21 -0
  192. package/lib/typescript/src/index.d.ts.map +1 -0
  193. package/lib/typescript/src/memory.d.ts +25 -0
  194. package/lib/typescript/src/memory.d.ts.map +1 -0
  195. package/lib/typescript/src/previewLifecycle.d.ts +22 -0
  196. package/lib/typescript/src/previewLifecycle.d.ts.map +1 -0
  197. package/lib/typescript/src/runtimeYield.d.ts +4 -0
  198. package/lib/typescript/src/runtimeYield.d.ts.map +1 -0
  199. package/lib/typescript/src/types.d.ts +631 -0
  200. package/lib/typescript/src/types.d.ts.map +1 -0
  201. package/lib/typescript/src/utils/useEventListener.d.ts +5 -0
  202. package/lib/typescript/src/utils/useEventListener.d.ts.map +1 -0
  203. package/lib/typescript/src/videoCompositionPlayer.d.ts +85 -0
  204. package/lib/typescript/src/videoCompositionPlayer.d.ts.map +1 -0
  205. package/lib/typescript/src/videoFrameImage.d.ts +60 -0
  206. package/lib/typescript/src/videoFrameImage.d.ts.map +1 -0
  207. package/lib/typescript/src/videoPlayer.d.ts +99 -0
  208. package/lib/typescript/src/videoPlayer.d.ts.map +1 -0
  209. package/lib/typescript/src/yuvBlit.d.ts +50 -0
  210. package/lib/typescript/src/yuvBlit.d.ts.map +1 -0
  211. package/package.json +154 -4
  212. package/patches/react-native-skia+3.7.0.patch +502 -0
  213. package/react-native-skia-webgpu-video.podspec +35 -0
  214. package/scripts/apply-skia-canvas-patch.mjs +48 -0
  215. package/scripts/skia-canvas-patch.json +291 -0
  216. package/src/NativeReactNativeSkiaVideo.ts +8 -0
  217. package/src/RNSkiaVideoModule.ts +27 -0
  218. package/src/canvas.ts +22 -0
  219. package/src/exportCache.ts +27 -0
  220. package/src/exportFrames.ts +421 -0
  221. package/src/exportQueue.ts +65 -0
  222. package/src/exportTarget.ts +179 -0
  223. package/src/exportVideoComposition.ts +718 -0
  224. package/src/frameInterop.ts +684 -0
  225. package/src/gpu.ts +290 -0
  226. package/src/gpuDevice.ts +80 -0
  227. package/src/index.tsx +45 -0
  228. package/src/memory.ts +48 -0
  229. package/src/previewLifecycle.ts +60 -0
  230. package/src/runtimeYield.ts +6 -0
  231. package/src/types.ts +661 -0
  232. package/src/utils/useEventListener.ts +22 -0
  233. package/src/videoCompositionPlayer.ts +520 -0
  234. package/src/videoFrameImage.ts +205 -0
  235. package/src/videoPlayer.ts +318 -0
  236. package/src/yuvBlit.ts +331 -0
@@ -0,0 +1,510 @@
1
+ package com.azzapp.rnskv;
2
+
3
+ import android.graphics.ImageFormat;
4
+ import android.graphics.Rect;
5
+ import android.graphics.SurfaceTexture;
6
+ import android.hardware.HardwareBuffer;
7
+ import android.hardware.SyncFence;
8
+ import android.media.Image;
9
+ import android.media.ImageReader;
10
+ import android.opengl.GLES11Ext;
11
+ import android.opengl.GLES20;
12
+ import android.os.Build;
13
+ import android.os.Handler;
14
+ import android.os.HandlerThread;
15
+ import android.view.Surface;
16
+ import java.io.IOException;
17
+ import java.time.Duration;
18
+ import java.util.concurrent.atomic.AtomicBoolean;
19
+ import java.nio.ByteBuffer;
20
+ import java.util.concurrent.atomic.AtomicLong;
21
+
22
+ /**
23
+ * A class that extracts frames from a SurfaceTexture streaming to an external texture and renders
24
+ * them to a 2D texture.
25
+ */
26
+ public class GLFrameExtractor implements SurfaceTexture.OnFrameAvailableListener {
27
+ private static final AtomicLong nextProducerId = new AtomicLong(1);
28
+ private final long producerId = nextProducerId.getAndIncrement();
29
+
30
+ private final AtomicBoolean frameAvailable = new AtomicBoolean(false);
31
+
32
+ private final float[] transformMatrix = new float[16];
33
+
34
+ private final Surface surface;
35
+
36
+ private final SurfaceTexture surfaceTexture;
37
+
38
+ private int frameWidth = -1;
39
+
40
+ private int frameHeight = -1;
41
+
42
+ private final int inputTexId;
43
+
44
+ private final int outputTexId;
45
+
46
+ private final int frameBuffer;
47
+
48
+ private final TextureRenderer textureRenderer;
49
+
50
+ private OnFrameAvailableListener onFrameAvailableListener;
51
+
52
+ private long latestTimeStampNs = -1;
53
+
54
+ private final boolean direct;
55
+ private final boolean hardwareBuffer;
56
+ private long pendingHardwareFrame;
57
+
58
+ private volatile boolean released;
59
+
60
+ private int bufferWidth = -1;
61
+
62
+ private int bufferHeight = -1;
63
+
64
+ // ImageReader mode (API 29+): the codec writes GPU-sampled YUV buffers that
65
+ // WebGPU imports directly, with no EGL render pass or glFinish per frame.
66
+ // Enough images for the codec plus the frames still leased downstream.
67
+ private static final int READER_IMAGES = 4;
68
+ private static Handler readerCallbacks;
69
+ private ImageReader imageReader;
70
+ private int imageRotation;
71
+
72
+ public GLFrameExtractor() {
73
+ this(false);
74
+ }
75
+
76
+ /**
77
+ * @param direct whether frames are handed over as the decoder's buffer, in
78
+ * the input texture, instead of being drawn into the output
79
+ * texture, whose storage is then never allocated
80
+ */
81
+ public GLFrameExtractor(boolean direct) {
82
+ this(direct, false);
83
+ }
84
+
85
+ public GLFrameExtractor(boolean direct, boolean hardwareBuffer) {
86
+ // Both historical modes use the correctness reference backend. An OES
87
+ // texture belongs to this isolated EGL context and cannot be imported by Graphite.
88
+ this.direct = false;
89
+ this.hardwareBuffer = hardwareBuffer;
90
+ EGLUtils.purgeOpenGLError();
91
+
92
+
93
+
94
+ int[] texIds = new int[2];
95
+ GLES20.glGenTextures(2, texIds,0);
96
+
97
+ inputTexId = texIds[0];
98
+ EGLUtils.configureTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, inputTexId);
99
+
100
+ outputTexId = texIds[1];
101
+ EGLUtils.configureTexture(GLES20.GL_TEXTURE_2D, outputTexId);
102
+
103
+ int[] bufferIds = new int[1];
104
+ GLES20.glGenFramebuffers(1, bufferIds, 0);
105
+ frameBuffer = bufferIds[0];
106
+
107
+ EGLUtils.checkGlError("GLFrameExtractor()");
108
+
109
+ textureRenderer = new TextureRenderer(true);
110
+
111
+ surfaceTexture = new SurfaceTexture(inputTexId);
112
+ surfaceTexture.setOnFrameAvailableListener(this);
113
+ surface = new Surface(surfaceTexture);
114
+ }
115
+
116
+ private static synchronized Handler readerCallbacks() {
117
+ if (readerCallbacks == null) {
118
+ HandlerThread thread = new HandlerThread("ReactNativeSkiaVideo-ImageReader");
119
+ thread.setDaemon(true);
120
+ thread.start();
121
+ readerCallbacks = new Handler(thread.getLooper());
122
+ }
123
+ return readerCallbacks;
124
+ }
125
+
126
+ /**
127
+ * The Surface the codec renders into. With {@code nativeImages}, on API 29+
128
+ * with hardware buffers, it is an ImageReader's: frames then keep the codec's
129
+ * YUV buffer, cropped and turned by {@code rotation} when WebGPU samples it,
130
+ * so they must be wanted at the encoded size.
131
+ */
132
+ @android.annotation.TargetApi(Build.VERSION_CODES.Q)
133
+ public Surface surfaceFor(int videoWidth, int videoHeight, int rotation, boolean nativeImages) {
134
+ if (!nativeImages || !hardwareBuffer || Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
135
+ return surface;
136
+ }
137
+ synchronized (this) {
138
+ if (released) return surface;
139
+ if (imageReader == null) {
140
+ ImageReader reader = ImageReader.newInstance(videoWidth, videoHeight,
141
+ ImageFormat.YUV_420_888, READER_IMAGES, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
142
+ reader.setOnImageAvailableListener(source -> onFrameAvailable(surfaceTexture),
143
+ readerCallbacks());
144
+ imageRotation = rotation;
145
+ imageReader = reader;
146
+ }
147
+ return imageReader.getSurface();
148
+ }
149
+ }
150
+
151
+ @android.annotation.TargetApi(Build.VERSION_CODES.Q)
152
+ private boolean decodeImage(ImageReader reader, int width, int height) {
153
+ Image image;
154
+ try {
155
+ image = reader.acquireLatestImage();
156
+ } catch (IllegalStateException everyImageLeased) {
157
+ // Downstream still holds them all; retry once a lease is released.
158
+ frameAvailable.set(true);
159
+ return false;
160
+ }
161
+ if (image == null) return false;
162
+ boolean owned = true;
163
+ try {
164
+ if (Build.VERSION.SDK_INT >= 33) awaitProducer(image);
165
+ Rect crop = image.getCropRect();
166
+ boolean swapped = imageRotation == 90 || imageRotation == 270;
167
+ int pictureWidth = swapped ? crop.height() : crop.width();
168
+ int pictureHeight = swapped ? crop.width() : crop.height();
169
+ if (pictureWidth != width || pictureHeight != height) {
170
+ throw new IllegalStateException("Decoded image " + pictureWidth + "x" + pictureHeight
171
+ + " does not match the requested " + width + "x" + height);
172
+ }
173
+ HardwareBuffer buffer = image.getHardwareBuffer();
174
+ if (buffer == null) throw new IllegalStateException("Decoded image has no hardware buffer");
175
+ long lease;
176
+ try {
177
+ lease = NativeHardwareBuffer.wrapImage(buffer, image, crop.left, crop.top,
178
+ crop.width(), crop.height(), imageRotation);
179
+ } finally {
180
+ buffer.close();
181
+ }
182
+ owned = false;
183
+ NativeHardwareBuffer.release(pendingHardwareFrame);
184
+ pendingHardwareFrame = lease;
185
+ latestTimeStampNs = image.getTimestamp();
186
+ frameWidth = width;
187
+ frameHeight = height;
188
+ return true;
189
+ } finally {
190
+ if (owned) image.close();
191
+ }
192
+ }
193
+
194
+ /** WebGPU's frame wrapper takes no producer fence: wait for it here. */
195
+ @android.annotation.TargetApi(Build.VERSION_CODES.TIRAMISU)
196
+ private static void awaitProducer(Image image) {
197
+ if (Build.VERSION.SDK_INT < 33) return;
198
+ try (SyncFence fence = image.getFence()) {
199
+ if (fence.isValid() && !fence.await(Duration.ofSeconds(5))) {
200
+ throw new IllegalStateException("Decoded image was not ready within 5 seconds");
201
+ }
202
+ } catch (IOException error) {
203
+ throw new IllegalStateException("Cannot read the decoded image's fence", error);
204
+ }
205
+ }
206
+
207
+ /**
208
+ * Set the listener that will be called when a new frame is available.
209
+ * @param onFrameAvailableListener the listener to set
210
+ */
211
+ public void setOnFrameAvailableListener(OnFrameAvailableListener onFrameAvailableListener) {
212
+ this.onFrameAvailableListener = onFrameAvailableListener;
213
+ }
214
+
215
+ /**
216
+ * @return whether a frame rendered by the producer is waiting to be decoded.
217
+ * Thread-safe and cheap, it lets callers skip the EGL context switch of
218
+ * {@link #decodeNextFrame} when there is nothing new.
219
+ */
220
+ public boolean hasPendingFrame() {
221
+ return frameAvailable.get();
222
+ }
223
+
224
+ /**
225
+ * Decode the next frame and render it to the output texture.
226
+ * @param width the width of the frame
227
+ * @param height the height of the frame
228
+ * @return true if a new frame was decoded, false otherwise
229
+ */
230
+ public boolean decodeNextFrame(int width, int height) {
231
+ if (released) {
232
+ return false;
233
+ }
234
+ if(!frameAvailable.compareAndSet(true, false)) {
235
+ return false;
236
+ }
237
+
238
+ ImageReader reader;
239
+ synchronized (this) {
240
+ reader = imageReader;
241
+ }
242
+ if (reader != null) {
243
+ return decodeImage(reader, width, height);
244
+ }
245
+
246
+ EGLUtils.purgeOpenGLError();
247
+
248
+ if (hardwareBuffer) {
249
+ RgbaLayout.byteSize(width, height);
250
+ surfaceTexture.updateTexImage();
251
+ latestTimeStampNs = surfaceTexture.getTimestamp();
252
+ surfaceTexture.getTransformMatrix(transformMatrix);
253
+ long writer = NativeHardwareBuffer.createRenderTarget(width, height);
254
+ Throwable primaryError = null;
255
+ try {
256
+ int texture = NativeHardwareBuffer.renderTargetTexture(writer);
257
+ GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, frameBuffer);
258
+ GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
259
+ GLES20.GL_TEXTURE_2D, texture, 0);
260
+ if (GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER) != GLES20.GL_FRAMEBUFFER_COMPLETE)
261
+ throw new IllegalStateException("Hardware decoder framebuffer is incomplete");
262
+ GLES20.glClearColor(0, 0, 0, 0);
263
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
264
+ GLES20.glViewport(0, 0, width, height);
265
+ textureRenderer.draw(inputTexId, transformMatrix);
266
+ EGLUtils.checkGlError("Hardware decoder draw");
267
+ // Detach before native texture/EGLImage cleanup so this FBO cannot keep
268
+ // an unaccounted backing alive after its allocation owner is released.
269
+ GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
270
+ GLES20.GL_TEXTURE_2D, 0, 0);
271
+ long ready = NativeHardwareBuffer.finishRenderTarget(writer);
272
+ writer = 0;
273
+ NativeHardwareBuffer.release(pendingHardwareFrame);
274
+ pendingHardwareFrame = ready;
275
+ frameWidth = width; frameHeight = height;
276
+ return true;
277
+ } catch (Throwable error) {
278
+ primaryError = error;
279
+ throw error;
280
+ } finally {
281
+ GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, frameBuffer);
282
+ GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
283
+ GLES20.GL_TEXTURE_2D, 0, 0);
284
+ GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
285
+ if (writer != 0) {
286
+ try { NativeHardwareBuffer.abortRenderTarget(writer); }
287
+ catch (RuntimeException cleanupError) {
288
+ if (primaryError == null) throw cleanupError;
289
+ primaryError.addSuppressed(cleanupError);
290
+ }
291
+ }
292
+ }
293
+ }
294
+
295
+ if (direct) {
296
+ surfaceTexture.updateTexImage();
297
+ latestTimeStampNs = surfaceTexture.getTimestamp();
298
+ surfaceTexture.getTransformMatrix(transformMatrix);
299
+ measureBuffer(width, height);
300
+ return true;
301
+ }
302
+
303
+ if (width != frameWidth || height != frameHeight) {
304
+ frameWidth = width;
305
+ frameHeight = height;
306
+ GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, outputTexId);
307
+ GLES20.glTexImage2D(
308
+ GLES20.GL_TEXTURE_2D,
309
+ 0,
310
+ GLES20.GL_RGBA,
311
+ width, height,
312
+ 0,
313
+ GLES20.GL_RGBA,
314
+ GLES20.GL_UNSIGNED_BYTE,
315
+ null
316
+ );
317
+ }
318
+ surfaceTexture.updateTexImage();
319
+ latestTimeStampNs = surfaceTexture.getTimestamp();
320
+ surfaceTexture.getTransformMatrix(transformMatrix);
321
+
322
+ GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, frameBuffer);
323
+ GLES20.glFramebufferTexture2D(
324
+ GLES20.GL_FRAMEBUFFER,
325
+ GLES20.GL_COLOR_ATTACHMENT0,
326
+ GLES20.GL_TEXTURE_2D,
327
+ outputTexId,
328
+ 0
329
+ );
330
+ GLES20.glClearColor(0,0,0,0);
331
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
332
+ GLES20.glViewport(0, 0, width, height);
333
+ textureRenderer.draw(inputTexId, transformMatrix);
334
+ EGLUtils.checkGlError("GLFrameExtractor.draw()");
335
+ GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
336
+ return true;
337
+ }
338
+
339
+
340
+ /**
341
+ * The decoder's buffer is often larger than the picture (1920x1088 for a
342
+ * 1080p H.264 stream), and the transform maps the picture into it, shrunk by
343
+ * up to a texel on each side against bilinear bleeding. The padding is at the
344
+ * right and the bottom, so the picture's two ends along each texture axis,
345
+ * summed, give its share of the buffer along that axis.
346
+ *
347
+ * @param pictureWidth the picture's width, in the buffer's orientation
348
+ * @param pictureHeight the picture's height, in the buffer's orientation
349
+ */
350
+ private void measureBuffer(int pictureWidth, int pictureHeight) {
351
+ float[] m = transformMatrix;
352
+ float minS = Float.MAX_VALUE;
353
+ float maxS = -Float.MAX_VALUE;
354
+ float minT = Float.MAX_VALUE;
355
+ float maxT = -Float.MAX_VALUE;
356
+ for (int corner = 0; corner < 4; corner++) {
357
+ float s = corner & 1;
358
+ float t = corner >> 1;
359
+ float u = m[0] * s + m[4] * t + m[12];
360
+ float v = m[1] * s + m[5] * t + m[13];
361
+ minS = Math.min(minS, u);
362
+ maxS = Math.max(maxS, u);
363
+ minT = Math.min(minT, v);
364
+ maxT = Math.max(maxT, v);
365
+ }
366
+ float sumS = minS + maxS;
367
+ float sumT = minT + maxT;
368
+ bufferWidth = Math.max(pictureWidth, sumS > 0 ? Math.round(pictureWidth / sumS) : pictureWidth);
369
+ bufferHeight = Math.max(pictureHeight, sumT > 0 ? Math.round(pictureHeight / sumT) : pictureHeight);
370
+ }
371
+
372
+ /**
373
+ * @return whether frames are handed over in the input texture
374
+ */
375
+ public boolean isDirect() {
376
+ return direct;
377
+ }
378
+
379
+ /**
380
+ * @return the external texture the decoder's buffers are bound to
381
+ */
382
+ public int getInputTexId() {
383
+ return inputTexId;
384
+ }
385
+
386
+ /**
387
+ * @return the width of the decoder's buffer, in direct mode, once a frame
388
+ * was decoded
389
+ */
390
+ public int getBufferWidth() {
391
+ return bufferWidth;
392
+ }
393
+
394
+ /**
395
+ * @return the height of the decoder's buffer, in direct mode, once a frame
396
+ * was decoded
397
+ */
398
+ public int getBufferHeight() {
399
+ return bufferHeight;
400
+ }
401
+
402
+ /**
403
+ * Get the name of the texture that contains the output frame.
404
+ */
405
+ public int getOutputTexId() {
406
+ return outputTexId;
407
+ }
408
+
409
+ /** Copies the rendered pixels while this extractor's private EGL context is current. */
410
+ public VideoFrame snapshotFrame(int rotation) {
411
+ if (hardwareBuffer) {
412
+ long ready = pendingHardwareFrame;
413
+ pendingHardwareFrame = 0;
414
+ if (ready == 0) throw new IllegalStateException("No completed hardware video frame");
415
+ try {
416
+ return new VideoFrame(ready, frameWidth, frameHeight, rotation, latestTimeStampNs, producerId);
417
+ } catch (Throwable error) { NativeHardwareBuffer.release(ready); throw error; }
418
+ }
419
+ int size = RgbaLayout.byteSize(frameWidth, frameHeight);
420
+ // One owned allocation is written once by GL and never recycled by this decoder.
421
+ ByteBuffer ownedPixels = NativeRgbaBuffer.allocate(size);
422
+ GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, frameBuffer);
423
+ GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
424
+ GLES20.GL_TEXTURE_2D, outputTexId, 0);
425
+ try {
426
+ if (GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER)
427
+ != GLES20.GL_FRAMEBUFFER_COMPLETE) {
428
+ throw new IllegalStateException("RGBA decoder framebuffer is incomplete");
429
+ }
430
+ GLES20.glPixelStorei(GLES20.GL_PACK_ALIGNMENT, 1);
431
+ GLES20.glReadPixels(0, 0, frameWidth, frameHeight, GLES20.GL_RGBA,
432
+ GLES20.GL_UNSIGNED_BYTE, ownedPixels);
433
+ EGLUtils.checkGlError("decoder RGBA readback");
434
+ return new VideoFrame(ownedPixels, frameWidth, frameHeight, rotation, latestTimeStampNs,
435
+ producerId);
436
+ } finally {
437
+ GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
438
+ }
439
+ }
440
+
441
+ /**
442
+ * Get the surface that can be used to render to the output texture.
443
+ */
444
+ public Surface getSurface() {
445
+ return surface;
446
+ }
447
+
448
+ @Override
449
+ public void onFrameAvailable(SurfaceTexture surfaceTexture) {
450
+ if (released) {
451
+ return;
452
+ }
453
+ frameAvailable.set(true);
454
+ if (onFrameAvailableListener != null) {
455
+ onFrameAvailableListener.onFrameAvailable();
456
+ }
457
+ }
458
+
459
+ public long getLatestTimeStampNs() {
460
+ return latestTimeStampNs;
461
+ }
462
+
463
+ /**
464
+ * Release the surface and the GL objects. The EGL context they were created
465
+ * with must be current on the calling thread: without it the GL deletes are
466
+ * silently ignored and the textures live on in the share group.
467
+ */
468
+ public void release() {
469
+ synchronized (this) {
470
+ if (released) {
471
+ return;
472
+ }
473
+ released = true;
474
+ }
475
+ NativeHardwareBuffer.release(pendingHardwareFrame);
476
+ pendingHardwareFrame = 0;
477
+ ImageReader reader;
478
+ synchronized (this) {
479
+ reader = imageReader;
480
+ imageReader = null;
481
+ }
482
+ if (reader != null) {
483
+ // Leased frames keep their own buffer reference; the codec is gone.
484
+ reader.setOnImageAvailableListener(null, null);
485
+ reader.close();
486
+ }
487
+ onFrameAvailableListener = null;
488
+ frameAvailable.set(false);
489
+ if (surfaceTexture != null) {
490
+ surfaceTexture.release();
491
+ }
492
+ if (surface != null) {
493
+ surface.release();
494
+ }
495
+ if (frameBuffer != -1) {
496
+ GLES20.glDeleteFramebuffers(1, new int[]{frameBuffer}, 0);
497
+ }
498
+ if (inputTexId != -1) {
499
+ GLES20.glDeleteTextures(2, new int[]{inputTexId, outputTexId}, 0);
500
+ }
501
+ textureRenderer.release();
502
+ }
503
+
504
+ /**
505
+ * A listener that will be called when a new frame is available.
506
+ */
507
+ public interface OnFrameAvailableListener {
508
+ void onFrameAvailable();
509
+ }
510
+ }
@@ -0,0 +1,12 @@
1
+ package com.azzapp.rnskv;
2
+
3
+ /**
4
+ * Helper class to get the Skia platform context.
5
+ */
6
+ public class JNIHelpers {
7
+
8
+ static public Object getCallInvoker() {
9
+ return ReactNativeSkiaVideoModule.currentReactApplicationContext()
10
+ .getCatalystInstance().getJSCallInvokerHolder();
11
+ }
12
+ }
@@ -0,0 +1,25 @@
1
+ package com.azzapp.rnskv;
2
+
3
+ public class NativeEventDispatcher {
4
+
5
+ private long eventReceiver;
6
+
7
+
8
+ public NativeEventDispatcher(long eventReceiver) {
9
+ this.eventReceiver = eventReceiver;
10
+ }
11
+
12
+ public synchronized void dispatchEvent(String eventName, Object data) {
13
+ if (eventReceiver != 0) nativeDispatchEvent(eventReceiver, eventName, data);
14
+ }
15
+
16
+ // Synchronizes with an in-flight JNI callback before its C++ receiver dies.
17
+ public synchronized void invalidate() {
18
+ if (eventReceiver != 0) nativeInvalidateReceiver(eventReceiver);
19
+ eventReceiver = 0;
20
+ }
21
+
22
+ private static native void nativeDispatchEvent(long eventReceiver, String eventName, Object data);
23
+ private static native void nativeInvalidateReceiver(long eventReceiver);
24
+
25
+ }
@@ -0,0 +1,62 @@
1
+ package com.azzapp.rnskv;
2
+
3
+ import java.nio.ByteBuffer;
4
+ import java.util.concurrent.atomic.AtomicLong;
5
+
6
+ /** Private immutable AHB ownership. Native writers may only be used on their EGL thread. */
7
+ final class NativeHardwareBuffer {
8
+ static { System.loadLibrary("react-native-skia-video"); }
9
+ private static final BufferLifetime lifetimes = new BufferLifetime(NativeHardwareBuffer::nativeRelease);
10
+ static {
11
+ Thread reaper = new Thread(lifetimes::run, "SkiaVideoHardwareBufferReaper");
12
+ reaper.setDaemon(true);
13
+ reaper.start();
14
+ }
15
+ private NativeHardwareBuffer() {}
16
+
17
+ static final class Lease {
18
+ private final AtomicLong handle;
19
+ Lease(long handle) {
20
+ if (handle <= 0) throw new IllegalArgumentException("Invalid hardware frame lease");
21
+ try {
22
+ this.handle = new AtomicLong(handle);
23
+ lifetimes.track(this, null, handle);
24
+ }
25
+ catch (Throwable error) { nativeRelease(handle); throw error; }
26
+ }
27
+ synchronized long get() {
28
+ long value = handle.get();
29
+ if (value == 0) throw new IllegalStateException("Hardware video frame is disposed");
30
+ return value;
31
+ }
32
+ synchronized long retain() { return nativeRetain(get()); }
33
+ synchronized void read(ByteBuffer destination) { nativeReadPixels(get(), destination); }
34
+ synchronized void close() {
35
+ long value = handle.getAndSet(0);
36
+ if (value != 0) nativeRelease(value);
37
+ }
38
+ }
39
+
40
+ static boolean isEnabled() { return nativeIsEnabled(); }
41
+ static long createRenderTarget(int width, int height) { return nativeCreateRenderTarget(width, height); }
42
+ static int renderTargetTexture(long handle) { return nativeRenderTargetTexture(handle); }
43
+ static long finishRenderTarget(long handle) { return nativeFinishRenderTarget(handle); }
44
+ static void abortRenderTarget(long handle) { nativeAbortRenderTarget(handle); }
45
+ static void release(long handle) { if (handle != 0) nativeRelease(handle); }
46
+ /** Takes over {@code image} on success; on failure the caller still owns it. */
47
+ static long wrapImage(android.hardware.HardwareBuffer buffer, android.media.Image image,
48
+ int cropX, int cropY, int cropWidth, int cropHeight, int rotation) {
49
+ return nativeWrapImage(buffer, image, cropX, cropY, cropWidth, cropHeight, rotation);
50
+ }
51
+ private static native boolean nativeIsEnabled();
52
+ private static native long nativeCreateRenderTarget(int width, int height);
53
+ private static native int nativeRenderTargetTexture(long handle);
54
+ private static native long nativeFinishRenderTarget(long handle);
55
+ private static native void nativeAbortRenderTarget(long handle);
56
+ private static native long nativeRetain(long handle);
57
+ private static native void nativeRelease(long handle);
58
+ private static native void nativeReadPixels(long handle, ByteBuffer destination);
59
+ // fbjni registers alias_ref<jobject> parameters as java.lang.Object.
60
+ private static native long nativeWrapImage(Object buffer, Object image, int cropX, int cropY,
61
+ int cropWidth, int cropHeight, int rotation);
62
+ }
@@ -0,0 +1,35 @@
1
+ package com.azzapp.rnskv;
2
+
3
+ import java.nio.ByteBuffer;
4
+
5
+ /** Owned pixel storage; no full-frame memcpy across the Java/JSI boundary. */
6
+ final class NativeRgbaBuffer {
7
+ static {
8
+ System.loadLibrary("react-native-skia-video");
9
+ }
10
+ private static final BufferLifetime lifetimes = new BufferLifetime(NativeRgbaBuffer::nativeRelease);
11
+ static {
12
+ Thread reaper = new Thread(lifetimes::run, "SkiaVideoBufferReaper");
13
+ reaper.setDaemon(true);
14
+ reaper.start();
15
+ }
16
+ private NativeRgbaBuffer() {}
17
+
18
+ private static final RgbaBufferAllocator allocator = new RgbaBufferAllocator(lifetimes,
19
+ NativeRgbaBuffer::nativeReserve, NativeRgbaBuffer::nativeRelease,
20
+ // ART-managed, non-moving backing creates real Java GC pressure.
21
+ ByteBuffer::allocateDirect, NativeRgbaBuffer::nativeAlias, size -> {
22
+ Runtime runtime = Runtime.getRuntime();
23
+ RgbaBufferAllocator.checkAllocationCapacity(size, runtime.maxMemory());
24
+ });
25
+
26
+ static ByteBuffer allocate(int size) { return allocator.allocate(size); }
27
+
28
+ static long[] budgetStats() { return nativeBudgetStats(); }
29
+ static void configureBudget(long bytes) { nativeConfigureBudget(bytes); }
30
+ private static native long nativeReserve(int size);
31
+ private static native ByteBuffer nativeAlias(ByteBuffer backing);
32
+ private static native void nativeRelease(long token);
33
+ private static native long[] nativeBudgetStats();
34
+ private static native void nativeConfigureBudget(long bytes);
35
+ }
@@ -0,0 +1,45 @@
1
+ package com.azzapp.rnskv;
2
+
3
+ import android.util.Log;
4
+
5
+ import androidx.annotation.NonNull;
6
+
7
+ import com.facebook.react.bridge.JavaScriptContextHolder;
8
+ import com.facebook.react.bridge.Promise;
9
+ import com.facebook.react.bridge.ReactApplicationContext;
10
+ import com.facebook.react.bridge.ReactMethod;
11
+
12
+ public class ReactNativeSkiaVideoModule extends ReactNativeSkiaVideoSpec {
13
+ public static final String NAME = "ReactNativeSkiaVideo";
14
+
15
+ private static ReactApplicationContext reactApplicationContext;
16
+
17
+ public static ReactApplicationContext currentReactApplicationContext() {
18
+ return reactApplicationContext;
19
+ }
20
+
21
+ ReactNativeSkiaVideoModule(ReactApplicationContext context) {
22
+ super(context);
23
+ }
24
+
25
+ @Override
26
+ @NonNull
27
+ public String getName() {
28
+ return NAME;
29
+ }
30
+
31
+ @ReactMethod(isBlockingSynchronousMethod = true)
32
+ public boolean install() {
33
+ reactApplicationContext = getReactApplicationContext();
34
+ System.loadLibrary("react-native-skia-video");
35
+ JavaScriptContextHolder jsContext = getReactApplicationContext().getJavaScriptContextHolder();
36
+ if (jsContext == null || jsContext.get() == 0) {
37
+ Log.e(NAME, "Failed to install react-native-skia-video JSI Bindings: no JavaScript runtime available!");
38
+ return false;
39
+ }
40
+ nativeInstall(jsContext.get());
41
+ return true;
42
+ }
43
+
44
+ private native void nativeInstall(long jsiPtr);
45
+ }