react-native-skia-webgpu-video 0.0.0-stage → 0.1.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +303 -2
- package/android/BUILD_VALIDATION.md +62 -0
- package/android/CMakeLists.txt +99 -0
- package/android/build-validation.json +262 -0
- package/android/build.gradle +185 -0
- package/android/cpp/EncoderTargets.cpp +226 -0
- package/android/cpp/EncoderTargets.h +70 -0
- package/android/cpp/JNIHelpers.cpp +15 -0
- package/android/cpp/JNIHelpers.h +20 -0
- package/android/cpp/NativeEventDispatcher.cpp +59 -0
- package/android/cpp/NativeEventDispatcher.h +31 -0
- package/android/cpp/NativeHardwareBuffer.cpp +397 -0
- package/android/cpp/NativeHardwareBuffer.h +34 -0
- package/android/cpp/NativeRgbaBuffer.cpp +45 -0
- package/android/cpp/NativeRgbaBuffer.h +9 -0
- package/android/cpp/ReturnedVideoFrames.h +37 -0
- package/android/cpp/RgbaFrameLease.h +53 -0
- package/android/cpp/VideoCapabilities.cpp +87 -0
- package/android/cpp/VideoCapabilities.h +44 -0
- package/android/cpp/VideoComposition.cpp +170 -0
- package/android/cpp/VideoComposition.h +36 -0
- package/android/cpp/VideoCompositionFramesExtractor.cpp +82 -0
- package/android/cpp/VideoCompositionFramesExtractor.h +48 -0
- package/android/cpp/VideoCompositionFramesExtractorHostObject.cpp +230 -0
- package/android/cpp/VideoCompositionFramesExtractorHostObject.h +42 -0
- package/android/cpp/VideoCompositionFramesExtractorSyncHostObject.cpp +133 -0
- package/android/cpp/VideoCompositionFramesExtractorSyncHostObject.h +46 -0
- package/android/cpp/VideoEncoderHostObject.cpp +318 -0
- package/android/cpp/VideoEncoderHostObject.h +76 -0
- package/android/cpp/VideoFrame.cpp +293 -0
- package/android/cpp/VideoFrame.h +27 -0
- package/android/cpp/VideoPlayer.cpp +96 -0
- package/android/cpp/VideoPlayer.h +52 -0
- package/android/cpp/VideoPlayerHostObject.cpp +225 -0
- package/android/cpp/VideoPlayerHostObject.h +43 -0
- package/android/cpp/cpp-adapter.cpp +316 -0
- package/android/gradle.properties +5 -0
- package/android/instrumentation-inventory.json +178 -0
- package/android/proguard-rules.pro +4 -0
- package/android/src/main/AndroidManifest.xml +2 -0
- package/android/src/main/java/com/azzapp/rnskv/AudioCompositionExporter.java +537 -0
- package/android/src/main/java/com/azzapp/rnskv/AudioCompositionPlayer.java +321 -0
- package/android/src/main/java/com/azzapp/rnskv/BufferLifetime.java +55 -0
- package/android/src/main/java/com/azzapp/rnskv/DecodeRequests.java +100 -0
- package/android/src/main/java/com/azzapp/rnskv/DecoderWindow.java +55 -0
- package/android/src/main/java/com/azzapp/rnskv/DisplayVsyncPeriod.java +72 -0
- package/android/src/main/java/com/azzapp/rnskv/EGLResourcesHolder.java +132 -0
- package/android/src/main/java/com/azzapp/rnskv/EGLUtils.java +225 -0
- package/android/src/main/java/com/azzapp/rnskv/FrameSize.java +50 -0
- package/android/src/main/java/com/azzapp/rnskv/GLFrameExtractor.java +510 -0
- package/android/src/main/java/com/azzapp/rnskv/JNIHelpers.java +12 -0
- package/android/src/main/java/com/azzapp/rnskv/NativeEventDispatcher.java +25 -0
- package/android/src/main/java/com/azzapp/rnskv/NativeHardwareBuffer.java +62 -0
- package/android/src/main/java/com/azzapp/rnskv/NativeRgbaBuffer.java +43 -0
- package/android/src/main/java/com/azzapp/rnskv/ReactNativeSkiaVideoModule.java +45 -0
- package/android/src/main/java/com/azzapp/rnskv/ReactNativeSkiaVideoPackage.java +43 -0
- package/android/src/main/java/com/azzapp/rnskv/ReactNativeSkiaVideoSpec.java +9 -0
- package/android/src/main/java/com/azzapp/rnskv/RgbaBufferAllocator.java +114 -0
- package/android/src/main/java/com/azzapp/rnskv/RgbaLayout.java +33 -0
- package/android/src/main/java/com/azzapp/rnskv/SurfaceOutputPolicy.java +13 -0
- package/android/src/main/java/com/azzapp/rnskv/TextureRenderer.java +150 -0
- package/android/src/main/java/com/azzapp/rnskv/TimeHelpers.java +11 -0
- package/android/src/main/java/com/azzapp/rnskv/VideoCapabilities.java +354 -0
- package/android/src/main/java/com/azzapp/rnskv/VideoComposition.java +147 -0
- package/android/src/main/java/com/azzapp/rnskv/VideoCompositionDecoder.java +779 -0
- package/android/src/main/java/com/azzapp/rnskv/VideoCompositionFramesExtractor.java +463 -0
- package/android/src/main/java/com/azzapp/rnskv/VideoCompositionFramesExtractorSync.java +241 -0
- package/android/src/main/java/com/azzapp/rnskv/VideoCompositionItemDecoder.java +492 -0
- package/android/src/main/java/com/azzapp/rnskv/VideoEncoder.java +639 -0
- package/android/src/main/java/com/azzapp/rnskv/VideoFrame.java +74 -0
- package/android/src/main/java/com/azzapp/rnskv/VideoPlayer.java +361 -0
- package/cpp/RNSVCheckedSizes.h +54 -0
- package/cpp/RNSVEventEmitter.cpp +199 -0
- package/cpp/RNSVEventEmitter.h +31 -0
- package/cpp/RNSVHostObject.cpp +79 -0
- package/cpp/RNSVHostObject.h +67 -0
- package/cpp/RNSVMemoryBudget.cpp +68 -0
- package/cpp/RNSVMemoryBudget.h +100 -0
- package/cpp/RNSVRuntimeLifecycleMonitor.cpp +88 -0
- package/cpp/RNSVRuntimeLifecycleMonitor.h +33 -0
- package/docs/ANDROID_DEVICE_306.md +95 -0
- package/docs/ANDROID_EXPORT_CACHE_306.md +48 -0
- package/docs/ANDROID_MEMORY_RETENTION_306.md +67 -0
- package/docs/BENCHMARKS.md +368 -0
- package/docs/IOS_DEVICE_TESTS.md +115 -0
- package/docs/IOS_SIMULATOR_TESTS.md +74 -0
- package/docs/MIGRATION.md +82 -0
- package/docs/NATIVE_MEMORY_INVESTIGATION_306.md +88 -0
- package/docs/PERFORMANCE_AB_IPHONE_303.md +196 -0
- package/docs/PERFORMANCE_AB_SIMULATOR.md +135 -0
- package/docs/PERFORMANCE_AB_SOAK_SIMULATOR.md +84 -0
- package/docs/PERFORMANCE_IPHONE_306_OPTIMIZATION.md +76 -0
- package/docs/PERFORMANCE_SIMULATOR.md +89 -0
- package/docs/PERFORMANCE_SIMULATOR_INITIAL.md +96 -0
- package/docs/PERFORMANCE_SOAK_IPHONE_303.md +41 -0
- package/docs/UPSTREAM_NOTES.md +188 -0
- package/docs/assets/banner.svg +26 -0
- package/examples/webgpu-overlay.ts +147 -0
- package/ios/AVAssetTrackUtils.h +24 -0
- package/ios/AudioCompositionUtils.h +27 -0
- package/ios/AudioCompositionUtils.mm +113 -0
- package/ios/DecoderWindow.h +52 -0
- package/ios/NativeResourceStats.h +32 -0
- package/ios/RNSVColorSpace.h +50 -0
- package/ios/RNSVJSIUtils.h +46 -0
- package/ios/RNSVScopedPixelBuffer.h +17 -0
- package/ios/RNSVVideoPlayer.h +46 -0
- package/ios/RNSVVideoPlayer.mm +410 -0
- package/ios/ReactNativeSkiaVideo.h +8 -0
- package/ios/ReactNativeSkiaVideo.mm +310 -0
- package/ios/SeekPolicy.h +27 -0
- package/ios/VideoComposition.h +148 -0
- package/ios/VideoCompositionFramesExtractorHostObject.h +105 -0
- package/ios/VideoCompositionFramesExtractorHostObject.mm +686 -0
- package/ios/VideoCompositionFramesExtractorSyncHostObject.h +37 -0
- package/ios/VideoCompositionFramesExtractorSyncHostObject.mm +168 -0
- package/ios/VideoCompositionItemDecoder.h +60 -0
- package/ios/VideoCompositionItemDecoder.mm +480 -0
- package/ios/VideoEncoderHostObject.h +83 -0
- package/ios/VideoEncoderHostObject.mm +706 -0
- package/ios/VideoFrame.h +66 -0
- package/ios/VideoFrame.mm +108 -0
- package/ios/VideoPlayerHostObject.h +52 -0
- package/ios/VideoPlayerHostObject.mm +332 -0
- package/lib/module/NativeReactNativeSkiaVideo.js +5 -0
- package/lib/module/NativeReactNativeSkiaVideo.js.map +1 -0
- package/lib/module/RNSkiaVideoModule.js +20 -0
- package/lib/module/RNSkiaVideoModule.js.map +1 -0
- package/lib/module/canvas.js +18 -0
- package/lib/module/canvas.js.map +1 -0
- package/lib/module/exportCache.js +22 -0
- package/lib/module/exportCache.js.map +1 -0
- package/lib/module/exportFrames.js +338 -0
- package/lib/module/exportFrames.js.map +1 -0
- package/lib/module/exportQueue.js +52 -0
- package/lib/module/exportQueue.js.map +1 -0
- package/lib/module/exportTarget.js +137 -0
- package/lib/module/exportTarget.js.map +1 -0
- package/lib/module/exportVideoComposition.js +492 -0
- package/lib/module/exportVideoComposition.js.map +1 -0
- package/lib/module/frameInterop.js +532 -0
- package/lib/module/frameInterop.js.map +1 -0
- package/lib/module/gpu.js +222 -0
- package/lib/module/gpu.js.map +1 -0
- package/lib/module/gpuDevice.js +54 -0
- package/lib/module/gpuDevice.js.map +1 -0
- package/lib/module/index.js +36 -0
- package/lib/module/index.js.map +1 -0
- package/lib/module/memory.js +38 -0
- package/lib/module/memory.js.map +1 -0
- package/lib/module/package.json +1 -0
- package/lib/module/previewLifecycle.js +52 -0
- package/lib/module/previewLifecycle.js.map +1 -0
- package/lib/module/runtimeYield.js +10 -0
- package/lib/module/runtimeYield.js.map +1 -0
- package/lib/module/types.js +4 -0
- package/lib/module/types.js.map +1 -0
- package/lib/module/utils/useEventListener.js +18 -0
- package/lib/module/utils/useEventListener.js.map +1 -0
- package/lib/module/videoCompositionPlayer.js +368 -0
- package/lib/module/videoCompositionPlayer.js.map +1 -0
- package/lib/module/videoFrameImage.js +185 -0
- package/lib/module/videoFrameImage.js.map +1 -0
- package/lib/module/videoPlayer.js +172 -0
- package/lib/module/videoPlayer.js.map +1 -0
- package/lib/module/yuvBlit.js +299 -0
- package/lib/module/yuvBlit.js.map +1 -0
- package/lib/typescript/package.json +1 -0
- package/lib/typescript/src/NativeReactNativeSkiaVideo.d.ts +7 -0
- package/lib/typescript/src/NativeReactNativeSkiaVideo.d.ts.map +1 -0
- package/lib/typescript/src/RNSkiaVideoModule.d.ts +4 -0
- package/lib/typescript/src/RNSkiaVideoModule.d.ts.map +1 -0
- package/lib/typescript/src/canvas.d.ts +4 -0
- package/lib/typescript/src/canvas.d.ts.map +1 -0
- package/lib/typescript/src/exportCache.d.ts +6 -0
- package/lib/typescript/src/exportCache.d.ts.map +1 -0
- package/lib/typescript/src/exportFrames.d.ts +57 -0
- package/lib/typescript/src/exportFrames.d.ts.map +1 -0
- package/lib/typescript/src/exportQueue.d.ts +5 -0
- package/lib/typescript/src/exportQueue.d.ts.map +1 -0
- package/lib/typescript/src/exportTarget.d.ts +28 -0
- package/lib/typescript/src/exportTarget.d.ts.map +1 -0
- package/lib/typescript/src/exportVideoComposition.d.ts +68 -0
- package/lib/typescript/src/exportVideoComposition.d.ts.map +1 -0
- package/lib/typescript/src/frameInterop.d.ts +17 -0
- package/lib/typescript/src/frameInterop.d.ts.map +1 -0
- package/lib/typescript/src/gpu.d.ts +39 -0
- package/lib/typescript/src/gpu.d.ts.map +1 -0
- package/lib/typescript/src/gpuDevice.d.ts +11 -0
- package/lib/typescript/src/gpuDevice.d.ts.map +1 -0
- package/lib/typescript/src/index.d.ts +21 -0
- package/lib/typescript/src/index.d.ts.map +1 -0
- package/lib/typescript/src/memory.d.ts +25 -0
- package/lib/typescript/src/memory.d.ts.map +1 -0
- package/lib/typescript/src/previewLifecycle.d.ts +22 -0
- package/lib/typescript/src/previewLifecycle.d.ts.map +1 -0
- package/lib/typescript/src/runtimeYield.d.ts +4 -0
- package/lib/typescript/src/runtimeYield.d.ts.map +1 -0
- package/lib/typescript/src/types.d.ts +631 -0
- package/lib/typescript/src/types.d.ts.map +1 -0
- package/lib/typescript/src/utils/useEventListener.d.ts +5 -0
- package/lib/typescript/src/utils/useEventListener.d.ts.map +1 -0
- package/lib/typescript/src/videoCompositionPlayer.d.ts +105 -0
- package/lib/typescript/src/videoCompositionPlayer.d.ts.map +1 -0
- package/lib/typescript/src/videoFrameImage.d.ts +60 -0
- package/lib/typescript/src/videoFrameImage.d.ts.map +1 -0
- package/lib/typescript/src/videoPlayer.d.ts +99 -0
- package/lib/typescript/src/videoPlayer.d.ts.map +1 -0
- package/lib/typescript/src/yuvBlit.d.ts +50 -0
- package/lib/typescript/src/yuvBlit.d.ts.map +1 -0
- package/package.json +155 -4
- package/patches/react-native-skia+3.7.0.patch +502 -0
- package/react-native-skia-webgpu-video.podspec +35 -0
- package/scripts/apply-skia-canvas-patch.mjs +48 -0
- package/scripts/skia-canvas-patch.json +291 -0
- package/src/NativeReactNativeSkiaVideo.ts +8 -0
- package/src/RNSkiaVideoModule.ts +27 -0
- package/src/canvas.ts +22 -0
- package/src/exportCache.ts +27 -0
- package/src/exportFrames.ts +421 -0
- package/src/exportQueue.ts +65 -0
- package/src/exportTarget.ts +179 -0
- package/src/exportVideoComposition.ts +718 -0
- package/src/frameInterop.ts +684 -0
- package/src/gpu.ts +290 -0
- package/src/gpuDevice.ts +80 -0
- package/src/index.tsx +45 -0
- package/src/memory.ts +48 -0
- package/src/previewLifecycle.ts +60 -0
- package/src/runtimeYield.ts +6 -0
- package/src/types.ts +661 -0
- package/src/utils/useEventListener.ts +22 -0
- package/src/videoCompositionPlayer.ts +568 -0
- package/src/videoFrameImage.ts +205 -0
- package/src/videoPlayer.ts +318 -0
- package/src/yuvBlit.ts +331 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sebastien Hecart
|
|
4
|
+
Portions Copyright (c) 2024 Azzapp
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,304 @@
|
|
|
1
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="docs/assets/banner.svg" alt="React Native Skia WebGPU Video — Play. Compose. Export." width="100%" />
|
|
3
|
+
</p>
|
|
2
4
|
|
|
3
|
-
|
|
5
|
+
<p align="center">
|
|
6
|
+
<a href="https://github.com/batical/react-native-skia-webgpu-video/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/batical/react-native-skia-webgpu-video/actions/workflows/ci.yml/badge.svg?branch=main" /></a>
|
|
7
|
+
<a href="https://www.npmjs.com/package/react-native-skia-webgpu-video"><img alt="npm" src="https://img.shields.io/npm/v/react-native-skia-webgpu-video/beta" /></a>
|
|
8
|
+
<a href="#status"><img alt="Status: beta" src="https://img.shields.io/badge/status-beta-f59e0b" /></a>
|
|
9
|
+
<img alt="React Native Skia 3.7.0" src="https://img.shields.io/badge/Skia-3.7.0-38bdf8" />
|
|
10
|
+
<img alt="React Native WebGPU 0.17.0" src="https://img.shields.io/badge/WebGPU-0.17.0-a78bfa" />
|
|
11
|
+
<a href="LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-34d399" /></a>
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
# React Native Skia WebGPU Video
|
|
15
|
+
|
|
16
|
+
**Video playback, composition and export for React Native Skia 3 / Graphite.** Draw video frames with Skia, add your own overlays, and reuse your composition when exporting a file.
|
|
17
|
+
|
|
18
|
+
A new Skia 3 / WebGPU frame-interop layer, written from scratch with explicit resource ownership and configurable video memory reservations, while keeping a familiar Skia Video API.
|
|
19
|
+
|
|
20
|
+
**[Get started](#installation) · [Example](#one-composition-preview-and-export) · [Migration](docs/MIGRATION.md) · [Status](#status) · [Credits](#credits)**
|
|
21
|
+
|
|
22
|
+
## What you can build
|
|
23
|
+
|
|
24
|
+
- **Video players and editors** with play, pause, seek, looping, and frame drawing callbacks.
|
|
25
|
+
- **Compositions and overlays** driven by a timeline in seconds, using the same drawing function for preview and export.
|
|
26
|
+
- **H.264 / HEVC exports**, subject to device capabilities, with cancellation and bounded export queues.
|
|
27
|
+
- **Audio compositions** with clip audio, separate tracks, and per-item volume.
|
|
28
|
+
- **Custom GPU processing** through an export frame processor and an optional WebGPU resource scope.
|
|
29
|
+
|
|
30
|
+
> **Beta.** The current Skia 3.7.0 / WebGPU 0.17.0 build has run on an iPhone 15 Pro and a Pixel 8a (export speed, pixel checks). Not yet qualified: memory over long sessions (last checked on 3.0.6), HDR, audible output and A/V synchronization; two 4K cases have known issues. See [status](#status) before adopting this in production.
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
### Requirements
|
|
35
|
+
|
|
36
|
+
| Dependency | Current configuration |
|
|
37
|
+
| --- | --- |
|
|
38
|
+
| React Native | `0.86.2` tested; declared peer range `>=0.83 <0.87` |
|
|
39
|
+
| React | `19.2.3` tested |
|
|
40
|
+
| React Native Skia | **`react-native-skia@3.7.0`** |
|
|
41
|
+
| React Native WebGPU | **`react-native-webgpu@0.17.0`** (both resolve `react-native-webgpu-dawn@157.0.0`) |
|
|
42
|
+
| Reanimated / Worklets | **`4.5.3` / `0.11.3`** |
|
|
43
|
+
| Native runtime | New Architecture and Hermes |
|
|
44
|
+
| Android | API **28+**, Vulkan 1.1 (declared as a required feature by Skia) |
|
|
45
|
+
|
|
46
|
+
A native rebuild is required. Expo Go cannot load this module. Skia and WebGPU now share Dawn through the `react-native-webgpu-dawn` npm package: `node` must be on the `PATH` during `pod install` and Gradle builds, and both packages must resolve the same version of it. For iOS, use the deployment target required by your React Native and Skia versions; the example targets iOS 16.4 or later.
|
|
47
|
+
|
|
48
|
+
### Install
|
|
49
|
+
|
|
50
|
+
The beta is published on npm under the `beta` tag:
|
|
51
|
+
|
|
52
|
+
```sh
|
|
53
|
+
npm install --save-exact react-native-skia@3.7.0 react-native-webgpu@0.17.0 react-native-reanimated@4.5.3 react-native-worklets@0.11.3
|
|
54
|
+
npm install --save-exact react-native-skia-webgpu-video@beta
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Each version requires exact Skia and WebGPU versions (its peer dependencies). The [GitHub releases](https://github.com/batical/react-native-skia-webgpu-video/releases) attach the same package archive, installable by URL. To build it from source instead, run `npm ci`, `npm run build` and `npm pack` in a clone, then `npm install` the resulting `.tgz`.
|
|
58
|
+
|
|
59
|
+
Keep these versions pinned. If migrating an existing app, remove `@shopify/react-native-skia` and the previous Skia Video package before rebuilding. Follow the [migration guide](docs/MIGRATION.md) to avoid loading both generations in one native application.
|
|
60
|
+
|
|
61
|
+
Add `react-native-worklets/plugin` **last** in your Babel plugins:
|
|
62
|
+
|
|
63
|
+
```js
|
|
64
|
+
module.exports = {
|
|
65
|
+
presets: ['module:@react-native/babel-preset'],
|
|
66
|
+
plugins: ['react-native-worklets/plugin'],
|
|
67
|
+
};
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Apply the required Skia patch
|
|
71
|
+
|
|
72
|
+
This package includes a version-checked Skia patch for canvas/image-provider ownership, checked GPU submissions and cleanup of the dedicated export recorder cache. Apply it in the **application**, before compiling native code:
|
|
73
|
+
|
|
74
|
+
```sh
|
|
75
|
+
npx --no-install skia-video-patch-canvas
|
|
76
|
+
cd ios
|
|
77
|
+
pod install
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Add `skia-video-patch-canvas` to your application's existing `postinstall` process so it runs after clean dependency installs. The script checks every target file before changing anything and is safe to run again. An unknown version or conflicting patch stops with an error. Review the [patch and upstream notes](docs/UPSTREAM_NOTES.md) when upgrading Skia, then rebuild the native app.
|
|
81
|
+
|
|
82
|
+
After an export closes its consumers, the library releases unused resources from its private export recorder. It preserves the shared Skia context cache and the preview/UI recorder cache. This reduces retained export resources without adding per-frame purges; it does not impose a limit on total process memory. Upgrading an already-patched application also requires rebuilding its native binary.
|
|
83
|
+
|
|
84
|
+
## One composition, preview and export
|
|
85
|
+
|
|
86
|
+
Use an app-accessible **absolute local file path** for each composition item. This example needs a source at least five seconds long; composition paths are not HTTP or `file://` URLs.
|
|
87
|
+
|
|
88
|
+
```tsx
|
|
89
|
+
import { useMemo } from 'react';
|
|
90
|
+
import { Button, View } from 'react-native';
|
|
91
|
+
import { Canvas, Image } from 'react-native-skia';
|
|
92
|
+
import {
|
|
93
|
+
drawVideoFrame,
|
|
94
|
+
exportVideoComposition,
|
|
95
|
+
useVideoComposition,
|
|
96
|
+
} from 'react-native-skia-webgpu-video';
|
|
97
|
+
import type {
|
|
98
|
+
FrameDrawer,
|
|
99
|
+
VideoComposition,
|
|
100
|
+
} from 'react-native-skia-webgpu-video';
|
|
101
|
+
|
|
102
|
+
const drawFrame: FrameDrawer = ({ canvas, frames, width, height }) => {
|
|
103
|
+
'worklet';
|
|
104
|
+
const frame = frames.clip;
|
|
105
|
+
if (frame) {
|
|
106
|
+
drawVideoFrame(canvas, frame, { x: 0, y: 0, width, height }, {
|
|
107
|
+
fit: 'contain',
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
// Draw your Skia text, shapes, or overlays here.
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
export function VideoEditor({ inputPath, outputPath }: {
|
|
114
|
+
inputPath: string;
|
|
115
|
+
outputPath: string;
|
|
116
|
+
}) {
|
|
117
|
+
const composition = useMemo<VideoComposition>(() => ({
|
|
118
|
+
duration: 5,
|
|
119
|
+
items: [{
|
|
120
|
+
id: 'clip',
|
|
121
|
+
path: inputPath,
|
|
122
|
+
compositionStartTime: 0,
|
|
123
|
+
startTime: 0,
|
|
124
|
+
duration: 5,
|
|
125
|
+
audio: true,
|
|
126
|
+
}],
|
|
127
|
+
}), [inputPath]);
|
|
128
|
+
|
|
129
|
+
const { currentFrame } = useVideoComposition({
|
|
130
|
+
composition,
|
|
131
|
+
drawFrame,
|
|
132
|
+
width: 320,
|
|
133
|
+
height: 180,
|
|
134
|
+
autoPlay: true,
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
const save = async () => {
|
|
138
|
+
await exportVideoComposition({
|
|
139
|
+
videoComposition: composition,
|
|
140
|
+
drawFrame,
|
|
141
|
+
outPath: outputPath,
|
|
142
|
+
width: 1280,
|
|
143
|
+
height: 720,
|
|
144
|
+
frameRate: 30,
|
|
145
|
+
bitRate: 5_000_000,
|
|
146
|
+
});
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
return (
|
|
150
|
+
<View>
|
|
151
|
+
<Canvas style={{ width: 320, height: 180 }}>
|
|
152
|
+
<Image image={currentFrame} x={0} y={0} width={320} height={180} />
|
|
153
|
+
</Canvas>
|
|
154
|
+
<Button title="Export video" onPress={() => {
|
|
155
|
+
void save().catch(console.error);
|
|
156
|
+
}} />
|
|
157
|
+
</View>
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Choose a writable output path. Preview dimensions are layout points; drawing callbacks receive physical pixel dimensions. Export dimensions are pixels. The [`useVideoPlayback`](src/videoPlayer.ts) hook is also available for single-video playback.
|
|
163
|
+
|
|
164
|
+
### Export speed
|
|
165
|
+
|
|
166
|
+
- **Static images drawn on every frame** (logos, stickers, rasterized text): convert them once with `image.makeTextureImage()` (Skia 3.4+). A raster image drawn directly is uploaded once per recorder and kept in a cache of up to 256 images.
|
|
167
|
+
- **Finding the slow stage:** pass `onProfile` to `exportVideoComposition`. It receives cumulative milliseconds per stage (`decodeMs`, `importMs`, `prepareMs`, `targetMs`, `drawMs`, `gpuMs`, `encodeMs`, `yieldMs`), measured on the export runtime, once before the export resolves:
|
|
168
|
+
|
|
169
|
+
```ts
|
|
170
|
+
await exportVideoComposition({
|
|
171
|
+
/* ... */
|
|
172
|
+
onProfile: (profile) => console.log(profile),
|
|
173
|
+
});
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
### Audio
|
|
177
|
+
|
|
178
|
+
Composition clips are **silent by default**. Set `audio: true` to include the source track in playback and export, or `audio: { volume: 0.8 }` to set its volume. Add music or voice-over using a composition item with `kind: 'audio'` and `volume`.
|
|
179
|
+
|
|
180
|
+
Audio implementations are retained on iOS and Android, including AAC export. The latest device performance campaign did **not** validate audible output or audio/video synchronization. See the [typed API](src/types.ts) for the complete options.
|
|
181
|
+
|
|
182
|
+
## Migrating from Skia Video
|
|
183
|
+
|
|
184
|
+
Use `useVideoComposition` for timelines and `useVideoPlayback` for single videos. Rename the previous hook imports and calls; the old names are no longer exported. Composition structure, timeline units, drawing callbacks, and player/export controls are retained. Texture interop changes: replace `MakeImageFromNativeTextureUnstable(frame.texture)` with `drawVideoFrame` or `makeVideoFrameImage`.
|
|
185
|
+
|
|
186
|
+
**[Read the migration guide →](docs/MIGRATION.md)**
|
|
187
|
+
|
|
188
|
+
The `copy` and `direct` options remain available for compatibility and use the same transport: exports draw on the GPU into the encoder's buffers on both platforms, falling back to CPU readback where the interop is unavailable. Inspect `getVideoResourceStats().backend` for the transport actually used.
|
|
189
|
+
|
|
190
|
+
## Memory and GPU extensions
|
|
191
|
+
|
|
192
|
+
Configure the library's tracked reservations before creating players or export sessions:
|
|
193
|
+
|
|
194
|
+
```ts
|
|
195
|
+
import { configureVideoMemory } from 'react-native-skia-webgpu-video';
|
|
196
|
+
|
|
197
|
+
configureVideoMemory({ maxBytes: 256 * 1024 * 1024 });
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
This budget covers **tracked video resources**, not all memory held by codecs, Skia, or GPU drivers. Use lazy decoders for sequential compositions, keep preview resolution appropriate, and measure process memory on your target device. Images returned by the frame helpers belong to the library; do not dispose them independently.
|
|
201
|
+
|
|
202
|
+
### How frames move
|
|
203
|
+
|
|
204
|
+
| Step | iOS | Android |
|
|
205
|
+
| --- | --- | --- |
|
|
206
|
+
| Decode | AVFoundation, BGRA IOSurface-backed buffers, no extra copy (`alwaysCopiesSampleData = NO`) | API 29+, 8-bit SDR H.264/HEVC at the encoded size: `ImageReader` YUV buffers. Otherwise: EGL render into an RGBA AHardwareBuffer (or a CPU scoped-copy fallback) |
|
|
207
|
+
| Into Skia, preview | WebGPU blit into a reusable texture, then an owned Graphite snapshot | WebGPU blit (YUV→RGB, crop and rotation in one pass), then an owned Graphite snapshot |
|
|
208
|
+
| Into Skia, export | Sampled in place through the frame's IOSurface: no copy, no per-clip GPU wait | Same as preview |
|
|
209
|
+
| Export output | Skia draws into the encoder's CVPixelBuffer (IOSurface) | Skia draws into an AHardwareBuffer lent by the encoder; EGL samples it into the codec Surface after its sync-fd |
|
|
210
|
+
| Fallback | CPU readback, with a warning | CPU readback, with a warning |
|
|
211
|
+
|
|
212
|
+
Every path waits for checked GPU completion before releasing source frames or handing a buffer to the encoder. Scaled, HDR and 10-bit Android sources keep the EGL decode path.
|
|
213
|
+
|
|
214
|
+
For advanced exports, `createFrameProcessor` supports asynchronous preparation at each composition timestamp. The optional `react-native-skia-webgpu-video/gpu` entry point provides a resource scope whose asynchronous disposal waits for tracked work. See the [WebGPU cube example](examples/webgpu-overlay.ts).
|
|
215
|
+
|
|
216
|
+
These are integration building blocks. Three.js adapters, Core ML models, and automatic Neural Engine acceleration are not included. Browser video support is not provided by this native library.
|
|
217
|
+
|
|
218
|
+
## Benchmarks
|
|
219
|
+
|
|
220
|
+
Release builds, 3 repetitions per case, deterministic workloads from catalog `2026-10-07.1`, median export wall time. Raw evidence: [iOS device](benchmark/ios-device-370-ab.json), [Android device](benchmark/android-device-370-ab.json); protocol and limits in the [verification report](migration/skia-3.7.0/verification.md). Earlier campaigns: [3.6.1](migration/skia-3.6.1/verification.md), [3.4.4](migration/skia-3.4.4/verification.md).
|
|
221
|
+
|
|
222
|
+
Skia 3.7.0 / WebGPU 0.17.0 move Graphite and Dawn from 154.3.0 to 157.0.0. Export speed is unchanged on both devices; the gains of this release are in on-screen canvases (below).
|
|
223
|
+
|
|
224
|
+
**iPhone 15 Pro (iOS 27.0.1)**, four alternating series off the charger (a first series on a wireless charger ran hot and is not used). Same per-frame profile and same output bytes in both versions:
|
|
225
|
+
|
|
226
|
+
| Export case | 3.6.1 | 3.7.0 |
|
|
227
|
+
| --- | --- | --- |
|
|
228
|
+
| HEVC 1080p, rotated, 180 frames | 734 ms | 731 ms |
|
|
229
|
+
| H.264 640×360, 60 frames | 173–211 ms | 168–171 ms |
|
|
230
|
+
| 8 × 4K clips scaled to 1280, 360 frames | 2.74–2.78 s | 2.73–2.79 s |
|
|
231
|
+
|
|
232
|
+
**Pixel 8a (Android 17)**:
|
|
233
|
+
|
|
234
|
+
| Export case | 3.6.1 | 3.7.0 |
|
|
235
|
+
| --- | --- | --- |
|
|
236
|
+
| HEVC 1080p, rotated, 180 frames | 3.02 s | 2.92 s |
|
|
237
|
+
| H.264 640×360, 60 frames | 1.18 s | 1.23 s |
|
|
238
|
+
| 8 × 4K clips at native size, 360 frames | 10.0 s | 9.8 s |
|
|
239
|
+
| 3 simultaneous 4K clips, 150 frames | 5.8 s | 5.6 s |
|
|
240
|
+
| 8 × 4K clips scaled to 1280 (EGL decode) | 12.2 s | 12.5 s |
|
|
241
|
+
|
|
242
|
+
The native 4K rows compare with the 3.6.1 run of October 16. Files exported from `ImageReader`-decoded clips are up to 5.7% smaller with 3.7.0 while sampled pixels still match CPU decoding within ±1; the cause is not yet confirmed ([verification](migration/skia-3.7.0/verification.md#pixel-8a)).
|
|
243
|
+
|
|
244
|
+
**Smoother canvases on Android 10.** Dawn 157 asks for three swapchain images instead of two on Android. On Android 10, a canvas no longer makes the main thread wait for the previous frame's buffer: upstream measured 407 → 65 ms of `dequeueBuffer` wait per ~1.9 s trace for one opaque canvas on a Mi MIX 3. Skia 3.6.3 also keeps the render pool one frame ahead of the display: on a 120 Hz Pixel 8, an opaque `SurfaceView` canvas went from 66 to 110 presented frames per second. The swapchain change applies to every Skia and WebGPU canvas and the render pool change to Skia canvases, video previews included, with no code change. We did not measure them ourselves (no Android 10 device); see the [audit](migration/skia-3.7.0/upstream-audit.md#canvas-plus-fluide-sur-android-10).
|
|
245
|
+
|
|
246
|
+
Pass `onProfile` to an export to see where the time goes on your own compositions. On the Pixel, a native-size HEVC 1080p frame spends about 2.8 ms decoding, 3.3 ms importing, 1.7 ms waiting for the GPU and 3.8 ms encoding; on the iPhone the whole frame takes under 4 ms.
|
|
247
|
+
|
|
248
|
+
Reproduce on your devices with the example app: `node scripts/test-ios-device-ab-run.mjs` (iPhone) and `node scripts/test-android-device-run.mjs` (Android); see the [benchmark guide](docs/BENCHMARKS.md).
|
|
249
|
+
|
|
250
|
+
## Status
|
|
251
|
+
|
|
252
|
+
| Area | Current evidence |
|
|
253
|
+
| --- | --- |
|
|
254
|
+
| iOS | iPhone 15 Pro: all export cases above passed in every run (72/72), with output sizes identical to 3.6.1 ([verification](migration/skia-3.7.0/verification.md)). Native XCTests were last run on the simulator with 3.6.1 (50/50); the iOS native sources have not changed since. |
|
|
255
|
+
| Android | Pixel 8a: CPU vs GPU decode comparison 10/10 cases (max channel error 1, including rotation, portrait and 4K); export checks of a color pattern and of a decoded rotated clip match CPU readback. With Dawn 157 the errors stay within ±1, as with 3.6.1 ([verification](migration/skia-3.7.0/verification.md#pixels)). Instrumented tests were last run on 3.4.4 (97/102; the 5 failures predate it). |
|
|
256
|
+
| JavaScript / types | 217 tests in 10 suites, typecheck, package tests and 112 benchmark-harness tests pass; CI runs them on every pull request. |
|
|
257
|
+
| Image / audio fidelity | Sampled SDR pixels and an exported color pattern are checked on Android; iOS exported files have the same byte size as with 3.0.6. Whole-image fidelity, HDR, audible output and A/V synchronization remain unqualified. |
|
|
258
|
+
| Known issues | A 4K lazy composition seek times out in 1 of 3 runs on Pixel 8a, and three simultaneous 4K clips stop on the video memory budget during iPhone playback; both happen before and after this release. With `ImageReader`, 1:1 exports are 5–16% larger files for matching sampled pixels; with 3.7.0 they are up to 5.7% smaller than with 3.6.1, cause not confirmed. |
|
|
259
|
+
| Long sessions | Not yet re-run since 3.0.6; the 3.0.6 results below are the latest. |
|
|
260
|
+
|
|
261
|
+
### Earlier results (Skia 3.0.6)
|
|
262
|
+
|
|
263
|
+
These were measured on the previous Skia 3.0.6 / WebGPU 0.12.1 builds and have not been repeated since.
|
|
264
|
+
|
|
265
|
+
- **Lifecycle:** 6/6 consecutive iPhone montages and 4/4 Android montages passed (240 and 160 remounts), with late memory readings flattening or decreasing.
|
|
266
|
+
- **Android AHardwareBuffer playback:** 24/24 focused Release attempts on Pixel 8a passed, including H.264/HEVC 4K and export, with tracked cleanup at zero at a 512 MiB budget ([device diagnostics](docs/ANDROID_DEVICE_306.md)).
|
|
267
|
+
|
|
268
|
+
On one iPhone 15 Pro, an earlier **3.0.6 before/after optimization** comparison observed roughly **11% less peak physical footprint during 4K playback**, with similar RSS. Small exports took **3.5–6% longer**. The phone was warm and used for tethering; these are descriptive results, not a general speed claim. [Protocol, measurements, and limitations](docs/PERFORMANCE_IPHONE_306_OPTIMIZATION.md).
|
|
269
|
+
|
|
270
|
+
The Android report preserves the earlier CPU checkpoints: **0/24 strict passes** in the original campaign, then **8/8 operations completed without OOM but 0/8 strict cleanup passes** with scoped-copy. That latter reference has one attempt per case and does not establish an A/B speed gain. The current AHB results cover the selected scenarios; zero tracked reservations do not prove zero driver memory or absence of leaks in longer use.
|
|
271
|
+
|
|
272
|
+
The eight-clip 4K lifecycle case remained memory intensive: sampled peaks reached **800 MiB RSS / 1,038 MiB PSS**, and RSS remained around **786 MiB after cleanup**, despite zero tracked reservations. RSS and PSS are separate process metrics; their sampled peaks occur independently. The [Android report](docs/ANDROID_DEVICE_306.md) retains these results and the limits of seek/pixel validation.
|
|
273
|
+
|
|
274
|
+
A subsequent export-worklet fix reduces repeated-session allocation overhead. In matched two-montage Release runs, memory after 60 seconds idle fell from **426 to 258 MiB physical footprint on iPhone 15 Pro**, and from **738 to 577 MiB PSS on Pixel 8a**. These are different platform metrics, not a cross-device comparison. Six consecutive iPhone workloads and four Android workloads passed with flatter or decreasing late-run readings; bounded tests do not prove zero leaks. See the [native memory investigation](docs/NATIVE_MEMORY_INVESTIGATION_306.md) for checkpoints and limits.
|
|
275
|
+
|
|
276
|
+
## Development
|
|
277
|
+
|
|
278
|
+
```sh
|
|
279
|
+
npm ci
|
|
280
|
+
npm test
|
|
281
|
+
npm run typecheck
|
|
282
|
+
npm run test:package
|
|
283
|
+
npm run benchmark:test
|
|
284
|
+
npm run build
|
|
285
|
+
npm run check-package
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
Native validation: [iPhone](docs/IOS_DEVICE_TESTS.md), [iOS simulator](docs/IOS_SIMULATOR_TESTS.md), [Android](android/BUILD_VALIDATION.md). The [benchmark guide](docs/BENCHMARKS.md) explains workloads and measured versus unavailable metrics. Upgrading Skia or WebGPU: follow the [upstream notes](docs/UPSTREAM_NOTES.md) and the [3.7.0 migration audit](migration/skia-3.7.0/upstream-audit.md), and re-run the Pixel `interop` pixel check, which caught a color regression in WebGPU 0.15. Before building an Android APK after a dependency bump, delete `example/android/app/build/generated/assets/react`: Gradle does not see root `node_modules` changes and can reuse the previous JS bundle.
|
|
289
|
+
|
|
290
|
+
Contributions are welcome through pull requests. Read the [contribution guide](.github/CONTRIBUTING.md) for the checks and native validation expected for a change.
|
|
291
|
+
|
|
292
|
+
## Credits
|
|
293
|
+
|
|
294
|
+
The **Skia 3 / WebGPU frame-interop layer was written from scratch** for this project. Its API takes inspiration from **[batical/react-native-skia-video](https://github.com/batical/react-native-skia-video)**, Sebastien Hecart's fork extending Skia Video with additional capabilities, including HDR source handling, HEVC export, and the `direct` frame mode.
|
|
295
|
+
|
|
296
|
+
That fork originated from **[AzzappApp/react-native-skia-video](https://github.com/AzzappApp/react-native-skia-video)**.
|
|
297
|
+
|
|
298
|
+
Maintained by **[Sebastien Hecart (@batical)](https://github.com/batical)**, who contributed to the initial development of the original library's iOS implementation.
|
|
299
|
+
|
|
300
|
+
Thanks to the original Skia Video authors and contributors, and to the teams behind [React Native Skia](https://github.com/Shopify/react-native-skia) and [React Native WebGPU](https://github.com/wcandillon/react-native-webgpu).
|
|
301
|
+
|
|
302
|
+
## License
|
|
303
|
+
|
|
304
|
+
[MIT](LICENSE). Copyright (c) 2026 Sebastien Hecart. See the license for notices applicable to portions originating in earlier work.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Android validation and frame ownership
|
|
2
|
+
|
|
3
|
+
The Android example in `../example/android` pins React Native 0.86.2, Skia 3.7.0, WebGPU 0.17.0, the shared `react-native-webgpu-dawn` 157.0.0 (chrome-m157) and Graphite Android 157.0.0; Gradle needs `node` to resolve that Dawn package. Export renders into encoder-lent AHardwareBuffers (`android/cpp/EncoderTargets.cpp`) sampled into the codec Surface by EGL, with CPU readback as the fallback. On API 29+, 8-bit SDR H.264/HEVC clips decoded at their encoded size go through an `ImageReader`, whose YUV buffers WebGPU converts, crops and rotates; since 3.6.1 each buffer is imported once, and exports sample the converted frame in place. The 3.7.0 build was assembled and run on a Pixel 8a on 2026-10-19 ([verification](../migration/skia-3.7.0/verification.md)), after the 3.6.1 build of 2026-10-16 ([verification](../migration/skia-3.6.1/verification.md)) and the 3.4.4 build of 2026-10-14/15 ([verification](../migration/skia-3.4.4/verification.md)); the evidence below is from Skia 3.0.6 / WebGPU 0.12.1 / Graphite 154.1.0. It uses Java 17, Gradle 9.3.1, Android SDK 36 and NDK 27.1.12297006. The library requires API 28; a consuming application with a lower minimum receives an explicit Gradle configuration error. The default example ABI is arm64-v8a; `RNSKV_ANDROID_ABI` selects another ABI for the validation script.
|
|
4
|
+
|
|
5
|
+
Run `android/scripts/validate.sh build` from the repository. It assembles the example Debug and unsigned Release APKs, library AAR, standalone instrumentation APK and JVM tests. Release uses the installed `hermes-compiler` package, and both native and Metro build concurrency are bounded to two workers. The Release APK is unsigned: device installation requires signing a separate local qualification artifact. The project supplies no publishing or release signing configuration.
|
|
6
|
+
|
|
7
|
+
Run `android/scripts/validate.sh instrument DEVICE_SERIAL` to execute the native cases on an explicitly selected connected device. `instrumentation-inventory.json` records the 40 original named cases / 86 parameterized executions and sixteen additional ownership executions, for 52 named cases / 102 planned executions. Unsupported hardware codecs produce JUnit assumption skips, never passes. The library's AndroidTest variant packages its own React JNI dependencies; its AAR keeps the original dependency exclusions.
|
|
8
|
+
|
|
9
|
+
## Verified build evidence
|
|
10
|
+
|
|
11
|
+
`build-validation.json` records hashes of produced artifacts, source fingerprints and the actual JVM results. The initial checkpoint is dated 08:59 UTC and contains compilation and host-test results with no connected device. Later physical-device follow-up is recorded separately in [the device report](../docs/ANDROID_DEVICE_306.md) and [machine-readable evidence](../benchmark/android-device-306-verification.json); it does not retroactively change the initial artifact hashes.
|
|
12
|
+
|
|
13
|
+
The initial successful full Debug build, before the direct-buffer optimization, was preserved at `/private/tmp/rnskwgpu-android-baseline-20261007` with the APK, AAR, test APK, source archive, fingerprints and build logs. This is a candidate implementation checkpoint, not the original Skia 2 reference. The original repository remains the separate benchmark reference.
|
|
14
|
+
|
|
15
|
+
## Historical direct-buffer checkpoint
|
|
16
|
+
|
|
17
|
+
A decoder owns an isolated EGL context and SurfaceTexture. It normalizes its external codec texture into a packed RGBA framebuffer at the requested output dimensions, then reads pixels once into a distinct owned direct buffer. It does not expose an OpenGL texture from another graphics context, share Skia EGL state, or recycle a frame by its age.
|
|
18
|
+
|
|
19
|
+
The preceding staging path retained a reusable readback buffer, copied into a Java byte array and copied again into a C++ vector. The historical direct-buffer checkpoint removed the reusable CPU scratch and both full-frame memcpy operations. Java and JSI viewed the same owned storage. This was a structural reduction in copies, not a measured millisecond or process-memory improvement. GPU-to-CPU readback and GPU upload still occurred; that checkpoint reported `cpu-rgba-readback`, `cpu-rgba-upload` and `zeroCopyDecode: false`. Normal rendering later switched to the scoped-copy fallback and the AHardwareBuffer transport described below.
|
|
20
|
+
|
|
21
|
+
The RGBA backing is allocated through Android's `ByteBuffer.allocateDirect`, so ART sees real allocation pressure. A native memory-budget token is reserved before allocating pixels, including the seven bytes used by Android's alignment. An allocation owner and its phantom cleanup are registered before allocating the backing. A JNI-created buffer points to those pixels; its tracked root holds the owner. Android's JNI buffer implementation preserves that root through slices, duplicates and read-only views. A JSI MutableBuffer holds a global reference to a view, and attaches its finalizer thread before deleting that reference. No second reservation is made for additional Java/JS aliases.
|
|
22
|
+
|
|
23
|
+
When all exposed Java/JS buffers die, the first phantom reference drops the allocation owner. Only after the owner and its backing are collected does its pre-registered phantom return the budget token. The reaper allocates no cleanup trackers. Allocation/alias-registration failures after the backing exists remain conservatively accounted; the token is released immediately only if registration failed before any pixel backing was allocated. `frame.dispose()` drops that frame's JSI ArrayBuffer reference. Existing ArrayBuffer aliases and retained Java frames remain valid and retain the reservation. Cleanup therefore need not immediately return to zero after disposing a reader; a tight budget may reject another frame while GC cleanup remains pending.
|
|
24
|
+
|
|
25
|
+
Each producer writes a buffer once and never mutates it for a later frame. Java consumers receive read-only views. Consumers of the raw JSI ArrayBuffer should treat it as source pixels; ordinary rendering copies them through the public frame helper before releasing its reference.
|
|
26
|
+
|
|
27
|
+
Device tests additionally retain a read-only slice after dropping its VideoFrame, verify unchanged bytes and a retained reservation, and require bounded GC cleanup before accepting return to the baseline. JVM tests cover pre-allocation rejection, failure after backing allocation, idempotent cleanup and both lifetime stages independently of GC timing. They do not replace the device tests.
|
|
28
|
+
|
|
29
|
+
The Surface decoder output callback queues its output index and presentation time directly. Android documents that `MediaCodec.getOutputBuffer()` returns null for an output Surface; querying it had prevented valid frames from being queued. Empty non-EOS Surface outputs are accepted, while empty EOS and codec-configuration buffers are not treated as frames.
|
|
30
|
+
|
|
31
|
+
## Example measurements
|
|
32
|
+
|
|
33
|
+
The Android-only `VideoBenchmarkHarness` streams bundled verified fixtures into the application files directory before timed work, creates scoped export/result files, and supplies Android device metadata. It records actual `/proc/self/status` RSS, `Debug.MemoryInfo` total PSS and native-heap PSS. Complete process GPU allocation is unavailable and remains explicitly unavailable. It reports Android power/thermal state and React foreground lifecycle.
|
|
34
|
+
|
|
35
|
+
Export verification decodes the output with MediaCodec, counts decoded frames and checks presentation timestamps. It releases the codec and extractor on success or failure and bounds stalled verification. JSON results live in the app-scoped `externalFilesDir/benchmark-results` directory, with an internal-files fallback; this is not a public Downloads directory. The shared example screen selects the real platform and Android helper; it no longer assumes the iOS helper or an iPhone identifier.
|
|
36
|
+
|
|
37
|
+
## Primary ownership/API evidence
|
|
38
|
+
|
|
39
|
+
- [AOSP Android 8.1 DirectByteBuffer](https://android.googlesource.com/platform/libcore/+/android-8.1.0_r2/ojluni/src/main/java/java/nio/DirectByteBuffer.java): JNI roots retained through shared MemoryRef; non-moving Java backing and alignment. The behavior predates the library's API 28 minimum.
|
|
40
|
+
- [Android MediaCodec.getOutputBuffer](https://developer.android.com/reference/android/media/MediaCodec#getOutputBuffer(int)): output Surface contract.
|
|
41
|
+
|
|
42
|
+
## Physical-device follow-up and scoped copy
|
|
43
|
+
|
|
44
|
+
A Pixel 8a was subsequently connected. The preserved 24-attempt candidate-only campaign completed six 1080p playback operations and six exports, but all attempts failed strict two-second cleanup and twelve uncapped 4K operations failed controlled ART allocation. A single bounded-allocation-retry follow-up also failed 4K; five JNI ownership/refusal tests and 37 JVM tests passed at that checkpoint. See the linked device report for exact binary identities and limits. No valid baseline A/B improvement is claimed. The current full inventory has 52 named cases / 102 planned executions and has not passed in full.
|
|
45
|
+
|
|
46
|
+
The next scoped-copy path exposes `texture.copyPixelsTo(destination: ArrayBuffer)` and retains the opaque `texture.data` compatibility getter lazily. Normal rendering copies into one budgeted, reusable JS scratch buffer, then performs the existing WebGPU upload/snapshot and checked fence. This deliberately adds one CPU memory copy to remove per-frame Java-backed ArrayBuffer ownership from normal rendering. The library reports the extra copy in backend metadata. Full-resolution decode and the quota remain unchanged.
|
|
47
|
+
|
|
48
|
+
`frame.dispose()` releases its native lease promptly and clears a JavaScript-owned raw-buffer cache. Bound JS getter/copy/dispose functions retain that cache through ordinary GC-traced edges; C++ holds no strong JSI object back to the container. Raw ArrayBuffer aliases obtained explicitly retain their own shared storage after disposal. The cache holder credits source memory to Hermes before materialization, transfers that credit to the raw ArrayBuffer if requested, and removes the holder credit at disposal. Copies validate readable lifetime and destination capacity before writing; `memmove` permits explicit overlapping aliases.
|
|
49
|
+
|
|
50
|
+
`bash android/scripts/test-frame-lease.sh` runs six focused C++ ownership/copy checks with AddressSanitizer and UndefinedBehaviorSanitizer. They cover incomplete destinations without partial writes, retained aliases, prompt owner release, disposed-source rejection and overlap; they do not replace the actual Hermes/ART/device path. The scoped-copy checkpoint completed all eight focused operations without OOM, but all eight failed strict two-second cleanup. It remains a one-attempt diagnostic reference rather than a qualified memory result.
|
|
51
|
+
|
|
52
|
+
## AHardwareBuffer transport and device validation
|
|
53
|
+
|
|
54
|
+
When the shared Skia WebGPU device supports AHardwareBuffer textures and sync-fd fences, Android probes its independent EGL context before enabling the transport for a new decoder. Unsupported hardware is rejected with an explicit capability error before decode and selects the CPU fallback. Quota, allocation and import errors remain visible. Existing decoders capture their chosen transport.
|
|
55
|
+
|
|
56
|
+
The codec SurfaceTexture is drawn into a new immutable RGBA8 AHardwareBuffer through an EGLImage texture. The producer detaches its FBO attachment and waits for checked GL completion before publishing a native-buffer lease. Normal rendering performs no RGBA CPU readback or full-frame CPU copy. Skia imports the allocation, blits it and creates an owned GPU snapshot; its checked completion precedes source-lease disposal. Export still uses the CPU RGBA readback/upload path. This is not a claim of zero-copy rendering or complete GPU memory accounting.
|
|
57
|
+
|
|
58
|
+
Producer frames, returned Java snapshots and JSI frames have independent leases of one shared native allocation. Replacement closes only the producer lease; retained Java or JS frames remain valid. JNI closes every returned Java wrapper, including cache hits and conversion errors. Explicit disposal releases the normal path promptly; raw consumers may retain a lease until close or GC. Budget reservation precedes allocation and accounts the described AHB stride, with uncertain GPU cleanup retaining its reservation. AHB uses CPU_READ_RARELY for the Java pixel oracle; possible layout costs have not been isolated.
|
|
59
|
+
|
|
60
|
+
The frozen Release APK passed ten physical CPU/AHB sampled-pixel comparisons on a Pixel 8a, covering padded dimensions, portrait, rotation, H264/HEVC full 4K and both copy/direct modes. All compared channels were identical, including an old frame imported after producer teardown. Six targeted native AHB ownership, fence, abort, quota and read-validation tests passed with zero skips. The same source passed 43 JVM tests. These focused results do not establish full-suite, HDR, whole-image or audio correctness; 24 repeated focused playback/export attempts passed strict cleanup on the same frozen binary; exact physical memory and limits are recorded separately in the device report.
|
|
61
|
+
|
|
62
|
+
Four existing lifecycle cases also passed once on the same AHB Release artifact: pause/resume, frame-map pruning, paused seek/wait plus further seeks, and a lazy/direct eight-4K montage with 25 churn and 40 remount cycles. Five CPU ownership tests passed again on the current instrumentation artifact. The montage reached 800.250 MiB RSS / 1037.908 MiB PSS and retained 785.816 MiB settled RSS despite tracked owned reservations returning to zero; this is not an endurance or memory-plateau qualification. Full scope, raw hashes and unavailable pixel/native-PTS checks are preserved in the device report.
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
cmake_minimum_required(VERSION 3.22.1)
|
|
2
|
+
project(ReactNativeSkiaVideo)
|
|
3
|
+
|
|
4
|
+
set(PACKAGE_NAME "react-native-skia-video")
|
|
5
|
+
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFOLLY_NO_CONFIG=1")
|
|
6
|
+
|
|
7
|
+
set(CMAKE_VERBOSE_MAKEFILE ON)
|
|
8
|
+
set(CMAKE_CXX_STANDARD 20)
|
|
9
|
+
|
|
10
|
+
set(build_DIR ${CMAKE_SOURCE_DIR}/build)
|
|
11
|
+
|
|
12
|
+
file(GLOB libfbjni_include_DIRS "${build_DIR}/fbjni-*-headers.jar/")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Consume shared libraries and headers from prefabs
|
|
16
|
+
find_package(fbjni REQUIRED CONFIG)
|
|
17
|
+
find_package(ReactAndroid REQUIRED CONFIG)
|
|
18
|
+
|
|
19
|
+
add_library(${PACKAGE_NAME}
|
|
20
|
+
SHARED
|
|
21
|
+
../cpp/RNSVMemoryBudget.cpp
|
|
22
|
+
../cpp/RNSVEventEmitter.h
|
|
23
|
+
../cpp/RNSVEventEmitter.cpp
|
|
24
|
+
../cpp/RNSVHostObject.h
|
|
25
|
+
../cpp/RNSVHostObject.cpp
|
|
26
|
+
../cpp/RNSVRuntimeLifecycleMonitor.h
|
|
27
|
+
../cpp/RNSVRuntimeLifecycleMonitor.cpp
|
|
28
|
+
cpp/JNIHelpers.h
|
|
29
|
+
cpp/JNIHelpers.cpp
|
|
30
|
+
cpp/cpp-adapter.cpp
|
|
31
|
+
cpp/NativeEventDispatcher.h
|
|
32
|
+
cpp/NativeEventDispatcher.cpp
|
|
33
|
+
cpp/VideoCapabilities.h
|
|
34
|
+
cpp/VideoCapabilities.cpp
|
|
35
|
+
cpp/VideoComposition.h
|
|
36
|
+
cpp/VideoComposition.cpp
|
|
37
|
+
cpp/VideoCompositionFramesExtractor.h
|
|
38
|
+
cpp/VideoCompositionFramesExtractor.cpp
|
|
39
|
+
cpp/VideoCompositionFramesExtractorHostObject.h
|
|
40
|
+
cpp/VideoCompositionFramesExtractorHostObject.cpp
|
|
41
|
+
cpp/VideoCompositionFramesExtractorSyncHostObject.h
|
|
42
|
+
cpp/VideoCompositionFramesExtractorSyncHostObject.cpp
|
|
43
|
+
cpp/VideoEncoderHostObject.h
|
|
44
|
+
cpp/VideoEncoderHostObject.cpp
|
|
45
|
+
cpp/VideoPlayerHostObject.h
|
|
46
|
+
cpp/VideoPlayerHostObject.cpp
|
|
47
|
+
cpp/VideoPlayer.h
|
|
48
|
+
cpp/VideoPlayer.cpp
|
|
49
|
+
cpp/VideoFrame.h
|
|
50
|
+
cpp/VideoFrame.cpp
|
|
51
|
+
cpp/RgbaFrameLease.h
|
|
52
|
+
cpp/ReturnedVideoFrames.h
|
|
53
|
+
cpp/NativeRgbaBuffer.h
|
|
54
|
+
cpp/NativeRgbaBuffer.cpp
|
|
55
|
+
cpp/NativeHardwareBuffer.h
|
|
56
|
+
cpp/NativeHardwareBuffer.cpp
|
|
57
|
+
cpp/EncoderTargets.h
|
|
58
|
+
cpp/EncoderTargets.cpp
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# Specifies a path to native header files.
|
|
62
|
+
include_directories(
|
|
63
|
+
../cpp/
|
|
64
|
+
cpp/
|
|
65
|
+
|
|
66
|
+
# React native
|
|
67
|
+
"${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker"
|
|
68
|
+
"${NODE_MODULES_DIR}/react-native/ReactCommon/jsi"
|
|
69
|
+
"${NODE_MODULES_DIR}/react-native/ReactCommon"
|
|
70
|
+
"${NODE_MODULES_DIR}/react-native/ReactCommon/react/nativemodule/core"
|
|
71
|
+
"${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/java/com/facebook/react/turbomodule/core/jni"
|
|
72
|
+
|
|
73
|
+
${libfbjni_include_DIRS}
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
find_library(
|
|
78
|
+
LOG_LIB
|
|
79
|
+
log
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
target_link_libraries(
|
|
83
|
+
${PACKAGE_NAME}
|
|
84
|
+
${LOG_LIB}
|
|
85
|
+
android
|
|
86
|
+
fbjni::fbjni
|
|
87
|
+
ReactAndroid::reactnative
|
|
88
|
+
ReactAndroid::jsi
|
|
89
|
+
-ljnigraphics
|
|
90
|
+
-lGLESv2
|
|
91
|
+
-lEGL
|
|
92
|
+
-landroid
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# Enable Android 16kb native library alignment
|
|
97
|
+
if(CMAKE_ANDROID_NDK_VERSION VERSION_LESS "27")
|
|
98
|
+
target_link_options(${PACKAGE_NAME} PRIVATE "-Wl,-z,max-page-size=16384")
|
|
99
|
+
endif()
|