react-native-skia-webgpu-video 0.1.0-beta.3 → 0.1.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -32
- package/android/BUILD_VALIDATION.md +1 -1
- package/android/CMakeLists.txt +3 -2
- package/android/cpp/ExportHintSession.cpp +114 -0
- package/android/cpp/ExportHintSession.h +35 -0
- package/android/cpp/VideoEncoderHostObject.cpp +76 -101
- package/android/cpp/VideoEncoderHostObject.h +14 -9
- package/android/cpp/VideoFrame.cpp +3 -5
- package/android/cpp/cpp-adapter.cpp +7 -4
- package/android/src/main/java/com/azzapp/rnskv/GLFrameExtractor.java +3 -9
- package/android/src/main/java/com/azzapp/rnskv/VideoCompositionDecoder.java +8 -11
- package/android/src/main/java/com/azzapp/rnskv/VideoEncoder.java +68 -28
- package/docs/FRAME_PIXEL_FORMAT.md +1 -1
- package/docs/MIGRATION.md +1 -1
- package/docs/UPSTREAM_NOTES.md +15 -2
- package/lib/module/canvas.js +3 -5
- package/lib/module/canvas.js.map +1 -1
- package/lib/module/exportFrames.js +6 -8
- package/lib/module/exportFrames.js.map +1 -1
- package/lib/module/exportTarget.js +183 -40
- package/lib/module/exportTarget.js.map +1 -1
- package/lib/module/exportVideoComposition.js +53 -12
- package/lib/module/exportVideoComposition.js.map +1 -1
- package/lib/module/frameInterop.js +9 -12
- package/lib/module/frameInterop.js.map +1 -1
- package/lib/module/gpu.js +0 -2
- package/lib/module/gpu.js.map +1 -1
- package/lib/module/gpuDevice.js +12 -0
- package/lib/module/gpuDevice.js.map +1 -1
- package/lib/module/previewLifecycle.js +0 -1
- package/lib/module/previewLifecycle.js.map +1 -1
- package/lib/module/yuvBlit.js +28 -21
- package/lib/module/yuvBlit.js.map +1 -1
- package/lib/typescript/src/canvas.d.ts.map +1 -1
- package/lib/typescript/src/exportFrames.d.ts +3 -2
- package/lib/typescript/src/exportFrames.d.ts.map +1 -1
- package/lib/typescript/src/exportTarget.d.ts +53 -14
- package/lib/typescript/src/exportTarget.d.ts.map +1 -1
- package/lib/typescript/src/exportVideoComposition.d.ts +2 -1
- package/lib/typescript/src/exportVideoComposition.d.ts.map +1 -1
- package/lib/typescript/src/frameInterop.d.ts.map +1 -1
- package/lib/typescript/src/gpu.d.ts.map +1 -1
- package/lib/typescript/src/gpuDevice.d.ts +5 -1
- package/lib/typescript/src/gpuDevice.d.ts.map +1 -1
- package/lib/typescript/src/previewLifecycle.d.ts.map +1 -1
- package/lib/typescript/src/types.d.ts +10 -5
- package/lib/typescript/src/types.d.ts.map +1 -1
- package/lib/typescript/src/yuvBlit.d.ts +16 -14
- package/lib/typescript/src/yuvBlit.d.ts.map +1 -1
- package/package.json +6 -6
- package/patches/react-native-skia+3.7.4.patch +146 -0
- package/scripts/skia-canvas-patch.json +22 -258
- package/src/canvas.ts +3 -7
- package/src/exportFrames.ts +8 -9
- package/src/exportTarget.ts +217 -56
- package/src/exportVideoComposition.ts +87 -26
- package/src/frameInterop.ts +9 -11
- package/src/gpu.ts +0 -2
- package/src/gpuDevice.ts +26 -1
- package/src/previewLifecycle.ts +0 -1
- package/src/types.ts +10 -5
- package/src/yuvBlit.ts +43 -28
- package/android/cpp/EncoderTargets.cpp +0 -226
- package/android/cpp/EncoderTargets.h +0 -70
- package/patches/react-native-skia+3.7.1.patch +0 -502
package/src/exportFrames.ts
CHANGED
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
blitNv12Frame,
|
|
23
23
|
blitYuvFrame,
|
|
24
24
|
releaseYuvImports,
|
|
25
|
-
|
|
25
|
+
yuvFrameRegion,
|
|
26
26
|
} from "./yuvBlit";
|
|
27
27
|
import type { YuvImport } from "./yuvBlit";
|
|
28
28
|
|
|
@@ -174,9 +174,9 @@ const importYuvExportFrame = (
|
|
|
174
174
|
entry.nativeFrame.height !== frame.height)
|
|
175
175
|
)
|
|
176
176
|
throw new Error("Native video buffer dimensions mismatch");
|
|
177
|
-
const
|
|
178
|
-
?
|
|
179
|
-
:
|
|
177
|
+
const region = nv12
|
|
178
|
+
? null
|
|
179
|
+
: yuvFrameRegion(source.region!, rotation, entry.nativeFrame);
|
|
180
180
|
const device = getVideoGpuDevice();
|
|
181
181
|
target = imports.textures.get(producer);
|
|
182
182
|
if (
|
|
@@ -220,9 +220,7 @@ const importYuvExportFrame = (
|
|
|
220
220
|
producer,
|
|
221
221
|
entry.nativeFrame,
|
|
222
222
|
rotation,
|
|
223
|
-
|
|
224
|
-
frame.width,
|
|
225
|
-
frame.height,
|
|
223
|
+
region!,
|
|
226
224
|
target.texture,
|
|
227
225
|
);
|
|
228
226
|
entry.image = Skia.Image.MakeImageFromGPUTexture(target.texture);
|
|
@@ -329,8 +327,9 @@ export const releaseExportFrames = (imports: ExportFrameImports): void => {
|
|
|
329
327
|
|
|
330
328
|
/** The export counterpart of ownVideoFrames on iOS: decoded frames are drawn
|
|
331
329
|
* in place, and a producer's previous frame is released once its next one is
|
|
332
|
-
* imported, since
|
|
333
|
-
* frames, or every frame after a first failed
|
|
330
|
+
* imported, since the export waits for the GPU to complete a frame before it
|
|
331
|
+
* imports the next one. Other frames, or every frame after a first failed
|
|
332
|
+
* import, use `fallback`. */
|
|
334
333
|
export const ownExportFrames = (
|
|
335
334
|
imports: ExportFrameImports,
|
|
336
335
|
decoded: Record<string, VideoFrame>,
|
package/src/exportTarget.ts
CHANGED
|
@@ -1,21 +1,33 @@
|
|
|
1
1
|
/// <reference types="@webgpu/types" />
|
|
2
|
-
import { Platform } from "react-native";
|
|
3
2
|
import { BlendMode, Skia } from "react-native-skia";
|
|
4
3
|
import type { SkCanvas, SkSurface } from "react-native-skia";
|
|
5
4
|
import type {
|
|
6
5
|
GPUSharedTextureMemory,
|
|
7
6
|
GPUSharedTextureMemoryDescriptor,
|
|
8
7
|
NativeVideoFrame,
|
|
8
|
+
RNCanvasContext,
|
|
9
9
|
} from "react-native-webgpu";
|
|
10
10
|
import { getVideoCanvas } from "./canvas";
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
createNativeVideoGpuFrame,
|
|
13
|
+
createNativeWindowGpuContext,
|
|
14
|
+
encoderWindowTextureUsage,
|
|
15
|
+
getVideoGpuDevice,
|
|
16
|
+
nativeVideoTextureUsage,
|
|
17
|
+
} from "./gpuDevice";
|
|
18
|
+
import {
|
|
19
|
+
frameByteSize,
|
|
20
|
+
releaseVideoMemory,
|
|
21
|
+
reserveVideoMemory,
|
|
22
|
+
} from "./memory";
|
|
12
23
|
import type { VideoEncoder } from "./types";
|
|
13
24
|
|
|
14
|
-
/**
|
|
25
|
+
/** A texture Skia draws an exported frame into through WebGPU: a lent
|
|
26
|
+
* encoder pool buffer on iOS, one of the encoder window's own textures on
|
|
27
|
+
* Android. */
|
|
15
28
|
export type EncoderTarget = {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
fences: bigint[];
|
|
29
|
+
/** iOS: the lent CVPixelBuffer. */
|
|
30
|
+
pointer: bigint | null;
|
|
19
31
|
frame: NativeVideoFrame | null;
|
|
20
32
|
memory: GPUSharedTextureMemory | null;
|
|
21
33
|
texture: GPUTexture | null;
|
|
@@ -24,9 +36,25 @@ export type EncoderTarget = {
|
|
|
24
36
|
canvas: SkCanvas | null;
|
|
25
37
|
};
|
|
26
38
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
39
|
+
/** Android: a canvas context presenting to the codec's input Surface. Skia
|
|
40
|
+
* draws each frame into one of two textures of our own, which a GPU copy then
|
|
41
|
+
* puts into the window's texture. The window's textures are the codec's
|
|
42
|
+
* input buffers: the copy waits on the GPU for the codec to hand one back,
|
|
43
|
+
* while the export goes on with the next frame, rather than the frame's
|
|
44
|
+
* checked wait. */
|
|
45
|
+
export type EncoderWindow = {
|
|
46
|
+
context: RNCanvasContext;
|
|
47
|
+
device: GPUDevice;
|
|
48
|
+
configured: boolean;
|
|
49
|
+
/** Drawn into in turn, each copied into the window after its frame. */
|
|
50
|
+
slots: EncoderTarget[];
|
|
51
|
+
next: number;
|
|
52
|
+
reservations: number[];
|
|
53
|
+
/** The window texture the probe acquired, the first frame's copy target. */
|
|
54
|
+
pending: GPUTexture | null;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const windowSlots = 2;
|
|
30
58
|
|
|
31
59
|
type SharedMemoryDevice = GPUDevice & {
|
|
32
60
|
importSharedTextureMemory?: (
|
|
@@ -43,38 +71,19 @@ export const supportsEncoderTargets = (encoder: VideoEncoder): boolean => {
|
|
|
43
71
|
};
|
|
44
72
|
|
|
45
73
|
// Declared before its callers: worklets capture their closure on creation.
|
|
46
|
-
/**
|
|
47
|
-
* release. Call only after a checked flush(true) of its surface.
|
|
48
|
-
|
|
49
|
-
* encoder, which owns and closes them: ending an AHB access submits a
|
|
50
|
-
* release that flush(true) has not waited for. */
|
|
51
|
-
export const closeEncoderTarget = (
|
|
52
|
-
target: EncoderTarget,
|
|
53
|
-
encoding = false,
|
|
54
|
-
): void => {
|
|
74
|
+
/** iOS: ends GPU access; the lent buffer stays with the caller, to encode or
|
|
75
|
+
* release. Call only after a checked flush(true) of its surface. */
|
|
76
|
+
export const closeEncoderTarget = (target: EncoderTarget): void => {
|
|
55
77
|
"worklet";
|
|
56
|
-
|
|
78
|
+
// Disposing the surface detaches its canvas.
|
|
57
79
|
target.canvas = null;
|
|
58
80
|
target.surface?.dispose();
|
|
59
81
|
target.surface = null;
|
|
60
82
|
if (target.accessing && target.memory && target.texture) {
|
|
61
83
|
const ended = target.memory.endAccess(target.texture);
|
|
62
84
|
target.accessing = false;
|
|
63
|
-
for
|
|
64
|
-
|
|
65
|
-
if (encoding && !lendsPixelBuffers) {
|
|
66
|
-
const exported = state.fence.export();
|
|
67
|
-
// Unreachable when negotiated: AHB interop requires the device's
|
|
68
|
-
// only shared-fence feature to be sync-fd (see gpuDevice.ts).
|
|
69
|
-
if (exported.type !== "sync-fd")
|
|
70
|
-
throw new Error("GPU export needs sync-fd fences");
|
|
71
|
-
target.fences.push(exported.handle);
|
|
72
|
-
}
|
|
73
|
-
} finally {
|
|
74
|
-
// iOS: already waited for. The wrapper is released either way.
|
|
75
|
-
state.fence.destroy();
|
|
76
|
-
}
|
|
77
|
-
}
|
|
85
|
+
// Already waited for. The wrappers are released either way.
|
|
86
|
+
for (const state of ended?.fences ?? []) state.fence.destroy();
|
|
78
87
|
}
|
|
79
88
|
target.accessing = false;
|
|
80
89
|
target.texture?.destroy();
|
|
@@ -88,7 +97,7 @@ export const closeEncoderTarget = (
|
|
|
88
97
|
target.frame = null;
|
|
89
98
|
};
|
|
90
99
|
|
|
91
|
-
/**
|
|
100
|
+
/** iOS: lends the next pool buffer and wraps it as a Skia surface. */
|
|
92
101
|
export const acquireEncoderTarget = (
|
|
93
102
|
encoder: VideoEncoder,
|
|
94
103
|
width: number,
|
|
@@ -97,7 +106,6 @@ export const acquireEncoderTarget = (
|
|
|
97
106
|
"worklet";
|
|
98
107
|
const target: EncoderTarget = {
|
|
99
108
|
pointer: encoder.acquireFrameBuffer!(),
|
|
100
|
-
fences: [],
|
|
101
109
|
frame: null,
|
|
102
110
|
memory: null,
|
|
103
111
|
texture: null,
|
|
@@ -106,33 +114,23 @@ export const acquireEncoderTarget = (
|
|
|
106
114
|
canvas: null,
|
|
107
115
|
};
|
|
108
116
|
try {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
)
|
|
116
|
-
throw new Error("Encoder buffer dimensions or BGRA layout mismatch");
|
|
117
|
-
}
|
|
117
|
+
target.frame = createNativeVideoGpuFrame(target.pointer!);
|
|
118
|
+
if (
|
|
119
|
+
target.frame.width !== width ||
|
|
120
|
+
target.frame.height !== height ||
|
|
121
|
+
target.frame.pixelFormat !== "bgra8"
|
|
122
|
+
) {
|
|
123
|
+
throw new Error("Encoder buffer dimensions or BGRA layout mismatch");
|
|
118
124
|
}
|
|
119
125
|
const device = getVideoGpuDevice() as SharedMemoryDevice;
|
|
120
126
|
if (typeof device.importSharedTextureMemory !== "function") {
|
|
121
127
|
throw new Error("WebGPU shared texture memory is unavailable");
|
|
122
128
|
}
|
|
123
129
|
target.memory = device.importSharedTextureMemory({
|
|
124
|
-
handle: target.frame
|
|
130
|
+
handle: target.frame.handle,
|
|
125
131
|
label: "video export frame",
|
|
126
132
|
});
|
|
127
133
|
target.texture = target.memory.createTexture();
|
|
128
|
-
if (
|
|
129
|
-
!lendsPixelBuffers &&
|
|
130
|
-
(target.texture.width !== width ||
|
|
131
|
-
target.texture.height !== height ||
|
|
132
|
-
target.texture.format !== "rgba8unorm")
|
|
133
|
-
) {
|
|
134
|
-
throw new Error("Encoder buffer dimensions or RGBA layout mismatch");
|
|
135
|
-
}
|
|
136
134
|
// Uninitialized: every frame starts with a full clear.
|
|
137
135
|
target.memory.beginAccess(target.texture, false);
|
|
138
136
|
target.accessing = true;
|
|
@@ -151,8 +149,9 @@ export const acquireEncoderTarget = (
|
|
|
151
149
|
}
|
|
152
150
|
};
|
|
153
151
|
|
|
154
|
-
/**
|
|
155
|
-
* failure, or null when this device can export through encoder
|
|
152
|
+
/** iOS: clears one lent buffer on the GPU, then returns it unencoded.
|
|
153
|
+
* Returns the failure, or null when this device can export through encoder
|
|
154
|
+
* targets. */
|
|
156
155
|
export const probeEncoderTarget = (
|
|
157
156
|
encoder: VideoEncoder,
|
|
158
157
|
width: number,
|
|
@@ -177,3 +176,165 @@ export const probeEncoderTarget = (
|
|
|
177
176
|
encoder.releaseFrameBuffer!();
|
|
178
177
|
return failure;
|
|
179
178
|
};
|
|
179
|
+
|
|
180
|
+
/** Android: the window's next texture, checked against the export size. */
|
|
181
|
+
const windowTexture = (
|
|
182
|
+
window: EncoderWindow,
|
|
183
|
+
width: number,
|
|
184
|
+
height: number,
|
|
185
|
+
): GPUTexture => {
|
|
186
|
+
"worklet";
|
|
187
|
+
const texture = window.pending ?? window.context.getCurrentTexture();
|
|
188
|
+
window.pending = null;
|
|
189
|
+
if (
|
|
190
|
+
texture.width !== width ||
|
|
191
|
+
texture.height !== height ||
|
|
192
|
+
texture.format !== "rgba8unorm"
|
|
193
|
+
) {
|
|
194
|
+
throw new Error("Encoder window dimensions or RGBA layout mismatch");
|
|
195
|
+
}
|
|
196
|
+
return texture;
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
/** Android: the slot the next frame draws into. Its previous copy completed:
|
|
200
|
+
* the checked wait the export makes before each frame's import covers the
|
|
201
|
+
* queue before it. */
|
|
202
|
+
export const acquireWindowTarget = (window: EncoderWindow): EncoderTarget => {
|
|
203
|
+
"worklet";
|
|
204
|
+
return window.slots[window.next]!;
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
/** Android: copies the slot Skia drew into to the window's next texture and
|
|
208
|
+
* hands it to the encoder, with its time. Call once the slot's surface was
|
|
209
|
+
* flushed: the copy follows the drawing on the same queue. */
|
|
210
|
+
export const presentWindowTarget = (
|
|
211
|
+
encoder: VideoEncoder,
|
|
212
|
+
window: EncoderWindow,
|
|
213
|
+
target: EncoderTarget,
|
|
214
|
+
time: number,
|
|
215
|
+
): void => {
|
|
216
|
+
"worklet";
|
|
217
|
+
const source = target.texture!;
|
|
218
|
+
const destination = windowTexture(window, source.width, source.height);
|
|
219
|
+
const commands = window.device.createCommandEncoder({
|
|
220
|
+
label: "video export window copy",
|
|
221
|
+
});
|
|
222
|
+
commands.copyTextureToTexture(
|
|
223
|
+
{ texture: source },
|
|
224
|
+
{ texture: destination },
|
|
225
|
+
{ width: source.width, height: source.height },
|
|
226
|
+
);
|
|
227
|
+
window.device.queue.submit([commands.finish()]);
|
|
228
|
+
window.next = (window.next + 1) % window.slots.length;
|
|
229
|
+
// The encoder sets the frame's time on the window, then presents.
|
|
230
|
+
encoder.encodeFrame(
|
|
231
|
+
{ kind: "window", present: () => window.context.present() },
|
|
232
|
+
time,
|
|
233
|
+
);
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
/** Android: the swapchain gives the codec its buffers back at once, rather
|
|
237
|
+
* than once the context is collected. Call before disposing the encoder. */
|
|
238
|
+
export const closeEncoderWindow = (window: EncoderWindow): void => {
|
|
239
|
+
"worklet";
|
|
240
|
+
window.pending = null;
|
|
241
|
+
if (!window.configured) return;
|
|
242
|
+
window.context.unconfigure();
|
|
243
|
+
window.configured = false;
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
/** Android: frees the window's own textures. Call once their last recording
|
|
247
|
+
* was submitted: a copy still in flight keeps its source alive. */
|
|
248
|
+
export const releaseEncoderWindow = (window: EncoderWindow): void => {
|
|
249
|
+
"worklet";
|
|
250
|
+
while (window.slots.length) {
|
|
251
|
+
const slot = window.slots[window.slots.length - 1]!;
|
|
252
|
+
// Disposing the surface detaches its canvas.
|
|
253
|
+
slot.canvas = null;
|
|
254
|
+
slot.surface?.dispose();
|
|
255
|
+
slot.surface = null;
|
|
256
|
+
slot.texture?.destroy();
|
|
257
|
+
slot.texture = null;
|
|
258
|
+
window.slots.pop();
|
|
259
|
+
}
|
|
260
|
+
while (window.reservations.length) {
|
|
261
|
+
releaseVideoMemory(window.reservations[window.reservations.length - 1]!);
|
|
262
|
+
window.reservations.pop();
|
|
263
|
+
}
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
/** Android: opens a canvas context on the encoder's input window with its
|
|
267
|
+
* slots, acquires the window's first texture and draws a clear into the
|
|
268
|
+
* first slot: nothing reaches the encoder. Returns the window, or the
|
|
269
|
+
* failure, after which the window is closed and the export can fall back to
|
|
270
|
+
* readback. */
|
|
271
|
+
export const probeEncoderWindow = (
|
|
272
|
+
encoder: VideoEncoder,
|
|
273
|
+
width: number,
|
|
274
|
+
height: number,
|
|
275
|
+
): { window: EncoderWindow | null; failure: unknown } => {
|
|
276
|
+
"worklet";
|
|
277
|
+
let window: EncoderWindow | null = null;
|
|
278
|
+
let failure: unknown = null;
|
|
279
|
+
try {
|
|
280
|
+
if (typeof encoder.acquireInputWindow !== "function")
|
|
281
|
+
throw new Error("Encoder cannot present through its input window");
|
|
282
|
+
const device = getVideoGpuDevice();
|
|
283
|
+
window = {
|
|
284
|
+
context: createNativeWindowGpuContext(
|
|
285
|
+
encoder.acquireInputWindow(),
|
|
286
|
+
width,
|
|
287
|
+
height,
|
|
288
|
+
),
|
|
289
|
+
device,
|
|
290
|
+
configured: false,
|
|
291
|
+
slots: [],
|
|
292
|
+
next: 0,
|
|
293
|
+
reservations: [],
|
|
294
|
+
pending: null,
|
|
295
|
+
};
|
|
296
|
+
// Presented in FIFO order, opaque: every frame reaches the codec.
|
|
297
|
+
window.context.configure({
|
|
298
|
+
device,
|
|
299
|
+
format: "rgba8unorm",
|
|
300
|
+
usage: encoderWindowTextureUsage,
|
|
301
|
+
});
|
|
302
|
+
window.configured = true;
|
|
303
|
+
for (let i = 0; i < windowSlots; i++) {
|
|
304
|
+
window.reservations.push(
|
|
305
|
+
reserveVideoMemory(frameByteSize(width, height), "export window slot"),
|
|
306
|
+
);
|
|
307
|
+
const slot: EncoderTarget = {
|
|
308
|
+
pointer: null,
|
|
309
|
+
frame: null,
|
|
310
|
+
memory: null,
|
|
311
|
+
texture: device.createTexture({
|
|
312
|
+
label: "video export window slot",
|
|
313
|
+
size: { width, height },
|
|
314
|
+
format: "rgba8unorm",
|
|
315
|
+
usage: nativeVideoTextureUsage,
|
|
316
|
+
}),
|
|
317
|
+
accessing: false,
|
|
318
|
+
surface: null,
|
|
319
|
+
canvas: null,
|
|
320
|
+
};
|
|
321
|
+
window.slots.push(slot);
|
|
322
|
+
slot.surface = Skia.Surface.MakeFromGPUTexture(slot.texture!);
|
|
323
|
+
if (!slot.surface) throw new Error("Cannot wrap the export frame");
|
|
324
|
+
slot.canvas = getVideoCanvas(slot.surface);
|
|
325
|
+
}
|
|
326
|
+
window.pending = windowTexture(window, width, height);
|
|
327
|
+
const target = acquireWindowTarget(window);
|
|
328
|
+
target.canvas!.drawColor(Skia.Color("#00000000"), BlendMode.Clear);
|
|
329
|
+
target.surface!.flush(true);
|
|
330
|
+
} catch (error) {
|
|
331
|
+
failure = error ?? new Error("GPU export probe failed");
|
|
332
|
+
}
|
|
333
|
+
if (failure === null) return { window, failure };
|
|
334
|
+
if (window) {
|
|
335
|
+
// Not caught: a window still drawn by WebGPU cannot fall back to EGL.
|
|
336
|
+
closeEncoderWindow(window);
|
|
337
|
+
releaseEncoderWindow(window);
|
|
338
|
+
}
|
|
339
|
+
return { window: null, failure };
|
|
340
|
+
};
|
|
@@ -35,10 +35,16 @@ import {
|
|
|
35
35
|
import { enqueueExport, exportAbortError } from "./exportQueue";
|
|
36
36
|
import {
|
|
37
37
|
acquireEncoderTarget,
|
|
38
|
+
acquireWindowTarget,
|
|
38
39
|
closeEncoderTarget,
|
|
40
|
+
closeEncoderWindow,
|
|
41
|
+
presentWindowTarget,
|
|
39
42
|
probeEncoderTarget,
|
|
43
|
+
probeEncoderWindow,
|
|
44
|
+
releaseEncoderWindow,
|
|
40
45
|
supportsEncoderTargets,
|
|
41
46
|
type EncoderTarget,
|
|
47
|
+
type EncoderWindow,
|
|
42
48
|
} from "./exportTarget";
|
|
43
49
|
import { yieldToRuntime } from "./runtimeYield";
|
|
44
50
|
import {
|
|
@@ -82,7 +88,8 @@ export type ExportProfile = {
|
|
|
82
88
|
targetMs: number;
|
|
83
89
|
/** drawFrame, before/afterDrawFrame and progress scheduling. */
|
|
84
90
|
drawMs: number;
|
|
85
|
-
/** The checked flush(true): waiting for the GPU to complete the frame
|
|
91
|
+
/** The checked flush(true): waiting for the GPU to complete the frame,
|
|
92
|
+
* which Android's window export does once it decoded the next frame. */
|
|
86
93
|
gpuMs: number;
|
|
87
94
|
/** Readback when used, submission to the encoder including its
|
|
88
95
|
* backpressure, and the final file write. */
|
|
@@ -136,11 +143,16 @@ type ExportState = {
|
|
|
136
143
|
/** Draw into a lent encoder buffer instead of reading pixels back. */
|
|
137
144
|
gpu: boolean;
|
|
138
145
|
target: EncoderTarget | null;
|
|
146
|
+
/** Android GPU export: the encoder's input window, drawn into directly. */
|
|
147
|
+
window: EncoderWindow | null;
|
|
139
148
|
/** iOS GPU export: decoded frames sampled in place, by producer. */
|
|
140
149
|
imports: ExportFrameImports | null;
|
|
141
150
|
frames: Record<string, VideoFrame>;
|
|
142
151
|
processor: VideoFrameProcessor | null;
|
|
143
152
|
pendingAfterDraw: (() => void) | null;
|
|
153
|
+
/** Android window: the last frame was submitted without its checked GPU
|
|
154
|
+
* wait, which comes before anything it sampled is released. */
|
|
155
|
+
unchecked?: boolean;
|
|
144
156
|
reservations: number[];
|
|
145
157
|
index: number;
|
|
146
158
|
setupError?: unknown;
|
|
@@ -290,6 +302,16 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
|
|
|
290
302
|
if (clock.last >= 0) profile[stage] += now - clock.last;
|
|
291
303
|
clock.last = now;
|
|
292
304
|
};
|
|
305
|
+
/** Android window: the checked GPU wait the last frame left for later,
|
|
306
|
+
* so that the GPU converted and drew it while the CPU presented it and
|
|
307
|
+
* decoded the next one. An empty recording: Graphite still waits for
|
|
308
|
+
* every command submitted so far. */
|
|
309
|
+
const checkWindowFrame = () => {
|
|
310
|
+
"worklet";
|
|
311
|
+
if (!state.unchecked) return;
|
|
312
|
+
state.window!.slots[state.window!.next]!.surface!.flush(true);
|
|
313
|
+
state.unchecked = false;
|
|
314
|
+
};
|
|
293
315
|
const finish = async (
|
|
294
316
|
error: unknown,
|
|
295
317
|
aborted = false,
|
|
@@ -313,6 +335,7 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
|
|
|
313
335
|
};
|
|
314
336
|
// Submit any recording left by a drawing exception before an
|
|
315
337
|
// adapter is allowed to destroy textures referenced by that canvas.
|
|
338
|
+
clean(checkWindowFrame);
|
|
316
339
|
clean(() => state.surface?.flush(true));
|
|
317
340
|
clean(() => state.target?.surface?.flush(true));
|
|
318
341
|
if (cleanup.safeToRelease && state.pendingAfterDraw) {
|
|
@@ -329,6 +352,10 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
|
|
|
329
352
|
}
|
|
330
353
|
}
|
|
331
354
|
if (clean(() => state.extractor?.dispose())) state.extractor = null;
|
|
355
|
+
// Android: the window's swapchain lets go of the codec's buffers
|
|
356
|
+
// before the encoder goes. Its own textures are freed further down.
|
|
357
|
+
const window = state.window;
|
|
358
|
+
if (window) clean(() => closeEncoderWindow(window));
|
|
332
359
|
if (clean(() => state.encoder?.dispose())) state.encoder = null;
|
|
333
360
|
if (cleanup.safeToRelease) {
|
|
334
361
|
clean(() => state.surface?.flush(true));
|
|
@@ -341,10 +368,20 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
|
|
|
341
368
|
state.imports = null;
|
|
342
369
|
clean(() => clearVideoFrameImages(true));
|
|
343
370
|
}
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
if (cleanup.safeToRelease && clean(() => state.surface?.dispose()))
|
|
371
|
+
// Disposing the surface detaches its canvas.
|
|
372
|
+
if (cleanup.safeToRelease && clean(() => state.surface?.dispose())) {
|
|
347
373
|
state.surface = null;
|
|
374
|
+
state.canvas = null;
|
|
375
|
+
}
|
|
376
|
+
// The target, if any, is one of the window's slots.
|
|
377
|
+
if (
|
|
378
|
+
cleanup.safeToRelease &&
|
|
379
|
+
window &&
|
|
380
|
+
clean(() => releaseEncoderWindow(window))
|
|
381
|
+
) {
|
|
382
|
+
state.window = null;
|
|
383
|
+
state.target = null;
|
|
384
|
+
}
|
|
348
385
|
const target = state.target;
|
|
349
386
|
if (cleanup.safeToRelease && target) {
|
|
350
387
|
if (clean(() => closeEncoderTarget(target))) state.target = null;
|
|
@@ -394,6 +431,8 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
|
|
|
394
431
|
const owned = runWithVideoAutoreleasePool(() => {
|
|
395
432
|
const decoded = state.extractor!.decodeCompositionFrames(time);
|
|
396
433
|
lap("decodeMs");
|
|
434
|
+
checkWindowFrame();
|
|
435
|
+
lap("gpuMs");
|
|
397
436
|
const imports = state.imports;
|
|
398
437
|
// In place, a producer's frame is released by its next import.
|
|
399
438
|
const imported = imports
|
|
@@ -427,11 +466,13 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
|
|
|
427
466
|
const draw = () => {
|
|
428
467
|
"worklet";
|
|
429
468
|
if (state.gpu)
|
|
430
|
-
state.target =
|
|
431
|
-
state.
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
469
|
+
state.target = state.window
|
|
470
|
+
? acquireWindowTarget(state.window)
|
|
471
|
+
: acquireEncoderTarget(
|
|
472
|
+
state.encoder!,
|
|
473
|
+
options.width,
|
|
474
|
+
options.height,
|
|
475
|
+
);
|
|
435
476
|
lap("targetMs");
|
|
436
477
|
const target = state.target;
|
|
437
478
|
const surface = target ? target.surface! : state.surface!;
|
|
@@ -454,18 +495,25 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
|
|
|
454
495
|
height: options.height,
|
|
455
496
|
});
|
|
456
497
|
lap("drawMs");
|
|
457
|
-
|
|
498
|
+
// Android window, no context to release: the next frame waits
|
|
499
|
+
// for this one (see checkWindowFrame).
|
|
500
|
+
const unchecked =
|
|
501
|
+
!!state.window &&
|
|
502
|
+
!!target &&
|
|
503
|
+
!afterDrawFrame &&
|
|
504
|
+
!state.processor;
|
|
505
|
+
surface.flush(!unchecked);
|
|
506
|
+
state.unchecked = unchecked;
|
|
458
507
|
drawn.drained = true;
|
|
459
508
|
lap("gpuMs");
|
|
460
|
-
if (target) {
|
|
461
|
-
|
|
509
|
+
if (target && state.window) {
|
|
510
|
+
state.target = null;
|
|
511
|
+
presentWindowTarget(state.encoder!, state.window, target, time);
|
|
512
|
+
} else if (target) {
|
|
513
|
+
closeEncoderTarget(target);
|
|
462
514
|
state.target = null;
|
|
463
515
|
state.encoder!.encodeFrame(
|
|
464
|
-
{
|
|
465
|
-
kind: "native-buffer",
|
|
466
|
-
nativeBuffer: target.pointer,
|
|
467
|
-
fences: target.fences,
|
|
468
|
-
},
|
|
516
|
+
{ kind: "native-buffer", nativeBuffer: target.pointer },
|
|
469
517
|
time,
|
|
470
518
|
);
|
|
471
519
|
} else {
|
|
@@ -615,6 +663,7 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
|
|
|
615
663
|
readback: null,
|
|
616
664
|
gpu: false,
|
|
617
665
|
target: null,
|
|
666
|
+
window: null,
|
|
618
667
|
imports: null,
|
|
619
668
|
frames: {},
|
|
620
669
|
processor: null,
|
|
@@ -666,16 +715,28 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
|
|
|
666
715
|
);
|
|
667
716
|
state.encoder.prepare();
|
|
668
717
|
if (gpuExport) {
|
|
669
|
-
// Skia draws straight into the encoder's buffers
|
|
670
|
-
//
|
|
718
|
+
// Skia draws straight into the encoder's buffers: lent pool
|
|
719
|
+
// buffers on iOS, the input window on Android. The probe keeps
|
|
720
|
+
// devices where that fails on the readback path.
|
|
671
721
|
state.hasRecorder = true;
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
722
|
+
let failure: unknown;
|
|
723
|
+
if (bgra) {
|
|
724
|
+
failure = supportsEncoderTargets(state.encoder)
|
|
725
|
+
? probeEncoderTarget(
|
|
726
|
+
state.encoder,
|
|
727
|
+
options.width,
|
|
728
|
+
options.height,
|
|
729
|
+
)
|
|
730
|
+
: new Error("Encoder cannot lend frame buffers");
|
|
731
|
+
} else {
|
|
732
|
+
const probe = probeEncoderWindow(
|
|
733
|
+
state.encoder,
|
|
734
|
+
options.width,
|
|
735
|
+
options.height,
|
|
736
|
+
);
|
|
737
|
+
state.window = probe.window;
|
|
738
|
+
failure = probe.failure;
|
|
739
|
+
}
|
|
679
740
|
state.gpu = failure === null;
|
|
680
741
|
// Decoded frames are sampled in place too: on iOS through the
|
|
681
742
|
// same IOSurface import, on Android once converted from YUV.
|
package/src/frameInterop.ts
CHANGED
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
blitNv12Frame,
|
|
20
20
|
blitYuvFrame,
|
|
21
21
|
releaseYuvImports,
|
|
22
|
-
|
|
22
|
+
yuvFrameRegion,
|
|
23
23
|
} from "./yuvBlit";
|
|
24
24
|
import type { YuvImport } from "./yuvBlit";
|
|
25
25
|
|
|
@@ -116,7 +116,6 @@ const clearEntry = (cache: Cache, producer: number) => {
|
|
|
116
116
|
* The caller has either not submitted a copy or has completed its GPU fence. */
|
|
117
117
|
const clearSurfaceView = (cache: Cache) => {
|
|
118
118
|
"worklet";
|
|
119
|
-
cache.canvas?.dispose();
|
|
120
119
|
cache.canvas = null;
|
|
121
120
|
cache.stage?.dispose();
|
|
122
121
|
cache.stage = null;
|
|
@@ -449,14 +448,15 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
449
448
|
const region = texture.region;
|
|
450
449
|
const rotation = texture.rotation ?? 0;
|
|
451
450
|
const nv12 = !region && texture.format === "nv12";
|
|
452
|
-
let
|
|
451
|
+
let upright: ReturnType<typeof yuvFrameRegion> | null = null;
|
|
453
452
|
if (region) {
|
|
454
453
|
// An Android codec buffer: padded, YUV, in encoded orientation. The
|
|
455
|
-
// blit converts, crops and
|
|
456
|
-
// as the GL path's SurfaceTexture transform did. RN WebGPU
|
|
457
|
-
// driver-defined YUV formats "bgra8", yet Dawn imports them
|
|
458
|
-
// the native side checked the format, so the label is not
|
|
459
|
-
|
|
454
|
+
// blit converts, crops, turns and scales it into the frame-sized
|
|
455
|
+
// scratch, as the GL path's SurfaceTexture transform did. RN WebGPU
|
|
456
|
+
// labels driver-defined YUV formats "bgra8", yet Dawn imports them
|
|
457
|
+
// as YUV: the native side checked the format, so the label is not
|
|
458
|
+
// read.
|
|
459
|
+
upright = yuvFrameRegion(region, rotation, nativeSource);
|
|
460
460
|
} else if (nv12) {
|
|
461
461
|
if (
|
|
462
462
|
nativeSource.width !== frame.width ||
|
|
@@ -487,9 +487,7 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
487
487
|
key,
|
|
488
488
|
nativeSource,
|
|
489
489
|
rotation,
|
|
490
|
-
|
|
491
|
-
frame.width,
|
|
492
|
-
frame.height,
|
|
490
|
+
upright!,
|
|
493
491
|
cache.uploadTexture!,
|
|
494
492
|
);
|
|
495
493
|
} else if (nv12) {
|
package/src/gpu.ts
CHANGED
|
@@ -178,7 +178,6 @@ export const createVideoGpuScope = (options?: {
|
|
|
178
178
|
} catch (error) {
|
|
179
179
|
// No texture has been exposed or submitted to any consumer yet.
|
|
180
180
|
try {
|
|
181
|
-
resource.canvas?.dispose();
|
|
182
181
|
resource.canvas = null;
|
|
183
182
|
resource.surface?.dispose();
|
|
184
183
|
resource.surface = null;
|
|
@@ -265,7 +264,6 @@ export const createVideoGpuScope = (options?: {
|
|
|
265
264
|
try {
|
|
266
265
|
resource.image?.dispose();
|
|
267
266
|
resource.image = null;
|
|
268
|
-
resource.canvas?.dispose();
|
|
269
267
|
resource.canvas = null;
|
|
270
268
|
resource.surface?.dispose();
|
|
271
269
|
resource.surface = null;
|