react-native-skia-webgpu-video 0.1.0-beta.3 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -43
- package/android/BUILD_VALIDATION.md +1 -1
- package/android/CMakeLists.txt +3 -2
- package/android/cpp/ExportHintSession.cpp +114 -0
- package/android/cpp/ExportHintSession.h +35 -0
- package/android/cpp/VideoEncoderHostObject.cpp +76 -101
- package/android/cpp/VideoEncoderHostObject.h +14 -9
- package/android/cpp/VideoFrame.cpp +3 -5
- package/android/cpp/cpp-adapter.cpp +7 -4
- package/android/src/main/java/com/azzapp/rnskv/GLFrameExtractor.java +3 -9
- package/android/src/main/java/com/azzapp/rnskv/VideoCompositionDecoder.java +8 -11
- package/android/src/main/java/com/azzapp/rnskv/VideoEncoder.java +68 -28
- package/docs/FRAME_PIXEL_FORMAT.md +17 -2
- package/docs/MIGRATION.md +1 -1
- package/docs/UPSTREAM_NOTES.md +42 -2
- package/ios/RNSVColorSpace.h +8 -3
- package/lib/module/canvas.js +3 -5
- package/lib/module/canvas.js.map +1 -1
- package/lib/module/exportFrames.js +8 -10
- package/lib/module/exportFrames.js.map +1 -1
- package/lib/module/exportTarget.js +183 -40
- package/lib/module/exportTarget.js.map +1 -1
- package/lib/module/exportVideoComposition.js +78 -35
- package/lib/module/exportVideoComposition.js.map +1 -1
- package/lib/module/frameInterop.js +323 -82
- package/lib/module/frameInterop.js.map +1 -1
- package/lib/module/gpu.js +1 -3
- package/lib/module/gpu.js.map +1 -1
- package/lib/module/gpuDevice.js +12 -0
- package/lib/module/gpuDevice.js.map +1 -1
- package/lib/module/memory.js +7 -0
- package/lib/module/memory.js.map +1 -1
- package/lib/module/previewLifecycle.js +2 -1
- package/lib/module/previewLifecycle.js.map +1 -1
- package/lib/module/videoCompositionPlayer.js +65 -14
- package/lib/module/videoCompositionPlayer.js.map +1 -1
- package/lib/module/videoFrameImage.js +1 -1
- package/lib/module/yuvBlit.js +31 -24
- package/lib/module/yuvBlit.js.map +1 -1
- package/lib/typescript/src/canvas.d.ts.map +1 -1
- package/lib/typescript/src/exportFrames.d.ts +3 -2
- package/lib/typescript/src/exportFrames.d.ts.map +1 -1
- package/lib/typescript/src/exportTarget.d.ts +53 -14
- package/lib/typescript/src/exportTarget.d.ts.map +1 -1
- package/lib/typescript/src/exportVideoComposition.d.ts +2 -1
- package/lib/typescript/src/exportVideoComposition.d.ts.map +1 -1
- package/lib/typescript/src/frameInterop.d.ts +16 -5
- package/lib/typescript/src/frameInterop.d.ts.map +1 -1
- package/lib/typescript/src/gpu.d.ts.map +1 -1
- package/lib/typescript/src/gpuDevice.d.ts +5 -1
- package/lib/typescript/src/gpuDevice.d.ts.map +1 -1
- package/lib/typescript/src/memory.d.ts +2 -0
- package/lib/typescript/src/memory.d.ts.map +1 -1
- package/lib/typescript/src/previewLifecycle.d.ts +2 -0
- package/lib/typescript/src/previewLifecycle.d.ts.map +1 -1
- package/lib/typescript/src/types.d.ts +18 -12
- package/lib/typescript/src/types.d.ts.map +1 -1
- package/lib/typescript/src/videoCompositionPlayer.d.ts +4 -5
- package/lib/typescript/src/videoCompositionPlayer.d.ts.map +1 -1
- package/lib/typescript/src/videoFrameImage.d.ts +1 -1
- package/lib/typescript/src/yuvBlit.d.ts +16 -14
- package/lib/typescript/src/yuvBlit.d.ts.map +1 -1
- package/package.json +6 -6
- package/patches/react-native-skia+3.10.0.patch +150 -0
- package/scripts/skia-canvas-patch.json +22 -258
- package/src/canvas.ts +3 -7
- package/src/exportFrames.ts +10 -11
- package/src/exportTarget.ts +217 -56
- package/src/exportVideoComposition.ts +130 -85
- package/src/frameInterop.ts +389 -96
- package/src/gpu.ts +1 -3
- package/src/gpuDevice.ts +26 -1
- package/src/memory.ts +6 -0
- package/src/previewLifecycle.ts +4 -1
- package/src/types.ts +18 -12
- package/src/videoCompositionPlayer.ts +76 -23
- package/src/videoFrameImage.ts +1 -1
- package/src/yuvBlit.ts +46 -31
- package/android/cpp/EncoderTargets.cpp +0 -226
- package/android/cpp/EncoderTargets.h +0 -70
- package/patches/react-native-skia+3.7.1.patch +0 -502
package/src/frameInterop.ts
CHANGED
|
@@ -14,12 +14,13 @@ import {
|
|
|
14
14
|
frameByteSize,
|
|
15
15
|
releaseVideoMemory,
|
|
16
16
|
reserveVideoMemory,
|
|
17
|
+
videoMemoryBudget,
|
|
17
18
|
} from "./memory";
|
|
18
19
|
import {
|
|
19
20
|
blitNv12Frame,
|
|
20
21
|
blitYuvFrame,
|
|
21
22
|
releaseYuvImports,
|
|
22
|
-
|
|
23
|
+
yuvFrameRegion,
|
|
23
24
|
} from "./yuvBlit";
|
|
24
25
|
import type { YuvImport } from "./yuvBlit";
|
|
25
26
|
|
|
@@ -44,9 +45,18 @@ type FrameTexture =
|
|
|
44
45
|
type Entry = {
|
|
45
46
|
id: number;
|
|
46
47
|
image: SkImage | null;
|
|
48
|
+
/** The import texture the image wraps, if any. */
|
|
49
|
+
texture?: GPUTexture | null;
|
|
47
50
|
reservation: number;
|
|
48
51
|
releasedSources: WeakSet<VideoFrame>;
|
|
49
52
|
};
|
|
53
|
+
/** A decoder buffer an import's GPU copy still reads. */
|
|
54
|
+
type InFlightImport = {
|
|
55
|
+
frame: VideoFrame;
|
|
56
|
+
nativeSource: NativeVideoFrame | null;
|
|
57
|
+
/** The decoder buffer's size, still reserved in the memory budget. */
|
|
58
|
+
bytes: number;
|
|
59
|
+
};
|
|
50
60
|
type PendingSource = {
|
|
51
61
|
owned: SkImage | null;
|
|
52
62
|
reservation: number;
|
|
@@ -55,21 +65,58 @@ type PendingSource = {
|
|
|
55
65
|
sourceReleased: boolean;
|
|
56
66
|
nativeSource?: NativeVideoFrame | null;
|
|
57
67
|
};
|
|
68
|
+
/** A texture frames are converted into, and that their images wrap. */
|
|
69
|
+
type ImportTexture = {
|
|
70
|
+
texture: GPUTexture;
|
|
71
|
+
width: number;
|
|
72
|
+
height: number;
|
|
73
|
+
format: GPUTextureFormat;
|
|
74
|
+
reservation: number;
|
|
75
|
+
};
|
|
76
|
+
/** The textures of a producer: its frames go into them in turn, so that the
|
|
77
|
+
* image of the previous frame stays whole while the next is written. One
|
|
78
|
+
* texture only, written in place, when the memory budget is short. */
|
|
79
|
+
type ImportTarget = { textures: ImportTexture[] };
|
|
58
80
|
type Cache = {
|
|
59
81
|
images: Map<number, Entry>;
|
|
60
82
|
stage: SkSurface | null;
|
|
61
83
|
canvas: SkCanvas | null;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
format: GPUTextureFormat | null;
|
|
84
|
+
/** The texture `stage` wraps: the one of the last import, owned by its
|
|
85
|
+
* producer's target or by `freeTextures`. */
|
|
65
86
|
uploadTexture: GPUTexture | null;
|
|
66
|
-
|
|
87
|
+
targets: Map<number, ImportTarget>;
|
|
88
|
+
/** Textures of released producers, for the next producer of that size. */
|
|
89
|
+
freeTextures: ImportTexture[];
|
|
67
90
|
cpuPixels: Uint8Array<ArrayBuffer> | null;
|
|
68
91
|
cpuReservation: number;
|
|
69
92
|
cpuBytesPerRow: number;
|
|
70
93
|
pendingSources: PendingSource[];
|
|
71
94
|
/** Android decoder buffers imported once, per producer. */
|
|
72
95
|
yuvImports: Map<number, YuvImport[]>;
|
|
96
|
+
/** Checked GPU waits so far (see videoFrameImportWaits). */
|
|
97
|
+
checkedWaits: number;
|
|
98
|
+
/** Oldest first; at most maxInFlightImports. */
|
|
99
|
+
inFlight: InFlightImport[];
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
// Imports whose GPU copy may still be running, their decoder buffers out. Kept
|
|
103
|
+
// below the YUV imports cached per producer (yuvBlit.ts), so that the one it
|
|
104
|
+
// evicts, the least recently drawn, is never still being read.
|
|
105
|
+
const maxInFlightImports = 2;
|
|
106
|
+
|
|
107
|
+
/** Whether the memory budget affords keeping a decoder buffer of `bytes` out
|
|
108
|
+
* until the GPU is done with it: a buffer within a sixteenth of the budget
|
|
109
|
+
* (two out stay within an eighth), and room left for the decoder's next two
|
|
110
|
+
* frames, which it reserves ahead. With the default 256 MB, a 4K BGRA frame
|
|
111
|
+
* (33 MB) is waited for; two 4K NV12 ones (12 MB each) are not. */
|
|
112
|
+
const affordsInFlight = (bytes: number): boolean => {
|
|
113
|
+
"worklet";
|
|
114
|
+
const budget = videoMemoryBudget();
|
|
115
|
+
return (
|
|
116
|
+
!!budget &&
|
|
117
|
+
bytes <= budget.maxBytes / 16 &&
|
|
118
|
+
budget.maxBytes - budget.currentBytes >= 2 * bytes
|
|
119
|
+
);
|
|
73
120
|
};
|
|
74
121
|
|
|
75
122
|
// Capture a scalar on the RN runtime instead of importing Platform inside a
|
|
@@ -86,16 +133,16 @@ const getCache = (): Cache => {
|
|
|
86
133
|
images: new Map(),
|
|
87
134
|
stage: null,
|
|
88
135
|
canvas: null,
|
|
89
|
-
width: 0,
|
|
90
|
-
height: 0,
|
|
91
|
-
format: null,
|
|
92
136
|
uploadTexture: null,
|
|
93
|
-
|
|
137
|
+
targets: new Map(),
|
|
138
|
+
freeTextures: [],
|
|
94
139
|
cpuPixels: null,
|
|
95
140
|
cpuReservation: 0,
|
|
96
141
|
cpuBytesPerRow: 0,
|
|
97
142
|
pendingSources: [],
|
|
98
143
|
yuvImports: new Map(),
|
|
144
|
+
checkedWaits: 0,
|
|
145
|
+
inFlight: [],
|
|
99
146
|
};
|
|
100
147
|
}
|
|
101
148
|
return runtime.__rnskwgpuFrames;
|
|
@@ -116,12 +163,200 @@ const clearEntry = (cache: Cache, producer: number) => {
|
|
|
116
163
|
* The caller has either not submitted a copy or has completed its GPU fence. */
|
|
117
164
|
const clearSurfaceView = (cache: Cache) => {
|
|
118
165
|
"worklet";
|
|
119
|
-
cache.canvas?.dispose();
|
|
120
166
|
cache.canvas = null;
|
|
121
167
|
cache.stage?.dispose();
|
|
122
168
|
cache.stage = null;
|
|
123
169
|
};
|
|
124
170
|
|
|
171
|
+
/** Waits for all the GPU work submitted so far, on the shared context. */
|
|
172
|
+
const waitForGpu = (cache: Cache): void => {
|
|
173
|
+
"worklet";
|
|
174
|
+
if (cache.stage) cache.stage.flush(true);
|
|
175
|
+
else {
|
|
176
|
+
let texture = cache.uploadTexture ?? cache.freeTextures[0]?.texture;
|
|
177
|
+
if (!texture) {
|
|
178
|
+
for (const target of cache.targets.values()) {
|
|
179
|
+
texture = target.textures[0]?.texture;
|
|
180
|
+
if (texture) break;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
// A released producer's textures may all be gone (too big to keep free)
|
|
184
|
+
// while its imports are still in flight: any Graphite surface's checked
|
|
185
|
+
// flush waits on the shared context just the same.
|
|
186
|
+
const view = texture
|
|
187
|
+
? Skia.Surface.MakeFromGPUTexture(texture)
|
|
188
|
+
: Skia.Surface.MakeOffscreen(1, 1);
|
|
189
|
+
if (!view) throw new Error("Video import drain surface is unavailable");
|
|
190
|
+
try {
|
|
191
|
+
view.flush(true);
|
|
192
|
+
} finally {
|
|
193
|
+
view.dispose();
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
cache.checkedWaits++;
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
/** Hands the decoder buffers of the first `count` imports in flight back,
|
|
200
|
+
* once the GPU is done with them: because the GPU reported the last of these
|
|
201
|
+
* imports done (in order: the work before it is done too), or after a wait
|
|
202
|
+
* for it, made here when `count` is all of them and nobody `waited`. */
|
|
203
|
+
const settleInFlightImports = (
|
|
204
|
+
cache: Cache,
|
|
205
|
+
count: number,
|
|
206
|
+
waited = false,
|
|
207
|
+
): void => {
|
|
208
|
+
"worklet";
|
|
209
|
+
if (!count) return;
|
|
210
|
+
if (count === cache.inFlight.length && !waited) waitForGpu(cache);
|
|
211
|
+
for (let i = 0; i < count; i++) {
|
|
212
|
+
const entry = cache.inFlight.shift()!;
|
|
213
|
+
try {
|
|
214
|
+
entry.nativeSource?.release();
|
|
215
|
+
entry.nativeSource = null;
|
|
216
|
+
entry.frame.dispose?.();
|
|
217
|
+
} catch (error) {
|
|
218
|
+
// Like a failed release at import: only the final drain retries it.
|
|
219
|
+
cache.pendingSources.push({
|
|
220
|
+
nativeSource: entry.nativeSource,
|
|
221
|
+
owned: null,
|
|
222
|
+
reservation: 0,
|
|
223
|
+
frame: entry.frame,
|
|
224
|
+
needsDrain: false,
|
|
225
|
+
sourceReleased: false,
|
|
226
|
+
});
|
|
227
|
+
throw error;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
// Free textures kept for the next producer, when they are small enough.
|
|
233
|
+
const maxFreeImportTextures = 2;
|
|
234
|
+
|
|
235
|
+
/** Work already submitted still completes on a destroyed texture; a caller
|
|
236
|
+
* that cannot tell whether everything reading it was submitted waits first. */
|
|
237
|
+
const destroyImportTexture = (cache: Cache, entry: ImportTexture) => {
|
|
238
|
+
"worklet";
|
|
239
|
+
if (cache.uploadTexture === entry.texture) {
|
|
240
|
+
clearSurfaceView(cache);
|
|
241
|
+
cache.uploadTexture = null;
|
|
242
|
+
}
|
|
243
|
+
entry.texture.destroy();
|
|
244
|
+
releaseVideoMemory(entry.reservation);
|
|
245
|
+
entry.reservation = 0;
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
// A texture leaves its list only once destroyed: a failed teardown is
|
|
249
|
+
// retried by the next one, its memory still accounted.
|
|
250
|
+
const releaseFreeImportTextures = (cache: Cache) => {
|
|
251
|
+
"worklet";
|
|
252
|
+
while (cache.freeTextures.length) {
|
|
253
|
+
destroyImportTexture(cache, cache.freeTextures[cache.freeTextures.length - 1]!);
|
|
254
|
+
cache.freeTextures.pop();
|
|
255
|
+
}
|
|
256
|
+
};
|
|
257
|
+
|
|
258
|
+
/** Gives up the textures of a producer whose image is disposed. Small ones
|
|
259
|
+
* are kept for the next producer of that size (a montage goes from clip to
|
|
260
|
+
* clip), within the budget's sixteenth each, as for imports in flight. */
|
|
261
|
+
const releaseImportTarget = (cache: Cache, producer: number, keep = true) => {
|
|
262
|
+
"worklet";
|
|
263
|
+
const target = cache.targets.get(producer);
|
|
264
|
+
if (!target) return;
|
|
265
|
+
const budget = videoMemoryBudget();
|
|
266
|
+
while (target.textures.length) {
|
|
267
|
+
const entry = target.textures[target.textures.length - 1]!;
|
|
268
|
+
if (
|
|
269
|
+
keep &&
|
|
270
|
+
cache.freeTextures.length < maxFreeImportTextures &&
|
|
271
|
+
!!budget &&
|
|
272
|
+
frameByteSize(entry.width, entry.height) <= budget.maxBytes / 16
|
|
273
|
+
)
|
|
274
|
+
cache.freeTextures.push(entry);
|
|
275
|
+
else destroyImportTexture(cache, entry);
|
|
276
|
+
target.textures.pop();
|
|
277
|
+
}
|
|
278
|
+
cache.targets.delete(producer);
|
|
279
|
+
};
|
|
280
|
+
|
|
281
|
+
/** The texture the next frame of `producer` goes into: one that its current
|
|
282
|
+
* image does not wrap, when the producer has or can afford a second one. */
|
|
283
|
+
const acquireImportTexture = (
|
|
284
|
+
cache: Cache,
|
|
285
|
+
device: GPUDevice,
|
|
286
|
+
producer: number,
|
|
287
|
+
width: number,
|
|
288
|
+
height: number,
|
|
289
|
+
format: GPUTextureFormat,
|
|
290
|
+
usage: number,
|
|
291
|
+
label: string,
|
|
292
|
+
): GPUTexture => {
|
|
293
|
+
"worklet";
|
|
294
|
+
let target = cache.targets.get(producer);
|
|
295
|
+
if (
|
|
296
|
+
target &&
|
|
297
|
+
target.textures.some(
|
|
298
|
+
(entry) =>
|
|
299
|
+
entry.width !== width ||
|
|
300
|
+
entry.height !== height ||
|
|
301
|
+
entry.format !== format,
|
|
302
|
+
)
|
|
303
|
+
) {
|
|
304
|
+
// The producer changed size: its image wraps a texture given up here.
|
|
305
|
+
clearEntry(cache, producer);
|
|
306
|
+
releaseImportTarget(cache, producer, false);
|
|
307
|
+
target = undefined;
|
|
308
|
+
}
|
|
309
|
+
if (!target) {
|
|
310
|
+
target = { textures: [] };
|
|
311
|
+
cache.targets.set(producer, target);
|
|
312
|
+
}
|
|
313
|
+
const shown = cache.images.get(producer);
|
|
314
|
+
const live = shown?.image
|
|
315
|
+
? target.textures.findIndex((entry) => entry.texture === shown.texture)
|
|
316
|
+
: -1;
|
|
317
|
+
let index = target.textures.findIndex((_, i) => i !== live);
|
|
318
|
+
if (index < 0) {
|
|
319
|
+
const free = cache.freeTextures.findIndex(
|
|
320
|
+
(entry) =>
|
|
321
|
+
entry.width === width &&
|
|
322
|
+
entry.height === height &&
|
|
323
|
+
entry.format === format,
|
|
324
|
+
);
|
|
325
|
+
const bytes = frameByteSize(width, height);
|
|
326
|
+
const budget = videoMemoryBudget();
|
|
327
|
+
if (free >= 0) {
|
|
328
|
+
target.textures.push(cache.freeTextures.splice(free, 1)[0]!);
|
|
329
|
+
index = target.textures.length - 1;
|
|
330
|
+
} else if (
|
|
331
|
+
!target.textures.length ||
|
|
332
|
+
!budget ||
|
|
333
|
+
budget.maxBytes - budget.currentBytes >= 3 * bytes
|
|
334
|
+
) {
|
|
335
|
+
// Free textures of another size only hold memory back from here on.
|
|
336
|
+
releaseFreeImportTextures(cache);
|
|
337
|
+
const reservation = reserveVideoMemory(bytes, label);
|
|
338
|
+
let texture: GPUTexture;
|
|
339
|
+
try {
|
|
340
|
+
texture = device.createTexture({
|
|
341
|
+
size: { width, height },
|
|
342
|
+
format,
|
|
343
|
+
usage,
|
|
344
|
+
label,
|
|
345
|
+
});
|
|
346
|
+
} catch (error) {
|
|
347
|
+
releaseVideoMemory(reservation);
|
|
348
|
+
throw error;
|
|
349
|
+
}
|
|
350
|
+
target.textures.push({ texture, width, height, format, reservation });
|
|
351
|
+
index = target.textures.length - 1;
|
|
352
|
+
} else {
|
|
353
|
+
// No room for a second texture: write over the current frame.
|
|
354
|
+
index = live;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
return target.textures[index]!.texture;
|
|
358
|
+
};
|
|
359
|
+
|
|
125
360
|
const clearCpuPixels = (cache: Cache) => {
|
|
126
361
|
"worklet";
|
|
127
362
|
cache.cpuPixels = null;
|
|
@@ -132,34 +367,44 @@ const clearCpuPixels = (cache: Cache) => {
|
|
|
132
367
|
|
|
133
368
|
const clearStagingSurface = (cache: Cache) => {
|
|
134
369
|
"worklet";
|
|
370
|
+
settleInFlightImports(cache, cache.inFlight.length);
|
|
135
371
|
cache.stage?.flush(true);
|
|
136
372
|
clearSurfaceView(cache);
|
|
137
|
-
cache.
|
|
373
|
+
for (const producer of [...cache.targets.keys()])
|
|
374
|
+
releaseImportTarget(cache, producer, false);
|
|
375
|
+
releaseFreeImportTextures(cache);
|
|
138
376
|
cache.uploadTexture = null;
|
|
139
|
-
releaseVideoMemory(cache.uploadReservation);
|
|
140
|
-
cache.uploadReservation = 0;
|
|
141
377
|
clearCpuPixels(cache);
|
|
142
|
-
cache.width = cache.height = 0;
|
|
143
|
-
cache.format = null;
|
|
144
378
|
};
|
|
145
379
|
|
|
146
380
|
/** Record explicit recovery work after a possibly submitted external copy.
|
|
147
|
-
*
|
|
148
|
-
*
|
|
381
|
+
* It clears the texture of the import that failed, whose image is dropped;
|
|
382
|
+
* with two textures, the producer's previous image is in the other one. The
|
|
383
|
+
* checked fence must succeed before either native lease or the texture can be released. */
|
|
149
384
|
const drainNativeCopies = (cache: Cache): void => {
|
|
150
385
|
"worklet";
|
|
151
386
|
if (!cache.stage || !cache.canvas)
|
|
152
387
|
throw new Error("Video import drain surface is unavailable");
|
|
153
388
|
cache.canvas.drawColor(Skia.Color("#00000000"), BlendMode.Clear);
|
|
154
389
|
cache.stage.flush(true);
|
|
390
|
+
cache.checkedWaits++;
|
|
391
|
+
};
|
|
392
|
+
|
|
393
|
+
/** The checked GPU waits this runtime's imports have made so far, and the
|
|
394
|
+
* completions the GPU reported for them. Each one covers all the GPU work
|
|
395
|
+
* submitted before it, on the shared context. */
|
|
396
|
+
export const videoFrameImportWaits = (): number => {
|
|
397
|
+
"worklet";
|
|
398
|
+
return getCache().checkedWaits;
|
|
155
399
|
};
|
|
156
400
|
|
|
157
401
|
/** Clear a runtime's imported images after ending all its producers.
|
|
158
402
|
* Do not clear a paused producer's current frame: its native source has
|
|
159
|
-
* already been released, and
|
|
403
|
+
* already been released, and its image's texture is its remaining pixels. */
|
|
160
404
|
export const clearVideoFrameImages = (releaseStagingSurface = false): void => {
|
|
161
405
|
"worklet";
|
|
162
406
|
const cache = getCache();
|
|
407
|
+
settleInFlightImports(cache, cache.inFlight.length);
|
|
163
408
|
if (cache.pendingSources.length) {
|
|
164
409
|
// A failed drain retains leases; only a successful fence can release them.
|
|
165
410
|
if (cache.pendingSources.some((source) => source.needsDrain)) {
|
|
@@ -203,18 +448,30 @@ export const releaseVideoFrameImage = (frame: VideoFrame): void => {
|
|
|
203
448
|
const producer = frame.producerId ?? frame.id ?? -1;
|
|
204
449
|
const cache = getCache();
|
|
205
450
|
const entry = cache.images.get(producer);
|
|
206
|
-
if (entry && entry.id === (frame.id ?? -1))
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
451
|
+
if (entry && entry.id === (frame.id ?? -1)) {
|
|
452
|
+
clearEntry(cache, producer);
|
|
453
|
+
releaseImportTarget(cache, producer);
|
|
454
|
+
}
|
|
455
|
+
// Every import is done once the GPU is; after a failed one, only the drain
|
|
456
|
+
// in clearVideoFrameImages may release the decoder buffers.
|
|
457
|
+
if (!cache.pendingSources.length && cache.yuvImports.has(producer)) {
|
|
458
|
+
settleInFlightImports(cache, cache.inFlight.length);
|
|
210
459
|
releaseYuvImports(cache.yuvImports, producer);
|
|
460
|
+
}
|
|
211
461
|
if (cache.images.size === 0) clearVideoFrameImages(true);
|
|
212
462
|
};
|
|
213
463
|
|
|
214
|
-
/** Returns
|
|
215
|
-
*
|
|
216
|
-
*
|
|
217
|
-
*
|
|
464
|
+
/** Returns owned pixels. WebGPU uploads Android RGBA pixels or blits native
|
|
465
|
+
* buffers into a texture of the frame's producer, which the image wraps: no
|
|
466
|
+
* copy into a snapshot, and no entry in the raster ImageProvider's LRU.
|
|
467
|
+
* Sources go back once the GPU is done with them.
|
|
468
|
+
*
|
|
469
|
+
* The image is the producer's until its next frame is imported, which
|
|
470
|
+
* disposes it. A producer has two textures and writes its frames into them in
|
|
471
|
+
* turn, so the texture of an image is only written again by the import after
|
|
472
|
+
* the one that disposed it; with a memory budget too short for a second
|
|
473
|
+
* texture, the next frame is written over it, in queue order after the draws
|
|
474
|
+
* already submitted. */
|
|
218
475
|
export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
219
476
|
"worklet";
|
|
220
477
|
const cache = getCache();
|
|
@@ -223,7 +480,7 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
223
480
|
const existing = cache.images.get(producer);
|
|
224
481
|
if (existing?.id === frame.id && existing.image) {
|
|
225
482
|
// Android can issue a fresh lease for the same paused picture. The
|
|
226
|
-
//
|
|
483
|
+
// image already owns its pixels, so close that unused native lease.
|
|
227
484
|
// Weak tracking avoids retaining raw buffers or closing one wrapper twice.
|
|
228
485
|
if (!existing.releasedSources.has(frame)) {
|
|
229
486
|
frame.dispose?.();
|
|
@@ -238,6 +495,9 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
238
495
|
let ownsSurfaceView = false;
|
|
239
496
|
let nativeSource: NativeVideoFrame | null = null;
|
|
240
497
|
let sourceDrained = false;
|
|
498
|
+
let completion: Promise<undefined> | null = null;
|
|
499
|
+
let sourceBytes = 0;
|
|
500
|
+
let importTexture: GPUTexture | null = null;
|
|
241
501
|
let releaseSource = true;
|
|
242
502
|
let importFailed = false;
|
|
243
503
|
try {
|
|
@@ -311,7 +571,7 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
311
571
|
);
|
|
312
572
|
const data = Skia.Data.fromBytes(pixels);
|
|
313
573
|
try {
|
|
314
|
-
image = Skia.Image.
|
|
574
|
+
image = Skia.Image.MakeRaster(
|
|
315
575
|
{
|
|
316
576
|
width: frame.width,
|
|
317
577
|
height: frame.height,
|
|
@@ -344,49 +604,25 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
344
604
|
}
|
|
345
605
|
const format =
|
|
346
606
|
uploadRgba || uploadAndroidRgba ? "rgba8unorm" : "bgra8unorm";
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
format,
|
|
367
|
-
// Android AHB and CPU frames share the RGBA scratch. Include
|
|
368
|
-
// COPY_DST even for a first AHB frame, so a later CPU fallback
|
|
369
|
-
// cannot reuse a texture that forbids writeTexture.
|
|
370
|
-
usage:
|
|
371
|
-
uploadRgba || uploadAndroidRgba
|
|
372
|
-
? rgbaVideoTextureUsage
|
|
373
|
-
: nativeVideoTextureUsage,
|
|
374
|
-
label: uploadRgba
|
|
375
|
-
? "video RGBA frame upload"
|
|
376
|
-
: "video native frame upload",
|
|
377
|
-
});
|
|
378
|
-
cache.width = frame.width;
|
|
379
|
-
cache.height = frame.height;
|
|
380
|
-
cache.format = format;
|
|
381
|
-
} catch (error) {
|
|
382
|
-
try {
|
|
383
|
-
clearStagingSurface(cache);
|
|
384
|
-
} catch {
|
|
385
|
-
/* unfinished cleanup/accounting stays reachable */
|
|
386
|
-
}
|
|
387
|
-
throw error;
|
|
388
|
-
}
|
|
389
|
-
}
|
|
607
|
+
// The frame is converted straight into a texture of its producer, which
|
|
608
|
+
// the image wraps: no copy into a snapshot. Android AHB and CPU frames
|
|
609
|
+
// share RGBA textures, COPY_DST included even for a first AHB frame, so
|
|
610
|
+
// that a later CPU fallback can write into them.
|
|
611
|
+
clearSurfaceView(cache);
|
|
612
|
+
cache.uploadTexture = null;
|
|
613
|
+
cache.uploadTexture = acquireImportTexture(
|
|
614
|
+
cache,
|
|
615
|
+
device,
|
|
616
|
+
key,
|
|
617
|
+
frame.width,
|
|
618
|
+
frame.height,
|
|
619
|
+
format,
|
|
620
|
+
uploadRgba || uploadAndroidRgba
|
|
621
|
+
? rgbaVideoTextureUsage
|
|
622
|
+
: nativeVideoTextureUsage,
|
|
623
|
+
uploadRgba ? "video RGBA frame" : "video native frame",
|
|
624
|
+
);
|
|
625
|
+
importTexture = cache.uploadTexture;
|
|
390
626
|
if (copyRgbaPixels && texture.kind === "rgba") {
|
|
391
627
|
if (
|
|
392
628
|
!cache.cpuPixels ||
|
|
@@ -423,9 +659,8 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
423
659
|
clearCpuPixels(cache);
|
|
424
660
|
}
|
|
425
661
|
clearSurfaceView(cache);
|
|
426
|
-
//
|
|
427
|
-
//
|
|
428
|
-
// GPU allocation or a write through Skia that would overwrite the input.
|
|
662
|
+
// The surface only carries the checked submissions and waits of this
|
|
663
|
+
// import, and its recovery drain; the image wraps the texture itself.
|
|
429
664
|
cache.stage = Skia.Surface.MakeFromGPUTexture(cache.uploadTexture!);
|
|
430
665
|
if (!cache.stage) throw new Error("Cannot wrap video upload texture");
|
|
431
666
|
ownsSurfaceView = true;
|
|
@@ -449,14 +684,15 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
449
684
|
const region = texture.region;
|
|
450
685
|
const rotation = texture.rotation ?? 0;
|
|
451
686
|
const nv12 = !region && texture.format === "nv12";
|
|
452
|
-
let
|
|
687
|
+
let upright: ReturnType<typeof yuvFrameRegion> | null = null;
|
|
453
688
|
if (region) {
|
|
454
689
|
// An Android codec buffer: padded, YUV, in encoded orientation. The
|
|
455
|
-
// blit converts, crops and
|
|
456
|
-
// as the GL path's SurfaceTexture transform did. RN WebGPU
|
|
457
|
-
// driver-defined YUV formats "bgra8", yet Dawn imports them
|
|
458
|
-
// the native side checked the format, so the label is not
|
|
459
|
-
|
|
690
|
+
// blit converts, crops, turns and scales it into the frame-sized
|
|
691
|
+
// scratch, as the GL path's SurfaceTexture transform did. RN WebGPU
|
|
692
|
+
// labels driver-defined YUV formats "bgra8", yet Dawn imports them
|
|
693
|
+
// as YUV: the native side checked the format, so the label is not
|
|
694
|
+
// read.
|
|
695
|
+
upright = yuvFrameRegion(region, rotation, nativeSource);
|
|
460
696
|
} else if (nv12) {
|
|
461
697
|
if (
|
|
462
698
|
nativeSource.width !== frame.width ||
|
|
@@ -487,9 +723,7 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
487
723
|
key,
|
|
488
724
|
nativeSource,
|
|
489
725
|
rotation,
|
|
490
|
-
|
|
491
|
-
frame.width,
|
|
492
|
-
frame.height,
|
|
726
|
+
upright!,
|
|
493
727
|
cache.uploadTexture!,
|
|
494
728
|
);
|
|
495
729
|
} else if (nv12) {
|
|
@@ -511,25 +745,84 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
|
|
|
511
745
|
}
|
|
512
746
|
// Snapshot records an independent texture copy and submits it after
|
|
513
747
|
// the external blit on the shared queue. The wrapper never draws.
|
|
514
|
-
image =
|
|
515
|
-
|
|
516
|
-
|
|
748
|
+
image = Skia.Image.MakeFromGPUTexture(cache.uploadTexture!);
|
|
749
|
+
// Submitted without waiting for the GPU, which held the UI thread for
|
|
750
|
+
// the work of every Skia view on the shared context: the decoder buffer
|
|
751
|
+
// is handed back once the GPU reports the copy done. A wait still
|
|
752
|
+
// bounds the buffers out, and stands in where nothing can report.
|
|
753
|
+
// NV12 and Android's YUV buffers take a byte and a half per pixel.
|
|
754
|
+
sourceBytes =
|
|
755
|
+
texture.kind === "native-buffer" &&
|
|
756
|
+
(texture.region || texture.format === "nv12")
|
|
757
|
+
? Math.ceil(bytes * 0.375)
|
|
758
|
+
: bytes;
|
|
759
|
+
const wait =
|
|
760
|
+
typeof device.queue.onSubmittedWorkDone !== "function" ||
|
|
761
|
+
cache.inFlight.length >= maxInFlightImports ||
|
|
762
|
+
!affordsInFlight(sourceBytes);
|
|
763
|
+
cache.stage.flush(wait);
|
|
764
|
+
if (!wait) {
|
|
765
|
+
try {
|
|
766
|
+
completion = device.queue.onSubmittedWorkDone();
|
|
767
|
+
} catch {
|
|
768
|
+
// A runtime WebGPU cannot settle promises on.
|
|
769
|
+
completion = null;
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
if (wait || !completion) {
|
|
773
|
+
if (!wait) cache.stage.flush(true);
|
|
774
|
+
cache.checkedWaits++;
|
|
775
|
+
// Every import before this one is done too.
|
|
776
|
+
settleInFlightImports(cache, cache.inFlight.length, true);
|
|
777
|
+
completion = null;
|
|
778
|
+
sourceDrained = true;
|
|
779
|
+
}
|
|
517
780
|
}
|
|
518
781
|
if (!image) throw new Error("Cannot create a video frame image");
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
782
|
+
if (completion) {
|
|
783
|
+
// The GPU copy may still read the decoder buffer: it goes back once the
|
|
784
|
+
// GPU reports the copy done, or after a wait (settleInFlightImports).
|
|
785
|
+
// The view stays for that wait.
|
|
786
|
+
const entry: InFlightImport = { frame, nativeSource, bytes: sourceBytes };
|
|
787
|
+
cache.inFlight.push(entry);
|
|
788
|
+
nativeSource = null;
|
|
789
|
+
releaseSource = false;
|
|
524
790
|
ownsSurfaceView = false;
|
|
791
|
+
completion.then(
|
|
792
|
+
() => {
|
|
793
|
+
const settled = cache.inFlight.indexOf(entry) + 1;
|
|
794
|
+
// Zero: a wait handed it back already.
|
|
795
|
+
if (!settled) return;
|
|
796
|
+
cache.checkedWaits++;
|
|
797
|
+
try {
|
|
798
|
+
settleInFlightImports(cache, settled, true);
|
|
799
|
+
} catch (error) {
|
|
800
|
+
console.warn("Video frame release failed", error);
|
|
801
|
+
}
|
|
802
|
+
},
|
|
803
|
+
// Handed back by the next wait instead.
|
|
804
|
+
() => undefined,
|
|
805
|
+
);
|
|
806
|
+
} else {
|
|
807
|
+
// Complete the native handoff before replacing published pixels. A
|
|
808
|
+
// failed source close must not evict a paused image or leave an
|
|
809
|
+
// unpublished cache entry behind when the caller receives an import
|
|
810
|
+
// error.
|
|
811
|
+
if (ownsSurfaceView) {
|
|
812
|
+
clearSurfaceView(cache);
|
|
813
|
+
ownsSurfaceView = false;
|
|
814
|
+
}
|
|
815
|
+
nativeSource?.release();
|
|
816
|
+
nativeSource = null;
|
|
817
|
+
frame.dispose?.();
|
|
818
|
+
releaseSource = false;
|
|
525
819
|
}
|
|
526
|
-
nativeSource?.release();
|
|
527
|
-
nativeSource = null;
|
|
528
|
-
frame.dispose?.();
|
|
529
|
-
releaseSource = false;
|
|
530
820
|
clearEntry(cache, key);
|
|
531
821
|
cache.images.set(key, {
|
|
532
|
-
id: frame.id ?? -1,
|
|
822
|
+
id: frame.id ?? -1,
|
|
823
|
+
image,
|
|
824
|
+
reservation,
|
|
825
|
+
texture: importTexture,
|
|
533
826
|
releasedSources: new WeakSet([frame]),
|
|
534
827
|
});
|
|
535
828
|
return image;
|
package/src/gpu.ts
CHANGED
|
@@ -166,7 +166,7 @@ export const createVideoGpuScope = (options?: {
|
|
|
166
166
|
resource.surface = Skia.Surface.MakeFromGPUTexture(resource.texture);
|
|
167
167
|
resource.canvas = getVideoCanvas(resource.surface);
|
|
168
168
|
} else if (skia === "image") {
|
|
169
|
-
resource.image = Skia.Image.
|
|
169
|
+
resource.image = Skia.Image.MakeFromGPUTexture(resource.texture);
|
|
170
170
|
}
|
|
171
171
|
resources.push(resource);
|
|
172
172
|
return {
|
|
@@ -178,7 +178,6 @@ export const createVideoGpuScope = (options?: {
|
|
|
178
178
|
} catch (error) {
|
|
179
179
|
// No texture has been exposed or submitted to any consumer yet.
|
|
180
180
|
try {
|
|
181
|
-
resource.canvas?.dispose();
|
|
182
181
|
resource.canvas = null;
|
|
183
182
|
resource.surface?.dispose();
|
|
184
183
|
resource.surface = null;
|
|
@@ -265,7 +264,6 @@ export const createVideoGpuScope = (options?: {
|
|
|
265
264
|
try {
|
|
266
265
|
resource.image?.dispose();
|
|
267
266
|
resource.image = null;
|
|
268
|
-
resource.canvas?.dispose();
|
|
269
267
|
resource.canvas = null;
|
|
270
268
|
resource.surface?.dispose();
|
|
271
269
|
resource.surface = null;
|