react-native-skia-webgpu-video 0.1.0-beta.3 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +48 -43
  2. package/android/BUILD_VALIDATION.md +1 -1
  3. package/android/CMakeLists.txt +3 -2
  4. package/android/cpp/ExportHintSession.cpp +114 -0
  5. package/android/cpp/ExportHintSession.h +35 -0
  6. package/android/cpp/VideoEncoderHostObject.cpp +76 -101
  7. package/android/cpp/VideoEncoderHostObject.h +14 -9
  8. package/android/cpp/VideoFrame.cpp +3 -5
  9. package/android/cpp/cpp-adapter.cpp +7 -4
  10. package/android/src/main/java/com/azzapp/rnskv/GLFrameExtractor.java +3 -9
  11. package/android/src/main/java/com/azzapp/rnskv/VideoCompositionDecoder.java +8 -11
  12. package/android/src/main/java/com/azzapp/rnskv/VideoEncoder.java +68 -28
  13. package/docs/FRAME_PIXEL_FORMAT.md +17 -2
  14. package/docs/MIGRATION.md +1 -1
  15. package/docs/UPSTREAM_NOTES.md +42 -2
  16. package/ios/RNSVColorSpace.h +8 -3
  17. package/lib/module/canvas.js +3 -5
  18. package/lib/module/canvas.js.map +1 -1
  19. package/lib/module/exportFrames.js +8 -10
  20. package/lib/module/exportFrames.js.map +1 -1
  21. package/lib/module/exportTarget.js +183 -40
  22. package/lib/module/exportTarget.js.map +1 -1
  23. package/lib/module/exportVideoComposition.js +78 -35
  24. package/lib/module/exportVideoComposition.js.map +1 -1
  25. package/lib/module/frameInterop.js +323 -82
  26. package/lib/module/frameInterop.js.map +1 -1
  27. package/lib/module/gpu.js +1 -3
  28. package/lib/module/gpu.js.map +1 -1
  29. package/lib/module/gpuDevice.js +12 -0
  30. package/lib/module/gpuDevice.js.map +1 -1
  31. package/lib/module/memory.js +7 -0
  32. package/lib/module/memory.js.map +1 -1
  33. package/lib/module/previewLifecycle.js +2 -1
  34. package/lib/module/previewLifecycle.js.map +1 -1
  35. package/lib/module/videoCompositionPlayer.js +65 -14
  36. package/lib/module/videoCompositionPlayer.js.map +1 -1
  37. package/lib/module/videoFrameImage.js +1 -1
  38. package/lib/module/yuvBlit.js +31 -24
  39. package/lib/module/yuvBlit.js.map +1 -1
  40. package/lib/typescript/src/canvas.d.ts.map +1 -1
  41. package/lib/typescript/src/exportFrames.d.ts +3 -2
  42. package/lib/typescript/src/exportFrames.d.ts.map +1 -1
  43. package/lib/typescript/src/exportTarget.d.ts +53 -14
  44. package/lib/typescript/src/exportTarget.d.ts.map +1 -1
  45. package/lib/typescript/src/exportVideoComposition.d.ts +2 -1
  46. package/lib/typescript/src/exportVideoComposition.d.ts.map +1 -1
  47. package/lib/typescript/src/frameInterop.d.ts +16 -5
  48. package/lib/typescript/src/frameInterop.d.ts.map +1 -1
  49. package/lib/typescript/src/gpu.d.ts.map +1 -1
  50. package/lib/typescript/src/gpuDevice.d.ts +5 -1
  51. package/lib/typescript/src/gpuDevice.d.ts.map +1 -1
  52. package/lib/typescript/src/memory.d.ts +2 -0
  53. package/lib/typescript/src/memory.d.ts.map +1 -1
  54. package/lib/typescript/src/previewLifecycle.d.ts +2 -0
  55. package/lib/typescript/src/previewLifecycle.d.ts.map +1 -1
  56. package/lib/typescript/src/types.d.ts +18 -12
  57. package/lib/typescript/src/types.d.ts.map +1 -1
  58. package/lib/typescript/src/videoCompositionPlayer.d.ts +4 -5
  59. package/lib/typescript/src/videoCompositionPlayer.d.ts.map +1 -1
  60. package/lib/typescript/src/videoFrameImage.d.ts +1 -1
  61. package/lib/typescript/src/yuvBlit.d.ts +16 -14
  62. package/lib/typescript/src/yuvBlit.d.ts.map +1 -1
  63. package/package.json +6 -6
  64. package/patches/react-native-skia+3.10.0.patch +150 -0
  65. package/scripts/skia-canvas-patch.json +22 -258
  66. package/src/canvas.ts +3 -7
  67. package/src/exportFrames.ts +10 -11
  68. package/src/exportTarget.ts +217 -56
  69. package/src/exportVideoComposition.ts +130 -85
  70. package/src/frameInterop.ts +389 -96
  71. package/src/gpu.ts +1 -3
  72. package/src/gpuDevice.ts +26 -1
  73. package/src/memory.ts +6 -0
  74. package/src/previewLifecycle.ts +4 -1
  75. package/src/types.ts +18 -12
  76. package/src/videoCompositionPlayer.ts +76 -23
  77. package/src/videoFrameImage.ts +1 -1
  78. package/src/yuvBlit.ts +46 -31
  79. package/android/cpp/EncoderTargets.cpp +0 -226
  80. package/android/cpp/EncoderTargets.h +0 -70
  81. package/patches/react-native-skia+3.7.1.patch +0 -502
@@ -14,12 +14,13 @@ import {
14
14
  frameByteSize,
15
15
  releaseVideoMemory,
16
16
  reserveVideoMemory,
17
+ videoMemoryBudget,
17
18
  } from "./memory";
18
19
  import {
19
20
  blitNv12Frame,
20
21
  blitYuvFrame,
21
22
  releaseYuvImports,
22
- yuvFrameOrigin,
23
+ yuvFrameRegion,
23
24
  } from "./yuvBlit";
24
25
  import type { YuvImport } from "./yuvBlit";
25
26
 
@@ -44,9 +45,18 @@ type FrameTexture =
44
45
  type Entry = {
45
46
  id: number;
46
47
  image: SkImage | null;
48
+ /** The import texture the image wraps, if any. */
49
+ texture?: GPUTexture | null;
47
50
  reservation: number;
48
51
  releasedSources: WeakSet<VideoFrame>;
49
52
  };
53
+ /** A decoder buffer an import's GPU copy still reads. */
54
+ type InFlightImport = {
55
+ frame: VideoFrame;
56
+ nativeSource: NativeVideoFrame | null;
57
+ /** The decoder buffer's size, still reserved in the memory budget. */
58
+ bytes: number;
59
+ };
50
60
  type PendingSource = {
51
61
  owned: SkImage | null;
52
62
  reservation: number;
@@ -55,21 +65,58 @@ type PendingSource = {
55
65
  sourceReleased: boolean;
56
66
  nativeSource?: NativeVideoFrame | null;
57
67
  };
68
+ /** A texture frames are converted into, and that their images wrap. */
69
+ type ImportTexture = {
70
+ texture: GPUTexture;
71
+ width: number;
72
+ height: number;
73
+ format: GPUTextureFormat;
74
+ reservation: number;
75
+ };
76
+ /** The textures of a producer: its frames go into them in turn, so that the
77
+ * image of the previous frame stays whole while the next is written. One
78
+ * texture only, written in place, when the memory budget is short. */
79
+ type ImportTarget = { textures: ImportTexture[] };
58
80
  type Cache = {
59
81
  images: Map<number, Entry>;
60
82
  stage: SkSurface | null;
61
83
  canvas: SkCanvas | null;
62
- width: number;
63
- height: number;
64
- format: GPUTextureFormat | null;
84
+ /** The texture `stage` wraps: the one of the last import, owned by its
85
+ * producer's target or by `freeTextures`. */
65
86
  uploadTexture: GPUTexture | null;
66
- uploadReservation: number;
87
+ targets: Map<number, ImportTarget>;
88
+ /** Textures of released producers, for the next producer of that size. */
89
+ freeTextures: ImportTexture[];
67
90
  cpuPixels: Uint8Array<ArrayBuffer> | null;
68
91
  cpuReservation: number;
69
92
  cpuBytesPerRow: number;
70
93
  pendingSources: PendingSource[];
71
94
  /** Android decoder buffers imported once, per producer. */
72
95
  yuvImports: Map<number, YuvImport[]>;
96
+ /** Checked GPU waits so far (see videoFrameImportWaits). */
97
+ checkedWaits: number;
98
+ /** Oldest first; at most maxInFlightImports. */
99
+ inFlight: InFlightImport[];
100
+ };
101
+
102
+ // Imports whose GPU copy may still be running, their decoder buffers out. Kept
103
+ // below the YUV imports cached per producer (yuvBlit.ts), so that the one it
104
+ // evicts, the least recently drawn, is never still being read.
105
+ const maxInFlightImports = 2;
106
+
107
+ /** Whether the memory budget affords keeping a decoder buffer of `bytes` out
108
+ * until the GPU is done with it: a buffer within a sixteenth of the budget
109
+ * (two out stay within an eighth), and room left for the decoder's next two
110
+ * frames, which it reserves ahead. With the default 256 MB, a 4K BGRA frame
111
+ * (33 MB) is waited for; two 4K NV12 ones (12 MB each) are not. */
112
+ const affordsInFlight = (bytes: number): boolean => {
113
+ "worklet";
114
+ const budget = videoMemoryBudget();
115
+ return (
116
+ !!budget &&
117
+ bytes <= budget.maxBytes / 16 &&
118
+ budget.maxBytes - budget.currentBytes >= 2 * bytes
119
+ );
73
120
  };
74
121
 
75
122
  // Capture a scalar on the RN runtime instead of importing Platform inside a
@@ -86,16 +133,16 @@ const getCache = (): Cache => {
86
133
  images: new Map(),
87
134
  stage: null,
88
135
  canvas: null,
89
- width: 0,
90
- height: 0,
91
- format: null,
92
136
  uploadTexture: null,
93
- uploadReservation: 0,
137
+ targets: new Map(),
138
+ freeTextures: [],
94
139
  cpuPixels: null,
95
140
  cpuReservation: 0,
96
141
  cpuBytesPerRow: 0,
97
142
  pendingSources: [],
98
143
  yuvImports: new Map(),
144
+ checkedWaits: 0,
145
+ inFlight: [],
99
146
  };
100
147
  }
101
148
  return runtime.__rnskwgpuFrames;
@@ -116,12 +163,200 @@ const clearEntry = (cache: Cache, producer: number) => {
116
163
  * The caller has either not submitted a copy or has completed its GPU fence. */
117
164
  const clearSurfaceView = (cache: Cache) => {
118
165
  "worklet";
119
- cache.canvas?.dispose();
120
166
  cache.canvas = null;
121
167
  cache.stage?.dispose();
122
168
  cache.stage = null;
123
169
  };
124
170
 
171
+ /** Waits for all the GPU work submitted so far, on the shared context. */
172
+ const waitForGpu = (cache: Cache): void => {
173
+ "worklet";
174
+ if (cache.stage) cache.stage.flush(true);
175
+ else {
176
+ let texture = cache.uploadTexture ?? cache.freeTextures[0]?.texture;
177
+ if (!texture) {
178
+ for (const target of cache.targets.values()) {
179
+ texture = target.textures[0]?.texture;
180
+ if (texture) break;
181
+ }
182
+ }
183
+ // A released producer's textures may all be gone (too big to keep free)
184
+ // while its imports are still in flight: any Graphite surface's checked
185
+ // flush waits on the shared context just the same.
186
+ const view = texture
187
+ ? Skia.Surface.MakeFromGPUTexture(texture)
188
+ : Skia.Surface.MakeOffscreen(1, 1);
189
+ if (!view) throw new Error("Video import drain surface is unavailable");
190
+ try {
191
+ view.flush(true);
192
+ } finally {
193
+ view.dispose();
194
+ }
195
+ }
196
+ cache.checkedWaits++;
197
+ };
198
+
199
+ /** Hands the decoder buffers of the first `count` imports in flight back,
200
+ * once the GPU is done with them: because the GPU reported the last of these
201
+ * imports done (in order: the work before it is done too), or after a wait
202
+ * for it, made here when `count` is all of them and nobody `waited`. */
203
+ const settleInFlightImports = (
204
+ cache: Cache,
205
+ count: number,
206
+ waited = false,
207
+ ): void => {
208
+ "worklet";
209
+ if (!count) return;
210
+ if (count === cache.inFlight.length && !waited) waitForGpu(cache);
211
+ for (let i = 0; i < count; i++) {
212
+ const entry = cache.inFlight.shift()!;
213
+ try {
214
+ entry.nativeSource?.release();
215
+ entry.nativeSource = null;
216
+ entry.frame.dispose?.();
217
+ } catch (error) {
218
+ // Like a failed release at import: only the final drain retries it.
219
+ cache.pendingSources.push({
220
+ nativeSource: entry.nativeSource,
221
+ owned: null,
222
+ reservation: 0,
223
+ frame: entry.frame,
224
+ needsDrain: false,
225
+ sourceReleased: false,
226
+ });
227
+ throw error;
228
+ }
229
+ }
230
+ };
231
+
232
+ // Free textures kept for the next producer, when they are small enough.
233
+ const maxFreeImportTextures = 2;
234
+
235
+ /** Work already submitted still completes on a destroyed texture; a caller
236
+ * that cannot tell whether everything reading it was submitted waits first. */
237
+ const destroyImportTexture = (cache: Cache, entry: ImportTexture) => {
238
+ "worklet";
239
+ if (cache.uploadTexture === entry.texture) {
240
+ clearSurfaceView(cache);
241
+ cache.uploadTexture = null;
242
+ }
243
+ entry.texture.destroy();
244
+ releaseVideoMemory(entry.reservation);
245
+ entry.reservation = 0;
246
+ };
247
+
248
+ // A texture leaves its list only once destroyed: a failed teardown is
249
+ // retried by the next one, its memory still accounted.
250
+ const releaseFreeImportTextures = (cache: Cache) => {
251
+ "worklet";
252
+ while (cache.freeTextures.length) {
253
+ destroyImportTexture(cache, cache.freeTextures[cache.freeTextures.length - 1]!);
254
+ cache.freeTextures.pop();
255
+ }
256
+ };
257
+
258
+ /** Gives up the textures of a producer whose image is disposed. Small ones
259
+ * are kept for the next producer of that size (a montage goes from clip to
260
+ * clip), within the budget's sixteenth each, as for imports in flight. */
261
+ const releaseImportTarget = (cache: Cache, producer: number, keep = true) => {
262
+ "worklet";
263
+ const target = cache.targets.get(producer);
264
+ if (!target) return;
265
+ const budget = videoMemoryBudget();
266
+ while (target.textures.length) {
267
+ const entry = target.textures[target.textures.length - 1]!;
268
+ if (
269
+ keep &&
270
+ cache.freeTextures.length < maxFreeImportTextures &&
271
+ !!budget &&
272
+ frameByteSize(entry.width, entry.height) <= budget.maxBytes / 16
273
+ )
274
+ cache.freeTextures.push(entry);
275
+ else destroyImportTexture(cache, entry);
276
+ target.textures.pop();
277
+ }
278
+ cache.targets.delete(producer);
279
+ };
280
+
281
+ /** The texture the next frame of `producer` goes into: one that its current
282
+ * image does not wrap, when the producer has or can afford a second one. */
283
+ const acquireImportTexture = (
284
+ cache: Cache,
285
+ device: GPUDevice,
286
+ producer: number,
287
+ width: number,
288
+ height: number,
289
+ format: GPUTextureFormat,
290
+ usage: number,
291
+ label: string,
292
+ ): GPUTexture => {
293
+ "worklet";
294
+ let target = cache.targets.get(producer);
295
+ if (
296
+ target &&
297
+ target.textures.some(
298
+ (entry) =>
299
+ entry.width !== width ||
300
+ entry.height !== height ||
301
+ entry.format !== format,
302
+ )
303
+ ) {
304
+ // The producer changed size: its image wraps a texture given up here.
305
+ clearEntry(cache, producer);
306
+ releaseImportTarget(cache, producer, false);
307
+ target = undefined;
308
+ }
309
+ if (!target) {
310
+ target = { textures: [] };
311
+ cache.targets.set(producer, target);
312
+ }
313
+ const shown = cache.images.get(producer);
314
+ const live = shown?.image
315
+ ? target.textures.findIndex((entry) => entry.texture === shown.texture)
316
+ : -1;
317
+ let index = target.textures.findIndex((_, i) => i !== live);
318
+ if (index < 0) {
319
+ const free = cache.freeTextures.findIndex(
320
+ (entry) =>
321
+ entry.width === width &&
322
+ entry.height === height &&
323
+ entry.format === format,
324
+ );
325
+ const bytes = frameByteSize(width, height);
326
+ const budget = videoMemoryBudget();
327
+ if (free >= 0) {
328
+ target.textures.push(cache.freeTextures.splice(free, 1)[0]!);
329
+ index = target.textures.length - 1;
330
+ } else if (
331
+ !target.textures.length ||
332
+ !budget ||
333
+ budget.maxBytes - budget.currentBytes >= 3 * bytes
334
+ ) {
335
+ // Free textures of another size only hold memory back from here on.
336
+ releaseFreeImportTextures(cache);
337
+ const reservation = reserveVideoMemory(bytes, label);
338
+ let texture: GPUTexture;
339
+ try {
340
+ texture = device.createTexture({
341
+ size: { width, height },
342
+ format,
343
+ usage,
344
+ label,
345
+ });
346
+ } catch (error) {
347
+ releaseVideoMemory(reservation);
348
+ throw error;
349
+ }
350
+ target.textures.push({ texture, width, height, format, reservation });
351
+ index = target.textures.length - 1;
352
+ } else {
353
+ // No room for a second texture: write over the current frame.
354
+ index = live;
355
+ }
356
+ }
357
+ return target.textures[index]!.texture;
358
+ };
359
+
125
360
  const clearCpuPixels = (cache: Cache) => {
126
361
  "worklet";
127
362
  cache.cpuPixels = null;
@@ -132,34 +367,44 @@ const clearCpuPixels = (cache: Cache) => {
132
367
 
133
368
  const clearStagingSurface = (cache: Cache) => {
134
369
  "worklet";
370
+ settleInFlightImports(cache, cache.inFlight.length);
135
371
  cache.stage?.flush(true);
136
372
  clearSurfaceView(cache);
137
- cache.uploadTexture?.destroy();
373
+ for (const producer of [...cache.targets.keys()])
374
+ releaseImportTarget(cache, producer, false);
375
+ releaseFreeImportTextures(cache);
138
376
  cache.uploadTexture = null;
139
- releaseVideoMemory(cache.uploadReservation);
140
- cache.uploadReservation = 0;
141
377
  clearCpuPixels(cache);
142
- cache.width = cache.height = 0;
143
- cache.format = null;
144
378
  };
145
379
 
146
380
  /** Record explicit recovery work after a possibly submitted external copy.
147
- * Clearing scratch cannot modify an already owned snapshot. The checked fence
148
- * must succeed before either native lease or the texture can be released. */
381
+ * It clears the texture of the import that failed, whose image is dropped;
382
+ * with two textures, the producer's previous image is in the other one. The
383
+ * checked fence must succeed before either native lease or the texture can be released. */
149
384
  const drainNativeCopies = (cache: Cache): void => {
150
385
  "worklet";
151
386
  if (!cache.stage || !cache.canvas)
152
387
  throw new Error("Video import drain surface is unavailable");
153
388
  cache.canvas.drawColor(Skia.Color("#00000000"), BlendMode.Clear);
154
389
  cache.stage.flush(true);
390
+ cache.checkedWaits++;
391
+ };
392
+
393
+ /** The checked GPU waits this runtime's imports have made so far, and the
394
+ * completions the GPU reported for them. Each one covers all the GPU work
395
+ * submitted before it, on the shared context. */
396
+ export const videoFrameImportWaits = (): number => {
397
+ "worklet";
398
+ return getCache().checkedWaits;
155
399
  };
156
400
 
157
401
  /** Clear a runtime's imported images after ending all its producers.
158
402
  * Do not clear a paused producer's current frame: its native source has
159
- * already been released, and the owned snapshot is its remaining pixels. */
403
+ * already been released, and its image's texture is its remaining pixels. */
160
404
  export const clearVideoFrameImages = (releaseStagingSurface = false): void => {
161
405
  "worklet";
162
406
  const cache = getCache();
407
+ settleInFlightImports(cache, cache.inFlight.length);
163
408
  if (cache.pendingSources.length) {
164
409
  // A failed drain retains leases; only a successful fence can release them.
165
410
  if (cache.pendingSources.some((source) => source.needsDrain)) {
@@ -203,18 +448,30 @@ export const releaseVideoFrameImage = (frame: VideoFrame): void => {
203
448
  const producer = frame.producerId ?? frame.id ?? -1;
204
449
  const cache = getCache();
205
450
  const entry = cache.images.get(producer);
206
- if (entry && entry.id === (frame.id ?? -1)) clearEntry(cache, producer);
207
- // Each import ends with a checked flush; after a failed one, only the
208
- // drain in clearVideoFrameImages may release the decoder buffers.
209
- if (!cache.pendingSources.length)
451
+ if (entry && entry.id === (frame.id ?? -1)) {
452
+ clearEntry(cache, producer);
453
+ releaseImportTarget(cache, producer);
454
+ }
455
+ // Every import is done once the GPU is; after a failed one, only the drain
456
+ // in clearVideoFrameImages may release the decoder buffers.
457
+ if (!cache.pendingSources.length && cache.yuvImports.has(producer)) {
458
+ settleInFlightImports(cache, cache.inFlight.length);
210
459
  releaseYuvImports(cache.yuvImports, producer);
460
+ }
211
461
  if (cache.images.size === 0) clearVideoFrameImages(true);
212
462
  };
213
463
 
214
- /** Returns immutable, owned pixels. WebGPU uploads Android RGBA pixels or blits
215
- * native buffers into one reusable texture, then snapshots a fresh Skia view.
216
- * Graphite-backed snapshots bypass the raster ImageProvider's image-count LRU.
217
- * Wait for completion before releasing sources or rewriting the scratch texture. */
464
+ /** Returns owned pixels. WebGPU uploads Android RGBA pixels or blits native
465
+ * buffers into a texture of the frame's producer, which the image wraps: no
466
+ * copy into a snapshot, and no entry in the raster ImageProvider's LRU.
467
+ * Sources go back once the GPU is done with them.
468
+ *
469
+ * The image is the producer's until its next frame is imported, which
470
+ * disposes it. A producer has two textures and writes its frames into them in
471
+ * turn, so the texture of an image is only written again by the import after
472
+ * the one that disposed it; with a memory budget too short for a second
473
+ * texture, the next frame is written over it, in queue order after the draws
474
+ * already submitted. */
218
475
  export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
219
476
  "worklet";
220
477
  const cache = getCache();
@@ -223,7 +480,7 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
223
480
  const existing = cache.images.get(producer);
224
481
  if (existing?.id === frame.id && existing.image) {
225
482
  // Android can issue a fresh lease for the same paused picture. The
226
- // snapshot already owns its pixels, so close that unused native lease.
483
+ // image already owns its pixels, so close that unused native lease.
227
484
  // Weak tracking avoids retaining raw buffers or closing one wrapper twice.
228
485
  if (!existing.releasedSources.has(frame)) {
229
486
  frame.dispose?.();
@@ -238,6 +495,9 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
238
495
  let ownsSurfaceView = false;
239
496
  let nativeSource: NativeVideoFrame | null = null;
240
497
  let sourceDrained = false;
498
+ let completion: Promise<undefined> | null = null;
499
+ let sourceBytes = 0;
500
+ let importTexture: GPUTexture | null = null;
241
501
  let releaseSource = true;
242
502
  let importFailed = false;
243
503
  try {
@@ -311,7 +571,7 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
311
571
  );
312
572
  const data = Skia.Data.fromBytes(pixels);
313
573
  try {
314
- image = Skia.Image.MakeRasterImage(
574
+ image = Skia.Image.MakeRaster(
315
575
  {
316
576
  width: frame.width,
317
577
  height: frame.height,
@@ -344,49 +604,25 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
344
604
  }
345
605
  const format =
346
606
  uploadRgba || uploadAndroidRgba ? "rgba8unorm" : "bgra8unorm";
347
- reservation = reserveVideoMemory(bytes, "owned video image");
348
- if (
349
- !cache.uploadTexture ||
350
- cache.width !== frame.width ||
351
- cache.height !== frame.height ||
352
- cache.format !== format
353
- ) {
354
- clearStagingSurface(cache);
355
- cache.uploadReservation = reserveVideoMemory(
356
- bytes,
357
- uploadRgba
358
- ? "RGBA frame WebGPU upload texture"
359
- : "native frame WebGPU upload texture",
360
- );
361
- try {
362
- // One scratch texture per runtime. COPY_SRC lets Graphite snapshot
363
- // it with a GPU blit rather than draw into another staging target.
364
- cache.uploadTexture = device.createTexture({
365
- size: { width: frame.width, height: frame.height },
366
- format,
367
- // Android AHB and CPU frames share the RGBA scratch. Include
368
- // COPY_DST even for a first AHB frame, so a later CPU fallback
369
- // cannot reuse a texture that forbids writeTexture.
370
- usage:
371
- uploadRgba || uploadAndroidRgba
372
- ? rgbaVideoTextureUsage
373
- : nativeVideoTextureUsage,
374
- label: uploadRgba
375
- ? "video RGBA frame upload"
376
- : "video native frame upload",
377
- });
378
- cache.width = frame.width;
379
- cache.height = frame.height;
380
- cache.format = format;
381
- } catch (error) {
382
- try {
383
- clearStagingSurface(cache);
384
- } catch {
385
- /* unfinished cleanup/accounting stays reachable */
386
- }
387
- throw error;
388
- }
389
- }
607
+ // The frame is converted straight into a texture of its producer, which
608
+ // the image wraps: no copy into a snapshot. Android AHB and CPU frames
609
+ // share RGBA textures, COPY_DST included even for a first AHB frame, so
610
+ // that a later CPU fallback can write into them.
611
+ clearSurfaceView(cache);
612
+ cache.uploadTexture = null;
613
+ cache.uploadTexture = acquireImportTexture(
614
+ cache,
615
+ device,
616
+ key,
617
+ frame.width,
618
+ frame.height,
619
+ format,
620
+ uploadRgba || uploadAndroidRgba
621
+ ? rgbaVideoTextureUsage
622
+ : nativeVideoTextureUsage,
623
+ uploadRgba ? "video RGBA frame" : "video native frame",
624
+ );
625
+ importTexture = cache.uploadTexture;
390
626
  if (copyRgbaPixels && texture.kind === "rgba") {
391
627
  if (
392
628
  !cache.cpuPixels ||
@@ -423,9 +659,8 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
423
659
  clearCpuPixels(cache);
424
660
  }
425
661
  clearSurfaceView(cache);
426
- // WebGPU writes do not invalidate SkSurface's cached snapshot. A fresh
427
- // surface wrapper for every frame prevents stale pixels without another
428
- // GPU allocation or a write through Skia that would overwrite the input.
662
+ // The surface only carries the checked submissions and waits of this
663
+ // import, and its recovery drain; the image wraps the texture itself.
429
664
  cache.stage = Skia.Surface.MakeFromGPUTexture(cache.uploadTexture!);
430
665
  if (!cache.stage) throw new Error("Cannot wrap video upload texture");
431
666
  ownsSurfaceView = true;
@@ -449,14 +684,15 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
449
684
  const region = texture.region;
450
685
  const rotation = texture.rotation ?? 0;
451
686
  const nv12 = !region && texture.format === "nv12";
452
- let origin = { x: 0, y: 0 };
687
+ let upright: ReturnType<typeof yuvFrameRegion> | null = null;
453
688
  if (region) {
454
689
  // An Android codec buffer: padded, YUV, in encoded orientation. The
455
- // blit converts, crops and turns it into the display-sized scratch,
456
- // as the GL path's SurfaceTexture transform did. RN WebGPU labels
457
- // driver-defined YUV formats "bgra8", yet Dawn imports them as YUV:
458
- // the native side checked the format, so the label is not read.
459
- origin = yuvFrameOrigin(region, rotation, nativeSource, frame);
690
+ // blit converts, crops, turns and scales it into the frame-sized
691
+ // scratch, as the GL path's SurfaceTexture transform did. RN WebGPU
692
+ // labels driver-defined YUV formats "bgra8", yet Dawn imports them
693
+ // as YUV: the native side checked the format, so the label is not
694
+ // read.
695
+ upright = yuvFrameRegion(region, rotation, nativeSource);
460
696
  } else if (nv12) {
461
697
  if (
462
698
  nativeSource.width !== frame.width ||
@@ -487,9 +723,7 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
487
723
  key,
488
724
  nativeSource,
489
725
  rotation,
490
- origin,
491
- frame.width,
492
- frame.height,
726
+ upright!,
493
727
  cache.uploadTexture!,
494
728
  );
495
729
  } else if (nv12) {
@@ -511,25 +745,84 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
511
745
  }
512
746
  // Snapshot records an independent texture copy and submits it after
513
747
  // the external blit on the shared queue. The wrapper never draws.
514
- image = cache.stage.makeImageSnapshot();
515
- cache.stage.flush(true);
516
- sourceDrained = true;
748
+ image = Skia.Image.MakeFromGPUTexture(cache.uploadTexture!);
749
+ // Submitted without waiting for the GPU, which held the UI thread for
750
+ // the work of every Skia view on the shared context: the decoder buffer
751
+ // is handed back once the GPU reports the copy done. A wait still
752
+ // bounds the buffers out, and stands in where nothing can report.
753
+ // NV12 and Android's YUV buffers take a byte and a half per pixel.
754
+ sourceBytes =
755
+ texture.kind === "native-buffer" &&
756
+ (texture.region || texture.format === "nv12")
757
+ ? Math.ceil(bytes * 0.375)
758
+ : bytes;
759
+ const wait =
760
+ typeof device.queue.onSubmittedWorkDone !== "function" ||
761
+ cache.inFlight.length >= maxInFlightImports ||
762
+ !affordsInFlight(sourceBytes);
763
+ cache.stage.flush(wait);
764
+ if (!wait) {
765
+ try {
766
+ completion = device.queue.onSubmittedWorkDone();
767
+ } catch {
768
+ // A runtime WebGPU cannot settle promises on.
769
+ completion = null;
770
+ }
771
+ }
772
+ if (wait || !completion) {
773
+ if (!wait) cache.stage.flush(true);
774
+ cache.checkedWaits++;
775
+ // Every import before this one is done too.
776
+ settleInFlightImports(cache, cache.inFlight.length, true);
777
+ completion = null;
778
+ sourceDrained = true;
779
+ }
517
780
  }
518
781
  if (!image) throw new Error("Cannot create a video frame image");
519
- // Complete the native handoff before replacing published pixels. A failed
520
- // source close must not evict a paused image or leave an unpublished cache
521
- // entry behind when the caller receives an import error.
522
- if (ownsSurfaceView) {
523
- clearSurfaceView(cache);
782
+ if (completion) {
783
+ // The GPU copy may still read the decoder buffer: it goes back once the
784
+ // GPU reports the copy done, or after a wait (settleInFlightImports).
785
+ // The view stays for that wait.
786
+ const entry: InFlightImport = { frame, nativeSource, bytes: sourceBytes };
787
+ cache.inFlight.push(entry);
788
+ nativeSource = null;
789
+ releaseSource = false;
524
790
  ownsSurfaceView = false;
791
+ completion.then(
792
+ () => {
793
+ const settled = cache.inFlight.indexOf(entry) + 1;
794
+ // Zero: a wait handed it back already.
795
+ if (!settled) return;
796
+ cache.checkedWaits++;
797
+ try {
798
+ settleInFlightImports(cache, settled, true);
799
+ } catch (error) {
800
+ console.warn("Video frame release failed", error);
801
+ }
802
+ },
803
+ // Handed back by the next wait instead.
804
+ () => undefined,
805
+ );
806
+ } else {
807
+ // Complete the native handoff before replacing published pixels. A
808
+ // failed source close must not evict a paused image or leave an
809
+ // unpublished cache entry behind when the caller receives an import
810
+ // error.
811
+ if (ownsSurfaceView) {
812
+ clearSurfaceView(cache);
813
+ ownsSurfaceView = false;
814
+ }
815
+ nativeSource?.release();
816
+ nativeSource = null;
817
+ frame.dispose?.();
818
+ releaseSource = false;
525
819
  }
526
- nativeSource?.release();
527
- nativeSource = null;
528
- frame.dispose?.();
529
- releaseSource = false;
530
820
  clearEntry(cache, key);
531
821
  cache.images.set(key, {
532
- id: frame.id ?? -1, image, reservation,
822
+ id: frame.id ?? -1,
823
+ image,
824
+ reservation,
825
+ texture: importTexture,
533
826
  releasedSources: new WeakSet([frame]),
534
827
  });
535
828
  return image;
package/src/gpu.ts CHANGED
@@ -166,7 +166,7 @@ export const createVideoGpuScope = (options?: {
166
166
  resource.surface = Skia.Surface.MakeFromGPUTexture(resource.texture);
167
167
  resource.canvas = getVideoCanvas(resource.surface);
168
168
  } else if (skia === "image") {
169
- resource.image = Skia.Image.MakeImageFromGPUTexture(resource.texture);
169
+ resource.image = Skia.Image.MakeFromGPUTexture(resource.texture);
170
170
  }
171
171
  resources.push(resource);
172
172
  return {
@@ -178,7 +178,6 @@ export const createVideoGpuScope = (options?: {
178
178
  } catch (error) {
179
179
  // No texture has been exposed or submitted to any consumer yet.
180
180
  try {
181
- resource.canvas?.dispose();
182
181
  resource.canvas = null;
183
182
  resource.surface?.dispose();
184
183
  resource.surface = null;
@@ -265,7 +264,6 @@ export const createVideoGpuScope = (options?: {
265
264
  try {
266
265
  resource.image?.dispose();
267
266
  resource.image = null;
268
- resource.canvas?.dispose();
269
267
  resource.canvas = null;
270
268
  resource.surface?.dispose();
271
269
  resource.surface = null;