react-native-skia-webgpu-video 0.1.0-beta.3 → 0.1.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/README.md +39 -32
  2. package/android/BUILD_VALIDATION.md +1 -1
  3. package/android/CMakeLists.txt +3 -2
  4. package/android/cpp/ExportHintSession.cpp +114 -0
  5. package/android/cpp/ExportHintSession.h +35 -0
  6. package/android/cpp/VideoEncoderHostObject.cpp +76 -101
  7. package/android/cpp/VideoEncoderHostObject.h +14 -9
  8. package/android/cpp/VideoFrame.cpp +3 -5
  9. package/android/cpp/cpp-adapter.cpp +7 -4
  10. package/android/src/main/java/com/azzapp/rnskv/GLFrameExtractor.java +3 -9
  11. package/android/src/main/java/com/azzapp/rnskv/VideoCompositionDecoder.java +8 -11
  12. package/android/src/main/java/com/azzapp/rnskv/VideoEncoder.java +68 -28
  13. package/docs/FRAME_PIXEL_FORMAT.md +1 -1
  14. package/docs/MIGRATION.md +1 -1
  15. package/docs/UPSTREAM_NOTES.md +15 -2
  16. package/lib/module/canvas.js +3 -5
  17. package/lib/module/canvas.js.map +1 -1
  18. package/lib/module/exportFrames.js +6 -8
  19. package/lib/module/exportFrames.js.map +1 -1
  20. package/lib/module/exportTarget.js +183 -40
  21. package/lib/module/exportTarget.js.map +1 -1
  22. package/lib/module/exportVideoComposition.js +53 -12
  23. package/lib/module/exportVideoComposition.js.map +1 -1
  24. package/lib/module/frameInterop.js +9 -12
  25. package/lib/module/frameInterop.js.map +1 -1
  26. package/lib/module/gpu.js +0 -2
  27. package/lib/module/gpu.js.map +1 -1
  28. package/lib/module/gpuDevice.js +12 -0
  29. package/lib/module/gpuDevice.js.map +1 -1
  30. package/lib/module/previewLifecycle.js +0 -1
  31. package/lib/module/previewLifecycle.js.map +1 -1
  32. package/lib/module/yuvBlit.js +28 -21
  33. package/lib/module/yuvBlit.js.map +1 -1
  34. package/lib/typescript/src/canvas.d.ts.map +1 -1
  35. package/lib/typescript/src/exportFrames.d.ts +3 -2
  36. package/lib/typescript/src/exportFrames.d.ts.map +1 -1
  37. package/lib/typescript/src/exportTarget.d.ts +53 -14
  38. package/lib/typescript/src/exportTarget.d.ts.map +1 -1
  39. package/lib/typescript/src/exportVideoComposition.d.ts +2 -1
  40. package/lib/typescript/src/exportVideoComposition.d.ts.map +1 -1
  41. package/lib/typescript/src/frameInterop.d.ts.map +1 -1
  42. package/lib/typescript/src/gpu.d.ts.map +1 -1
  43. package/lib/typescript/src/gpuDevice.d.ts +5 -1
  44. package/lib/typescript/src/gpuDevice.d.ts.map +1 -1
  45. package/lib/typescript/src/previewLifecycle.d.ts.map +1 -1
  46. package/lib/typescript/src/types.d.ts +10 -5
  47. package/lib/typescript/src/types.d.ts.map +1 -1
  48. package/lib/typescript/src/yuvBlit.d.ts +16 -14
  49. package/lib/typescript/src/yuvBlit.d.ts.map +1 -1
  50. package/package.json +6 -6
  51. package/patches/react-native-skia+3.7.4.patch +146 -0
  52. package/scripts/skia-canvas-patch.json +22 -258
  53. package/src/canvas.ts +3 -7
  54. package/src/exportFrames.ts +8 -9
  55. package/src/exportTarget.ts +217 -56
  56. package/src/exportVideoComposition.ts +87 -26
  57. package/src/frameInterop.ts +9 -11
  58. package/src/gpu.ts +0 -2
  59. package/src/gpuDevice.ts +26 -1
  60. package/src/previewLifecycle.ts +0 -1
  61. package/src/types.ts +10 -5
  62. package/src/yuvBlit.ts +43 -28
  63. package/android/cpp/EncoderTargets.cpp +0 -226
  64. package/android/cpp/EncoderTargets.h +0 -70
  65. package/patches/react-native-skia+3.7.1.patch +0 -502
@@ -22,7 +22,7 @@ import {
22
22
  blitNv12Frame,
23
23
  blitYuvFrame,
24
24
  releaseYuvImports,
25
- yuvFrameOrigin,
25
+ yuvFrameRegion,
26
26
  } from "./yuvBlit";
27
27
  import type { YuvImport } from "./yuvBlit";
28
28
 
@@ -174,9 +174,9 @@ const importYuvExportFrame = (
174
174
  entry.nativeFrame.height !== frame.height)
175
175
  )
176
176
  throw new Error("Native video buffer dimensions mismatch");
177
- const origin = nv12
178
- ? { x: 0, y: 0 }
179
- : yuvFrameOrigin(source.region!, rotation, entry.nativeFrame, frame);
177
+ const region = nv12
178
+ ? null
179
+ : yuvFrameRegion(source.region!, rotation, entry.nativeFrame);
180
180
  const device = getVideoGpuDevice();
181
181
  target = imports.textures.get(producer);
182
182
  if (
@@ -220,9 +220,7 @@ const importYuvExportFrame = (
220
220
  producer,
221
221
  entry.nativeFrame,
222
222
  rotation,
223
- origin,
224
- frame.width,
225
- frame.height,
223
+ region!,
226
224
  target.texture,
227
225
  );
228
226
  entry.image = Skia.Image.MakeImageFromGPUTexture(target.texture);
@@ -329,8 +327,9 @@ export const releaseExportFrames = (imports: ExportFrameImports): void => {
329
327
 
330
328
  /** The export counterpart of ownVideoFrames on iOS: decoded frames are drawn
331
329
  * in place, and a producer's previous frame is released once its next one is
332
- * imported, since every exported frame ends with a checked GPU wait. Other
333
- * frames, or every frame after a first failed import, use `fallback`. */
330
+ * imported, since the export waits for the GPU to complete a frame before it
331
+ * imports the next one. Other frames, or every frame after a first failed
332
+ * import, use `fallback`. */
334
333
  export const ownExportFrames = (
335
334
  imports: ExportFrameImports,
336
335
  decoded: Record<string, VideoFrame>,
@@ -1,21 +1,33 @@
1
1
  /// <reference types="@webgpu/types" />
2
- import { Platform } from "react-native";
3
2
  import { BlendMode, Skia } from "react-native-skia";
4
3
  import type { SkCanvas, SkSurface } from "react-native-skia";
5
4
  import type {
6
5
  GPUSharedTextureMemory,
7
6
  GPUSharedTextureMemoryDescriptor,
8
7
  NativeVideoFrame,
8
+ RNCanvasContext,
9
9
  } from "react-native-webgpu";
10
10
  import { getVideoCanvas } from "./canvas";
11
- import { createNativeVideoGpuFrame, getVideoGpuDevice } from "./gpuDevice";
11
+ import {
12
+ createNativeVideoGpuFrame,
13
+ createNativeWindowGpuContext,
14
+ encoderWindowTextureUsage,
15
+ getVideoGpuDevice,
16
+ nativeVideoTextureUsage,
17
+ } from "./gpuDevice";
18
+ import {
19
+ frameByteSize,
20
+ releaseVideoMemory,
21
+ reserveVideoMemory,
22
+ } from "./memory";
12
23
  import type { VideoEncoder } from "./types";
13
24
 
14
- /** An encoder pool buffer that Skia draws into through WebGPU. */
25
+ /** A texture Skia draws an exported frame into through WebGPU: a lent
26
+ * encoder pool buffer on iOS, one of the encoder window's own textures on
27
+ * Android. */
15
28
  export type EncoderTarget = {
16
- pointer: bigint;
17
- /** Android: sync-fds ending the GPU access, handed over to the encoder. */
18
- fences: bigint[];
29
+ /** iOS: the lent CVPixelBuffer. */
30
+ pointer: bigint | null;
19
31
  frame: NativeVideoFrame | null;
20
32
  memory: GPUSharedTextureMemory | null;
21
33
  texture: GPUTexture | null;
@@ -24,9 +36,25 @@ export type EncoderTarget = {
24
36
  canvas: SkCanvas | null;
25
37
  };
26
38
 
27
- // Captured on the RN runtime. iOS lends CVPixelBuffers, imported through
28
- // their IOSurface; Android lends the AHardwareBuffer itself.
29
- const lendsPixelBuffers = Platform.OS === "ios";
39
+ /** Android: a canvas context presenting to the codec's input Surface. Skia
40
+ * draws each frame into one of two textures of our own, which a GPU copy then
41
+ * puts into the window's texture. The window's textures are the codec's
42
+ * input buffers: the copy waits on the GPU for the codec to hand one back,
43
+ * while the export goes on with the next frame, rather than the frame's
44
+ * checked wait. */
45
+ export type EncoderWindow = {
46
+ context: RNCanvasContext;
47
+ device: GPUDevice;
48
+ configured: boolean;
49
+ /** Drawn into in turn, each copied into the window after its frame. */
50
+ slots: EncoderTarget[];
51
+ next: number;
52
+ reservations: number[];
53
+ /** The window texture the probe acquired, the first frame's copy target. */
54
+ pending: GPUTexture | null;
55
+ };
56
+
57
+ const windowSlots = 2;
30
58
 
31
59
  type SharedMemoryDevice = GPUDevice & {
32
60
  importSharedTextureMemory?: (
@@ -43,38 +71,19 @@ export const supportsEncoderTargets = (encoder: VideoEncoder): boolean => {
43
71
  };
44
72
 
45
73
  // Declared before its callers: worklets capture their closure on creation.
46
- /** Ends GPU access; the lent buffer stays with the caller, to encode or
47
- * release. Call only after a checked flush(true) of its surface. With
48
- * `encoding`, Android keeps the access's sync-fds in `target.fences` for the
49
- * encoder, which owns and closes them: ending an AHB access submits a
50
- * release that flush(true) has not waited for. */
51
- export const closeEncoderTarget = (
52
- target: EncoderTarget,
53
- encoding = false,
54
- ): void => {
74
+ /** iOS: ends GPU access; the lent buffer stays with the caller, to encode or
75
+ * release. Call only after a checked flush(true) of its surface. */
76
+ export const closeEncoderTarget = (target: EncoderTarget): void => {
55
77
  "worklet";
56
- target.canvas?.dispose();
78
+ // Disposing the surface detaches its canvas.
57
79
  target.canvas = null;
58
80
  target.surface?.dispose();
59
81
  target.surface = null;
60
82
  if (target.accessing && target.memory && target.texture) {
61
83
  const ended = target.memory.endAccess(target.texture);
62
84
  target.accessing = false;
63
- for (const state of ended?.fences ?? []) {
64
- try {
65
- if (encoding && !lendsPixelBuffers) {
66
- const exported = state.fence.export();
67
- // Unreachable when negotiated: AHB interop requires the device's
68
- // only shared-fence feature to be sync-fd (see gpuDevice.ts).
69
- if (exported.type !== "sync-fd")
70
- throw new Error("GPU export needs sync-fd fences");
71
- target.fences.push(exported.handle);
72
- }
73
- } finally {
74
- // iOS: already waited for. The wrapper is released either way.
75
- state.fence.destroy();
76
- }
77
- }
85
+ // Already waited for. The wrappers are released either way.
86
+ for (const state of ended?.fences ?? []) state.fence.destroy();
78
87
  }
79
88
  target.accessing = false;
80
89
  target.texture?.destroy();
@@ -88,7 +97,7 @@ export const closeEncoderTarget = (
88
97
  target.frame = null;
89
98
  };
90
99
 
91
- /** Lends the next pool buffer and wraps it as a Skia surface. */
100
+ /** iOS: lends the next pool buffer and wraps it as a Skia surface. */
92
101
  export const acquireEncoderTarget = (
93
102
  encoder: VideoEncoder,
94
103
  width: number,
@@ -97,7 +106,6 @@ export const acquireEncoderTarget = (
97
106
  "worklet";
98
107
  const target: EncoderTarget = {
99
108
  pointer: encoder.acquireFrameBuffer!(),
100
- fences: [],
101
109
  frame: null,
102
110
  memory: null,
103
111
  texture: null,
@@ -106,33 +114,23 @@ export const acquireEncoderTarget = (
106
114
  canvas: null,
107
115
  };
108
116
  try {
109
- if (lendsPixelBuffers) {
110
- target.frame = createNativeVideoGpuFrame(target.pointer);
111
- if (
112
- target.frame.width !== width ||
113
- target.frame.height !== height ||
114
- target.frame.pixelFormat !== "bgra8"
115
- ) {
116
- throw new Error("Encoder buffer dimensions or BGRA layout mismatch");
117
- }
117
+ target.frame = createNativeVideoGpuFrame(target.pointer!);
118
+ if (
119
+ target.frame.width !== width ||
120
+ target.frame.height !== height ||
121
+ target.frame.pixelFormat !== "bgra8"
122
+ ) {
123
+ throw new Error("Encoder buffer dimensions or BGRA layout mismatch");
118
124
  }
119
125
  const device = getVideoGpuDevice() as SharedMemoryDevice;
120
126
  if (typeof device.importSharedTextureMemory !== "function") {
121
127
  throw new Error("WebGPU shared texture memory is unavailable");
122
128
  }
123
129
  target.memory = device.importSharedTextureMemory({
124
- handle: target.frame ? target.frame.handle : target.pointer,
130
+ handle: target.frame.handle,
125
131
  label: "video export frame",
126
132
  });
127
133
  target.texture = target.memory.createTexture();
128
- if (
129
- !lendsPixelBuffers &&
130
- (target.texture.width !== width ||
131
- target.texture.height !== height ||
132
- target.texture.format !== "rgba8unorm")
133
- ) {
134
- throw new Error("Encoder buffer dimensions or RGBA layout mismatch");
135
- }
136
134
  // Uninitialized: every frame starts with a full clear.
137
135
  target.memory.beginAccess(target.texture, false);
138
136
  target.accessing = true;
@@ -151,8 +149,9 @@ export const acquireEncoderTarget = (
151
149
  }
152
150
  };
153
151
 
154
- /** Clears one lent buffer on the GPU, then returns it unencoded. Returns the
155
- * failure, or null when this device can export through encoder targets. */
152
+ /** iOS: clears one lent buffer on the GPU, then returns it unencoded.
153
+ * Returns the failure, or null when this device can export through encoder
154
+ * targets. */
156
155
  export const probeEncoderTarget = (
157
156
  encoder: VideoEncoder,
158
157
  width: number,
@@ -177,3 +176,165 @@ export const probeEncoderTarget = (
177
176
  encoder.releaseFrameBuffer!();
178
177
  return failure;
179
178
  };
179
+
180
+ /** Android: the window's next texture, checked against the export size. */
181
+ const windowTexture = (
182
+ window: EncoderWindow,
183
+ width: number,
184
+ height: number,
185
+ ): GPUTexture => {
186
+ "worklet";
187
+ const texture = window.pending ?? window.context.getCurrentTexture();
188
+ window.pending = null;
189
+ if (
190
+ texture.width !== width ||
191
+ texture.height !== height ||
192
+ texture.format !== "rgba8unorm"
193
+ ) {
194
+ throw new Error("Encoder window dimensions or RGBA layout mismatch");
195
+ }
196
+ return texture;
197
+ };
198
+
199
+ /** Android: the slot the next frame draws into. Its previous copy completed:
200
+ * the checked wait the export makes before each frame's import covers the
201
+ * queue before it. */
202
+ export const acquireWindowTarget = (window: EncoderWindow): EncoderTarget => {
203
+ "worklet";
204
+ return window.slots[window.next]!;
205
+ };
206
+
207
+ /** Android: copies the slot Skia drew into to the window's next texture and
208
+ * hands it to the encoder, with its time. Call once the slot's surface was
209
+ * flushed: the copy follows the drawing on the same queue. */
210
+ export const presentWindowTarget = (
211
+ encoder: VideoEncoder,
212
+ window: EncoderWindow,
213
+ target: EncoderTarget,
214
+ time: number,
215
+ ): void => {
216
+ "worklet";
217
+ const source = target.texture!;
218
+ const destination = windowTexture(window, source.width, source.height);
219
+ const commands = window.device.createCommandEncoder({
220
+ label: "video export window copy",
221
+ });
222
+ commands.copyTextureToTexture(
223
+ { texture: source },
224
+ { texture: destination },
225
+ { width: source.width, height: source.height },
226
+ );
227
+ window.device.queue.submit([commands.finish()]);
228
+ window.next = (window.next + 1) % window.slots.length;
229
+ // The encoder sets the frame's time on the window, then presents.
230
+ encoder.encodeFrame(
231
+ { kind: "window", present: () => window.context.present() },
232
+ time,
233
+ );
234
+ };
235
+
236
+ /** Android: the swapchain gives the codec its buffers back at once, rather
237
+ * than once the context is collected. Call before disposing the encoder. */
238
+ export const closeEncoderWindow = (window: EncoderWindow): void => {
239
+ "worklet";
240
+ window.pending = null;
241
+ if (!window.configured) return;
242
+ window.context.unconfigure();
243
+ window.configured = false;
244
+ };
245
+
246
+ /** Android: frees the window's own textures. Call once their last recording
247
+ * was submitted: a copy still in flight keeps its source alive. */
248
+ export const releaseEncoderWindow = (window: EncoderWindow): void => {
249
+ "worklet";
250
+ while (window.slots.length) {
251
+ const slot = window.slots[window.slots.length - 1]!;
252
+ // Disposing the surface detaches its canvas.
253
+ slot.canvas = null;
254
+ slot.surface?.dispose();
255
+ slot.surface = null;
256
+ slot.texture?.destroy();
257
+ slot.texture = null;
258
+ window.slots.pop();
259
+ }
260
+ while (window.reservations.length) {
261
+ releaseVideoMemory(window.reservations[window.reservations.length - 1]!);
262
+ window.reservations.pop();
263
+ }
264
+ };
265
+
266
+ /** Android: opens a canvas context on the encoder's input window with its
267
+ * slots, acquires the window's first texture and draws a clear into the
268
+ * first slot: nothing reaches the encoder. Returns the window, or the
269
+ * failure, after which the window is closed and the export can fall back to
270
+ * readback. */
271
+ export const probeEncoderWindow = (
272
+ encoder: VideoEncoder,
273
+ width: number,
274
+ height: number,
275
+ ): { window: EncoderWindow | null; failure: unknown } => {
276
+ "worklet";
277
+ let window: EncoderWindow | null = null;
278
+ let failure: unknown = null;
279
+ try {
280
+ if (typeof encoder.acquireInputWindow !== "function")
281
+ throw new Error("Encoder cannot present through its input window");
282
+ const device = getVideoGpuDevice();
283
+ window = {
284
+ context: createNativeWindowGpuContext(
285
+ encoder.acquireInputWindow(),
286
+ width,
287
+ height,
288
+ ),
289
+ device,
290
+ configured: false,
291
+ slots: [],
292
+ next: 0,
293
+ reservations: [],
294
+ pending: null,
295
+ };
296
+ // Presented in FIFO order, opaque: every frame reaches the codec.
297
+ window.context.configure({
298
+ device,
299
+ format: "rgba8unorm",
300
+ usage: encoderWindowTextureUsage,
301
+ });
302
+ window.configured = true;
303
+ for (let i = 0; i < windowSlots; i++) {
304
+ window.reservations.push(
305
+ reserveVideoMemory(frameByteSize(width, height), "export window slot"),
306
+ );
307
+ const slot: EncoderTarget = {
308
+ pointer: null,
309
+ frame: null,
310
+ memory: null,
311
+ texture: device.createTexture({
312
+ label: "video export window slot",
313
+ size: { width, height },
314
+ format: "rgba8unorm",
315
+ usage: nativeVideoTextureUsage,
316
+ }),
317
+ accessing: false,
318
+ surface: null,
319
+ canvas: null,
320
+ };
321
+ window.slots.push(slot);
322
+ slot.surface = Skia.Surface.MakeFromGPUTexture(slot.texture!);
323
+ if (!slot.surface) throw new Error("Cannot wrap the export frame");
324
+ slot.canvas = getVideoCanvas(slot.surface);
325
+ }
326
+ window.pending = windowTexture(window, width, height);
327
+ const target = acquireWindowTarget(window);
328
+ target.canvas!.drawColor(Skia.Color("#00000000"), BlendMode.Clear);
329
+ target.surface!.flush(true);
330
+ } catch (error) {
331
+ failure = error ?? new Error("GPU export probe failed");
332
+ }
333
+ if (failure === null) return { window, failure };
334
+ if (window) {
335
+ // Not caught: a window still drawn by WebGPU cannot fall back to EGL.
336
+ closeEncoderWindow(window);
337
+ releaseEncoderWindow(window);
338
+ }
339
+ return { window: null, failure };
340
+ };
@@ -35,10 +35,16 @@ import {
35
35
  import { enqueueExport, exportAbortError } from "./exportQueue";
36
36
  import {
37
37
  acquireEncoderTarget,
38
+ acquireWindowTarget,
38
39
  closeEncoderTarget,
40
+ closeEncoderWindow,
41
+ presentWindowTarget,
39
42
  probeEncoderTarget,
43
+ probeEncoderWindow,
44
+ releaseEncoderWindow,
40
45
  supportsEncoderTargets,
41
46
  type EncoderTarget,
47
+ type EncoderWindow,
42
48
  } from "./exportTarget";
43
49
  import { yieldToRuntime } from "./runtimeYield";
44
50
  import {
@@ -82,7 +88,8 @@ export type ExportProfile = {
82
88
  targetMs: number;
83
89
  /** drawFrame, before/afterDrawFrame and progress scheduling. */
84
90
  drawMs: number;
85
- /** The checked flush(true): waiting for the GPU to complete the frame. */
91
+ /** The checked flush(true): waiting for the GPU to complete the frame,
92
+ * which Android's window export does once it decoded the next frame. */
86
93
  gpuMs: number;
87
94
  /** Readback when used, submission to the encoder including its
88
95
  * backpressure, and the final file write. */
@@ -136,11 +143,16 @@ type ExportState = {
136
143
  /** Draw into a lent encoder buffer instead of reading pixels back. */
137
144
  gpu: boolean;
138
145
  target: EncoderTarget | null;
146
+ /** Android GPU export: the encoder's input window, drawn into directly. */
147
+ window: EncoderWindow | null;
139
148
  /** iOS GPU export: decoded frames sampled in place, by producer. */
140
149
  imports: ExportFrameImports | null;
141
150
  frames: Record<string, VideoFrame>;
142
151
  processor: VideoFrameProcessor | null;
143
152
  pendingAfterDraw: (() => void) | null;
153
+ /** Android window: the last frame was submitted without its checked GPU
154
+ * wait, which comes before anything it sampled is released. */
155
+ unchecked?: boolean;
144
156
  reservations: number[];
145
157
  index: number;
146
158
  setupError?: unknown;
@@ -290,6 +302,16 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
290
302
  if (clock.last >= 0) profile[stage] += now - clock.last;
291
303
  clock.last = now;
292
304
  };
305
+ /** Android window: the checked GPU wait the last frame left for later,
306
+ * so that the GPU converted and drew it while the CPU presented it and
307
+ * decoded the next one. An empty recording: Graphite still waits for
308
+ * every command submitted so far. */
309
+ const checkWindowFrame = () => {
310
+ "worklet";
311
+ if (!state.unchecked) return;
312
+ state.window!.slots[state.window!.next]!.surface!.flush(true);
313
+ state.unchecked = false;
314
+ };
293
315
  const finish = async (
294
316
  error: unknown,
295
317
  aborted = false,
@@ -313,6 +335,7 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
313
335
  };
314
336
  // Submit any recording left by a drawing exception before an
315
337
  // adapter is allowed to destroy textures referenced by that canvas.
338
+ clean(checkWindowFrame);
316
339
  clean(() => state.surface?.flush(true));
317
340
  clean(() => state.target?.surface?.flush(true));
318
341
  if (cleanup.safeToRelease && state.pendingAfterDraw) {
@@ -329,6 +352,10 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
329
352
  }
330
353
  }
331
354
  if (clean(() => state.extractor?.dispose())) state.extractor = null;
355
+ // Android: the window's swapchain lets go of the codec's buffers
356
+ // before the encoder goes. Its own textures are freed further down.
357
+ const window = state.window;
358
+ if (window) clean(() => closeEncoderWindow(window));
332
359
  if (clean(() => state.encoder?.dispose())) state.encoder = null;
333
360
  if (cleanup.safeToRelease) {
334
361
  clean(() => state.surface?.flush(true));
@@ -341,10 +368,20 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
341
368
  state.imports = null;
342
369
  clean(() => clearVideoFrameImages(true));
343
370
  }
344
- if (cleanup.safeToRelease && clean(() => state.canvas?.dispose()))
345
- state.canvas = null;
346
- if (cleanup.safeToRelease && clean(() => state.surface?.dispose()))
371
+ // Disposing the surface detaches its canvas.
372
+ if (cleanup.safeToRelease && clean(() => state.surface?.dispose())) {
347
373
  state.surface = null;
374
+ state.canvas = null;
375
+ }
376
+ // The target, if any, is one of the window's slots.
377
+ if (
378
+ cleanup.safeToRelease &&
379
+ window &&
380
+ clean(() => releaseEncoderWindow(window))
381
+ ) {
382
+ state.window = null;
383
+ state.target = null;
384
+ }
348
385
  const target = state.target;
349
386
  if (cleanup.safeToRelease && target) {
350
387
  if (clean(() => closeEncoderTarget(target))) state.target = null;
@@ -394,6 +431,8 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
394
431
  const owned = runWithVideoAutoreleasePool(() => {
395
432
  const decoded = state.extractor!.decodeCompositionFrames(time);
396
433
  lap("decodeMs");
434
+ checkWindowFrame();
435
+ lap("gpuMs");
397
436
  const imports = state.imports;
398
437
  // In place, a producer's frame is released by its next import.
399
438
  const imported = imports
@@ -427,11 +466,13 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
427
466
  const draw = () => {
428
467
  "worklet";
429
468
  if (state.gpu)
430
- state.target = acquireEncoderTarget(
431
- state.encoder!,
432
- options.width,
433
- options.height,
434
- );
469
+ state.target = state.window
470
+ ? acquireWindowTarget(state.window)
471
+ : acquireEncoderTarget(
472
+ state.encoder!,
473
+ options.width,
474
+ options.height,
475
+ );
435
476
  lap("targetMs");
436
477
  const target = state.target;
437
478
  const surface = target ? target.surface! : state.surface!;
@@ -454,18 +495,25 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
454
495
  height: options.height,
455
496
  });
456
497
  lap("drawMs");
457
- surface.flush(true);
498
+ // Android window, no context to release: the next frame waits
499
+ // for this one (see checkWindowFrame).
500
+ const unchecked =
501
+ !!state.window &&
502
+ !!target &&
503
+ !afterDrawFrame &&
504
+ !state.processor;
505
+ surface.flush(!unchecked);
506
+ state.unchecked = unchecked;
458
507
  drawn.drained = true;
459
508
  lap("gpuMs");
460
- if (target) {
461
- closeEncoderTarget(target, true);
509
+ if (target && state.window) {
510
+ state.target = null;
511
+ presentWindowTarget(state.encoder!, state.window, target, time);
512
+ } else if (target) {
513
+ closeEncoderTarget(target);
462
514
  state.target = null;
463
515
  state.encoder!.encodeFrame(
464
- {
465
- kind: "native-buffer",
466
- nativeBuffer: target.pointer,
467
- fences: target.fences,
468
- },
516
+ { kind: "native-buffer", nativeBuffer: target.pointer },
469
517
  time,
470
518
  );
471
519
  } else {
@@ -615,6 +663,7 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
615
663
  readback: null,
616
664
  gpu: false,
617
665
  target: null,
666
+ window: null,
618
667
  imports: null,
619
668
  frames: {},
620
669
  processor: null,
@@ -666,16 +715,28 @@ const executeExport = <T>(args: VideoExportArguments<T>): Promise<void> =>
666
715
  );
667
716
  state.encoder.prepare();
668
717
  if (gpuExport) {
669
- // Skia draws straight into the encoder's buffers; the probe
670
- // keeps devices without IOSurface interop on the readback path.
718
+ // Skia draws straight into the encoder's buffers: lent pool
719
+ // buffers on iOS, the input window on Android. The probe keeps
720
+ // devices where that fails on the readback path.
671
721
  state.hasRecorder = true;
672
- const failure = supportsEncoderTargets(state.encoder)
673
- ? probeEncoderTarget(
674
- state.encoder,
675
- options.width,
676
- options.height,
677
- )
678
- : new Error("Encoder cannot lend frame buffers");
722
+ let failure: unknown;
723
+ if (bgra) {
724
+ failure = supportsEncoderTargets(state.encoder)
725
+ ? probeEncoderTarget(
726
+ state.encoder,
727
+ options.width,
728
+ options.height,
729
+ )
730
+ : new Error("Encoder cannot lend frame buffers");
731
+ } else {
732
+ const probe = probeEncoderWindow(
733
+ state.encoder,
734
+ options.width,
735
+ options.height,
736
+ );
737
+ state.window = probe.window;
738
+ failure = probe.failure;
739
+ }
679
740
  state.gpu = failure === null;
680
741
  // Decoded frames are sampled in place too: on iOS through the
681
742
  // same IOSurface import, on Android once converted from YUV.
@@ -19,7 +19,7 @@ import {
19
19
  blitNv12Frame,
20
20
  blitYuvFrame,
21
21
  releaseYuvImports,
22
- yuvFrameOrigin,
22
+ yuvFrameRegion,
23
23
  } from "./yuvBlit";
24
24
  import type { YuvImport } from "./yuvBlit";
25
25
 
@@ -116,7 +116,6 @@ const clearEntry = (cache: Cache, producer: number) => {
116
116
  * The caller has either not submitted a copy or has completed its GPU fence. */
117
117
  const clearSurfaceView = (cache: Cache) => {
118
118
  "worklet";
119
- cache.canvas?.dispose();
120
119
  cache.canvas = null;
121
120
  cache.stage?.dispose();
122
121
  cache.stage = null;
@@ -449,14 +448,15 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
449
448
  const region = texture.region;
450
449
  const rotation = texture.rotation ?? 0;
451
450
  const nv12 = !region && texture.format === "nv12";
452
- let origin = { x: 0, y: 0 };
451
+ let upright: ReturnType<typeof yuvFrameRegion> | null = null;
453
452
  if (region) {
454
453
  // An Android codec buffer: padded, YUV, in encoded orientation. The
455
- // blit converts, crops and turns it into the display-sized scratch,
456
- // as the GL path's SurfaceTexture transform did. RN WebGPU labels
457
- // driver-defined YUV formats "bgra8", yet Dawn imports them as YUV:
458
- // the native side checked the format, so the label is not read.
459
- origin = yuvFrameOrigin(region, rotation, nativeSource, frame);
454
+ // blit converts, crops, turns and scales it into the frame-sized
455
+ // scratch, as the GL path's SurfaceTexture transform did. RN WebGPU
456
+ // labels driver-defined YUV formats "bgra8", yet Dawn imports them
457
+ // as YUV: the native side checked the format, so the label is not
458
+ // read.
459
+ upright = yuvFrameRegion(region, rotation, nativeSource);
460
460
  } else if (nv12) {
461
461
  if (
462
462
  nativeSource.width !== frame.width ||
@@ -487,9 +487,7 @@ export const imageFromVideoFrame = (frame: VideoFrame): SkImage | null => {
487
487
  key,
488
488
  nativeSource,
489
489
  rotation,
490
- origin,
491
- frame.width,
492
- frame.height,
490
+ upright!,
493
491
  cache.uploadTexture!,
494
492
  );
495
493
  } else if (nv12) {
package/src/gpu.ts CHANGED
@@ -178,7 +178,6 @@ export const createVideoGpuScope = (options?: {
178
178
  } catch (error) {
179
179
  // No texture has been exposed or submitted to any consumer yet.
180
180
  try {
181
- resource.canvas?.dispose();
182
181
  resource.canvas = null;
183
182
  resource.surface?.dispose();
184
183
  resource.surface = null;
@@ -265,7 +264,6 @@ export const createVideoGpuScope = (options?: {
265
264
  try {
266
265
  resource.image?.dispose();
267
266
  resource.image = null;
268
- resource.canvas?.dispose();
269
267
  resource.canvas = null;
270
268
  resource.surface?.dispose();
271
269
  resource.surface = null;