visualfries 0.1.10120 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +97 -0
  2. package/bin/visualfries.js +1570 -0
  3. package/dist/DIContainer.d.ts +2 -1
  4. package/dist/DIContainer.js +16 -5
  5. package/dist/SceneBuilder.svelte.d.ts +70 -1
  6. package/dist/SceneBuilder.svelte.js +96 -29
  7. package/dist/agent/audioMixer.d.ts +34 -0
  8. package/dist/agent/audioMixer.js +308 -0
  9. package/dist/agent/broll.d.ts +35 -0
  10. package/dist/agent/broll.js +226 -0
  11. package/dist/agent/capabilities.d.ts +34 -0
  12. package/dist/agent/capabilities.js +246 -0
  13. package/dist/agent/captionScene.d.ts +23 -0
  14. package/dist/agent/captionScene.js +204 -0
  15. package/dist/agent/catalog.d.ts +24 -0
  16. package/dist/agent/catalog.js +67 -0
  17. package/dist/agent/cueFiles.d.ts +54 -0
  18. package/dist/agent/cueFiles.js +219 -0
  19. package/dist/agent/cuePresets.d.ts +7 -0
  20. package/dist/agent/cuePresets.js +50 -0
  21. package/dist/agent/deterministicMedia.d.ts +61 -0
  22. package/dist/agent/deterministicMedia.js +303 -0
  23. package/dist/agent/index.d.ts +18 -0
  24. package/dist/agent/index.js +18 -0
  25. package/dist/agent/inspectScene.d.ts +36 -0
  26. package/dist/agent/inspectScene.js +139 -0
  27. package/dist/agent/localRender.d.ts +82 -0
  28. package/dist/agent/localRender.js +702 -0
  29. package/dist/agent/overlays.d.ts +40 -0
  30. package/dist/agent/overlays.js +325 -0
  31. package/dist/agent/productionPlan.d.ts +131 -0
  32. package/dist/agent/productionPlan.js +322 -0
  33. package/dist/agent/renderPlan.d.ts +27 -0
  34. package/dist/agent/renderPlan.js +68 -0
  35. package/dist/agent/renderRanges.d.ts +24 -0
  36. package/dist/agent/renderRanges.js +73 -0
  37. package/dist/agent/runtimeSupport.d.ts +19 -0
  38. package/dist/agent/runtimeSupport.js +192 -0
  39. package/dist/agent/streamingEncoder.d.ts +31 -0
  40. package/dist/agent/streamingEncoder.js +208 -0
  41. package/dist/agent/transcripts.d.ts +29 -0
  42. package/dist/agent/transcripts.js +146 -0
  43. package/dist/agent/transitions.d.ts +20 -0
  44. package/dist/agent/transitions.js +169 -0
  45. package/dist/animations/PixiAnimationTarget.d.ts +29 -0
  46. package/dist/animations/PixiAnimationTarget.js +63 -0
  47. package/dist/animations/builders/LineHighlighterAnimationBuilder.js +20 -3
  48. package/dist/animations/builders/WordHighlighterAnimationBuilder.js +15 -2
  49. package/dist/browser/browserExport.d.ts +73 -0
  50. package/dist/browser/browserExport.js +180 -0
  51. package/dist/browser/glTransitions.d.ts +37 -0
  52. package/dist/browser/glTransitions.js +261 -0
  53. package/dist/browser/index.d.ts +3 -0
  54. package/dist/browser/index.js +3 -0
  55. package/dist/browser/mediaDecode.d.ts +31 -0
  56. package/dist/browser/mediaDecode.js +152 -0
  57. package/dist/builders/PixiComponentBuilder.d.ts +6 -0
  58. package/dist/builders/PixiComponentBuilder.js +12 -0
  59. package/dist/builders/_ComponentState.svelte.d.ts +7 -5
  60. package/dist/builders/_ComponentState.svelte.js +29 -15
  61. package/dist/builders/html/StyleBuilder.js +9 -8
  62. package/dist/builders/html/TextComponentHtmlBuilder.js +4 -2
  63. package/dist/builders/html/processors/AppearanceStyleProcessor.js +1 -1
  64. package/dist/builders/html/processors/TextAppearanceStyleProcessor.js +7 -0
  65. package/dist/builders/html/processors/TextEffectsStyleProcessor.js +32 -13
  66. package/dist/commands/ReplaceSourceOnTimeCommand.js +9 -1
  67. package/dist/commands/SeekCommand.js +19 -6
  68. package/dist/components/AnimatedGIF.d.ts +3 -0
  69. package/dist/components/AnimatedGIF.js +28 -1
  70. package/dist/components/Component.svelte.d.ts +5 -5
  71. package/dist/components/Component.svelte.js +29 -13
  72. package/dist/components/ComponentContext.svelte.js +68 -23
  73. package/dist/components/hooks/AnimationHook.js +1 -1
  74. package/dist/components/hooks/CanvasFillHook.d.ts +12 -0
  75. package/dist/components/hooks/CanvasFillHook.js +75 -0
  76. package/dist/components/hooks/DeterministicMediaFrameHook.js +3 -0
  77. package/dist/components/hooks/HtmlTextHook.js +11 -3
  78. package/dist/components/hooks/HtmlToCanvasHook.js +8 -25
  79. package/dist/components/hooks/MediaHook.js +172 -47
  80. package/dist/components/hooks/MediaSeekingHook.js +138 -88
  81. package/dist/components/hooks/PixiGifHook.js +4 -4
  82. package/dist/components/hooks/PixiSplitScreenDisplayObjectHook.js +73 -6
  83. package/dist/components/hooks/PixiVideoTextureHook.js +10 -4
  84. package/dist/components/hooks/PixiVisualTransformHook.d.ts +16 -0
  85. package/dist/components/hooks/PixiVisualTransformHook.js +102 -0
  86. package/dist/components/hooks/SubtitlesHook.js +48 -24
  87. package/dist/components/hooks/VerifyMediaHook.js +2 -2
  88. package/dist/directors/ComponentDirector.d.ts +1 -0
  89. package/dist/directors/ComponentDirector.js +41 -8
  90. package/dist/fonts/fontDiscovery.js +12 -1
  91. package/dist/layers/Layer.svelte.d.ts +54 -0
  92. package/dist/layers/Layer.svelte.js +2 -25
  93. package/dist/managers/AppManager.svelte.js +8 -5
  94. package/dist/managers/ComponentsManager.svelte.d.ts +7 -7
  95. package/dist/managers/ComponentsManager.svelte.js +29 -15
  96. package/dist/managers/DeterministicMediaManager.js +38 -3
  97. package/dist/managers/EventManager.js +3 -1
  98. package/dist/managers/LayersManager.svelte.d.ts +54 -0
  99. package/dist/managers/RenderManager.d.ts +5 -0
  100. package/dist/managers/RenderManager.js +67 -21
  101. package/dist/managers/StateManager.svelte.d.ts +56 -0
  102. package/dist/managers/StateManager.svelte.js +60 -58
  103. package/dist/managers/SubtitlesManager.svelte.js +1 -3
  104. package/dist/managers/TimeManager.svelte.d.ts +4 -0
  105. package/dist/managers/TimeManager.svelte.js +23 -7
  106. package/dist/schemas/runtime/deterministic.js +5 -1
  107. package/dist/schemas/runtime/types.d.ts +36 -15
  108. package/dist/schemas/scene/components.d.ts +126 -0
  109. package/dist/schemas/scene/components.js +11 -0
  110. package/dist/schemas/scene/core.d.ts +109 -10
  111. package/dist/schemas/scene/core.js +1 -1
  112. package/dist/utils/mediaWindow.d.ts +13 -0
  113. package/dist/utils/mediaWindow.js +25 -0
  114. package/dist/utils/svgGenerator.js +1 -1
  115. package/dist/utils/timelineWindow.d.ts +2 -0
  116. package/dist/utils/timelineWindow.js +7 -0
  117. package/dist/utils/utils.js +2 -4
  118. package/docs/AGENT_PATTERNS.md +243 -0
  119. package/docs/AGENT_WORKFLOW.md +364 -0
  120. package/docs/AUTHORING_BEST_PRACTICES.md +188 -0
  121. package/docs/AUTHORING_SYSTEM_AUDIT.md +98 -0
  122. package/docs/BROWSER_EXPORT_SPIKE.md +55 -0
  123. package/docs/DETERMINISTIC_RENDERER.md +83 -0
  124. package/docs/NEXT_RELEASE.md +28 -0
  125. package/docs/PROPOSED_VISUALFRIES_SKILL.md +68 -0
  126. package/docs/examples/agent-cues.json +49 -0
  127. package/docs/examples/agent-transcript.json +23 -0
  128. package/docs/examples/agent-transcript.srt +7 -0
  129. package/docs/examples/agent-transcript.vtt +7 -0
  130. package/docs/examples/image-animation.json +87 -0
  131. package/docs/examples/native-text-metric-badge.json +109 -0
  132. package/package.json +133 -100
  133. package/skills/visualfries/SKILL.md +328 -0
@@ -0,0 +1,261 @@
1
+ /**
2
+ * Curated, gl-transitions-compatible WebGL transitions for deterministic browser export.
3
+ *
4
+ * Shader source is intentionally private. Callers select a known transition and may only
5
+ * provide the parameters declared by that transition; arbitrary GLSL is never accepted.
6
+ */
7
+ export const GL_TRANSITION_NAMES = ['fade', 'radial-wipe'];
8
+ const DEFINITIONS = {
9
+ fade: {
10
+ parameters: {},
11
+ body: 'return mix(getFromColor(p), getToColor(p), progress);'
12
+ },
13
+ 'radial-wipe': {
14
+ parameters: {
15
+ softness: { uniform: 'softness', defaultValue: 0.02, min: 0.001, max: 0.25 }
16
+ },
17
+ body: `
18
+ vec2 centered = p - vec2(0.5);
19
+ centered.x *= ratio;
20
+ float farthestCorner = 0.5 * sqrt(ratio * ratio + 1.0);
21
+ float radius = length(centered) / farthestCorner;
22
+ float leadingEdge = progress * (1.0 + softness);
23
+ float edge = smoothstep(leadingEdge - softness, leadingEdge, radius);
24
+ return mix(getToColor(p), getFromColor(p), edge);
25
+ `
26
+ }
27
+ };
28
+ export function listGlTransitions() {
29
+ return GL_TRANSITION_NAMES;
30
+ }
31
+ export function createGlTransitionRenderer(gl) {
32
+ return new GlTransitionRenderer(gl);
33
+ }
34
+ export class GlTransitionRenderer {
35
+ #gl;
36
+ #isWebGl2;
37
+ #programs = new Map();
38
+ #failures = new Map();
39
+ #quadBuffer;
40
+ #fromTexture;
41
+ #toTexture;
42
+ canvas;
43
+ constructor(gl) {
44
+ this.#gl = gl;
45
+ this.canvas = gl.canvas;
46
+ this.#isWebGl2 = typeof gl.texStorage2D === 'function';
47
+ const quadBuffer = gl.createBuffer();
48
+ if (!quadBuffer)
49
+ throw new Error('Unable to create WebGL transition vertex buffer');
50
+ this.#quadBuffer = quadBuffer;
51
+ this.#fromTexture = createTexture(gl);
52
+ this.#toTexture = createTexture(gl);
53
+ gl.bindBuffer(gl.ARRAY_BUFFER, quadBuffer);
54
+ gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]), gl.STATIC_DRAW);
55
+ }
56
+ render(input) {
57
+ const requested = input.transition;
58
+ let used = requested;
59
+ let fallbackReason;
60
+ let compiled;
61
+ try {
62
+ compiled = this.#getOrCompile(requested);
63
+ }
64
+ catch (error) {
65
+ if (requested === 'fade')
66
+ throw error;
67
+ used = 'fade';
68
+ fallbackReason = errorMessage(error);
69
+ compiled = this.#getOrCompile('fade');
70
+ }
71
+ this.#draw(compiled, DEFINITIONS[used], input);
72
+ return fallbackReason ? { requested, used, fallbackReason } : { requested, used };
73
+ }
74
+ destroy() {
75
+ for (const compiled of this.#programs.values())
76
+ this.#gl.deleteProgram(compiled.program);
77
+ this.#programs.clear();
78
+ this.#gl.deleteTexture(this.#fromTexture);
79
+ this.#gl.deleteTexture(this.#toTexture);
80
+ this.#gl.deleteBuffer(this.#quadBuffer);
81
+ }
82
+ #getOrCompile(name) {
83
+ const existing = this.#programs.get(name);
84
+ if (existing)
85
+ return existing;
86
+ const previousFailure = this.#failures.get(name);
87
+ if (previousFailure)
88
+ throw new Error(previousFailure);
89
+ try {
90
+ const compiled = this.#compile(name, DEFINITIONS[name]);
91
+ this.#programs.set(name, compiled);
92
+ return compiled;
93
+ }
94
+ catch (error) {
95
+ const message = `${name} transition unavailable: ${errorMessage(error)}`;
96
+ this.#failures.set(name, message);
97
+ throw new Error(message, { cause: error });
98
+ }
99
+ }
100
+ #compile(name, definition) {
101
+ const gl = this.#gl;
102
+ const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexSource(this.#isWebGl2));
103
+ let fragmentShader;
104
+ let program;
105
+ let compiledSuccessfully = false;
106
+ try {
107
+ fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragmentSource(this.#isWebGl2, definition));
108
+ program = gl.createProgram() ?? undefined;
109
+ if (!program)
110
+ throw new Error('program allocation failed');
111
+ gl.attachShader(program, vertexShader);
112
+ gl.attachShader(program, fragmentShader);
113
+ gl.linkProgram(program);
114
+ if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
115
+ const log = gl.getProgramInfoLog(program) || 'unknown link error';
116
+ throw new Error(`program link failed: ${log}`);
117
+ }
118
+ const parameterLocations = {};
119
+ for (const [parameterName, parameter] of Object.entries(definition.parameters)) {
120
+ parameterLocations[parameterName] = requiredUniform(gl, program, parameter.uniform, name);
121
+ }
122
+ const position = gl.getAttribLocation(program, 'position');
123
+ if (position < 0) {
124
+ gl.deleteProgram(program);
125
+ throw new Error(`${name} shader missing position attribute`);
126
+ }
127
+ const result = {
128
+ program,
129
+ position,
130
+ from: requiredUniform(gl, program, 'from', name),
131
+ to: requiredUniform(gl, program, 'to', name),
132
+ progress: requiredUniform(gl, program, 'progress', name),
133
+ // Crossfade does not mathematically need ratio, so a real GLSL compiler may optimize it out.
134
+ ratio: gl.getUniformLocation(program, 'ratio'),
135
+ parameters: parameterLocations
136
+ };
137
+ compiledSuccessfully = true;
138
+ return result;
139
+ }
140
+ finally {
141
+ if (program && !compiledSuccessfully)
142
+ gl.deleteProgram(program);
143
+ gl.deleteShader(vertexShader);
144
+ if (fragmentShader)
145
+ gl.deleteShader(fragmentShader);
146
+ }
147
+ }
148
+ #draw(compiled, definition, input) {
149
+ const gl = this.#gl;
150
+ gl.viewport(0, 0, this.canvas.width, this.canvas.height);
151
+ gl.useProgram(compiled.program);
152
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.#quadBuffer);
153
+ gl.enableVertexAttribArray(compiled.position);
154
+ gl.vertexAttribPointer(compiled.position, 2, gl.FLOAT, false, 0, 0);
155
+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1);
156
+ gl.activeTexture(gl.TEXTURE0);
157
+ uploadTexture(gl, this.#fromTexture, input.from);
158
+ gl.uniform1i(compiled.from, 0);
159
+ gl.activeTexture(gl.TEXTURE1);
160
+ uploadTexture(gl, this.#toTexture, input.to);
161
+ gl.uniform1i(compiled.to, 1);
162
+ gl.uniform1f(compiled.progress, clampFinite(input.progress, 0, 1, 0));
163
+ gl.uniform1f(compiled.ratio, clampFinite(input.ratio, 0.000001, 1_000_000, 1));
164
+ for (const [parameterName, parameter] of Object.entries(definition.parameters)) {
165
+ const requested = input.parameters?.[parameterName] ?? parameter.defaultValue;
166
+ gl.uniform1f(compiled.parameters[parameterName], clampFinite(requested, parameter.min, parameter.max, parameter.defaultValue));
167
+ }
168
+ gl.drawArrays(gl.TRIANGLES, 0, 6);
169
+ }
170
+ }
171
+ function vertexSource(webGl2) {
172
+ if (webGl2) {
173
+ return `#version 300 es
174
+ in vec2 position;
175
+ out vec2 _uv;
176
+ void main() {
177
+ _uv = position * 0.5 + 0.5;
178
+ gl_Position = vec4(position, 0.0, 1.0);
179
+ }`;
180
+ }
181
+ return `
182
+ attribute vec2 position;
183
+ varying vec2 _uv;
184
+ void main() {
185
+ _uv = position * 0.5 + 0.5;
186
+ gl_Position = vec4(position, 0.0, 1.0);
187
+ }`;
188
+ }
189
+ function fragmentSource(webGl2, definition) {
190
+ const parameterUniforms = Object.values(definition.parameters)
191
+ .map((parameter) => `uniform float ${parameter.uniform};`)
192
+ .join('\n');
193
+ const varying = webGl2 ? 'in vec2 _uv;' : 'varying vec2 _uv;';
194
+ const sample = webGl2 ? 'texture' : 'texture2D';
195
+ const output = webGl2 ? 'out vec4 transitionColor;' : '';
196
+ const setColor = webGl2 ? 'transitionColor =' : 'gl_FragColor =';
197
+ const version = webGl2 ? '#version 300 es' : '';
198
+ return `${version}
199
+ precision highp float;
200
+ uniform sampler2D from;
201
+ uniform sampler2D to;
202
+ uniform float progress;
203
+ uniform float ratio;
204
+ ${parameterUniforms}
205
+ ${varying}
206
+ ${output}
207
+
208
+ vec4 getFromColor(vec2 p) { return ${sample}(from, p); }
209
+ vec4 getToColor(vec2 p) { return ${sample}(to, p); }
210
+ vec4 transition(vec2 p) { ${definition.body} }
211
+
212
+ void main() {
213
+ if (progress <= 0.0) {
214
+ ${setColor} getFromColor(_uv);
215
+ } else if (progress >= 1.0) {
216
+ ${setColor} getToColor(_uv);
217
+ } else {
218
+ ${setColor} transition(_uv);
219
+ }
220
+ }`;
221
+ }
222
+ function compileShader(gl, type, source) {
223
+ const shader = gl.createShader(type);
224
+ if (!shader)
225
+ throw new Error('shader allocation failed');
226
+ gl.shaderSource(shader, source);
227
+ gl.compileShader(shader);
228
+ if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
229
+ const log = gl.getShaderInfoLog(shader) || 'unknown compile error';
230
+ gl.deleteShader(shader);
231
+ throw new Error(`shader compilation failed: ${log}`);
232
+ }
233
+ return shader;
234
+ }
235
+ function createTexture(gl) {
236
+ const texture = gl.createTexture();
237
+ if (!texture)
238
+ throw new Error('Unable to create WebGL transition texture');
239
+ gl.bindTexture(gl.TEXTURE_2D, texture);
240
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
241
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
242
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
243
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
244
+ return texture;
245
+ }
246
+ function uploadTexture(gl, texture, source) {
247
+ gl.bindTexture(gl.TEXTURE_2D, texture);
248
+ gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source);
249
+ }
250
+ function requiredUniform(gl, program, name, transition) {
251
+ const location = gl.getUniformLocation(program, name);
252
+ if (location === null)
253
+ throw new Error(`${transition} shader missing ${name} uniform`);
254
+ return location;
255
+ }
256
+ function clampFinite(value, min, max, fallback) {
257
+ return Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : fallback;
258
+ }
259
+ function errorMessage(error) {
260
+ return error instanceof Error ? error.message : String(error);
261
+ }
@@ -0,0 +1,3 @@
1
+ export * from './browserExport.js';
2
+ export * from './glTransitions.js';
3
+ export * from './mediaDecode.js';
@@ -0,0 +1,3 @@
1
+ export * from './browserExport.js';
2
+ export * from './glTransitions.js';
3
+ export * from './mediaDecode.js';
@@ -0,0 +1,31 @@
1
+ import type { DeterministicFrameProvider } from '../schemas/runtime/deterministic.js';
2
+ export type MediabunnyVideoFrameProviderOptions = {
3
+ url: string;
4
+ componentId: string;
5
+ sourceStart?: number;
6
+ fps: number;
7
+ frameCount?: number;
8
+ /** Alias for frameCount, useful when callers derive a bounded component window. */
9
+ maxFrames?: number;
10
+ };
11
+ export type MediabunnyAudioTrack = {
12
+ url: string;
13
+ startAt: number;
14
+ endAt?: number;
15
+ volume?: number;
16
+ muted?: boolean;
17
+ };
18
+ export type MixMediabunnyAudioTracksOptions = {
19
+ tracks: MediabunnyAudioTrack[];
20
+ duration: number;
21
+ sampleRate?: number;
22
+ channels?: number;
23
+ };
24
+ /**
25
+ * Opens one video input and exposes its decoded frames to deterministic composition.
26
+ * Requests must be sequential so MediaBunny can decode each packet at most once. Only
27
+ * one sample and its temporary VideoFrame are held by this adapter at any time.
28
+ */
29
+ export declare function createMediabunnyVideoFrameProvider(options: MediabunnyVideoFrameProviderOptions): Promise<DeterministicFrameProvider>;
30
+ /** Decode staged media tracks, align each track's first decoded timestamp to startAt, then mix. */
31
+ export declare function mixMediabunnyAudioTracks(options: MixMediabunnyAudioTracksOptions): Promise<AudioBuffer>;
@@ -0,0 +1,152 @@
1
+ import { ALL_FORMATS, AudioBufferSink, Input, UrlSource, VideoSampleSink } from 'mediabunny';
2
+ const assertPositiveFinite = (value, label) => {
3
+ if (!Number.isFinite(value) || value <= 0) {
4
+ throw new Error(`${label} must be a positive finite number`);
5
+ }
6
+ };
7
+ const openInput = (url) => new Input({ formats: ALL_FORMATS, source: new UrlSource(url) });
8
+ /**
9
+ * Opens one video input and exposes its decoded frames to deterministic composition.
10
+ * Requests must be sequential so MediaBunny can decode each packet at most once. Only
11
+ * one sample and its temporary VideoFrame are held by this adapter at any time.
12
+ */
13
+ export async function createMediabunnyVideoFrameProvider(options) {
14
+ assertPositiveFinite(options.fps, 'Video provider FPS');
15
+ const sourceStart = options.sourceStart ?? 0;
16
+ if (!Number.isFinite(sourceStart) || sourceStart < 0) {
17
+ throw new Error('Video provider sourceStart must be a non-negative finite number');
18
+ }
19
+ const maxFrames = options.frameCount ?? options.maxFrames;
20
+ if (!Number.isInteger(maxFrames) || maxFrames <= 0) {
21
+ throw new Error('Video provider frameCount/maxFrames must be a positive integer');
22
+ }
23
+ const input = openInput(options.url);
24
+ try {
25
+ const track = await input.getPrimaryVideoTrack();
26
+ if (!track)
27
+ throw new Error(`No video track found at ${options.url}`);
28
+ if (!(await track.canDecode())) {
29
+ throw new Error(`Primary video track cannot be decoded by this browser: ${options.url}`);
30
+ }
31
+ const sink = new VideoSampleSink(track);
32
+ const timestamps = (function* () {
33
+ for (let frameIndex = 0; frameIndex < maxFrames; frameIndex += 1) {
34
+ yield sourceStart + frameIndex / options.fps;
35
+ }
36
+ })();
37
+ const samples = sink.samplesAtTimestamps(timestamps)[Symbol.asyncIterator]();
38
+ let nextFrameIndex = 0;
39
+ let disposed = false;
40
+ const dispose = async () => {
41
+ if (disposed)
42
+ return;
43
+ disposed = true;
44
+ try {
45
+ await samples.return?.();
46
+ }
47
+ finally {
48
+ input.dispose();
49
+ }
50
+ };
51
+ return {
52
+ async getFrame(request) {
53
+ if (disposed)
54
+ throw new Error('MediaBunny video frame provider is disposed');
55
+ if (request.componentId !== options.componentId)
56
+ return null;
57
+ if (request.fps !== options.fps) {
58
+ throw new Error(`Video provider FPS mismatch: expected ${options.fps}, received ${request.fps}`);
59
+ }
60
+ if (request.frameIndex !== nextFrameIndex) {
61
+ throw new Error(`MediaBunny video frames must be requested sequentially; expected ${nextFrameIndex}, ` +
62
+ `received ${request.frameIndex}`);
63
+ }
64
+ if (nextFrameIndex >= maxFrames)
65
+ return null;
66
+ const frameIndex = nextFrameIndex;
67
+ nextFrameIndex += 1;
68
+ const result = await samples.next();
69
+ const sample = result.done ? null : result.value;
70
+ if (!sample)
71
+ return null;
72
+ let videoFrame;
73
+ try {
74
+ videoFrame = sample.toVideoFrame();
75
+ const imageBitmap = await createImageBitmap(videoFrame);
76
+ return {
77
+ kind: 'imageBitmap',
78
+ cacheKey: `${options.componentId}:mediabunny:${frameIndex}`,
79
+ imageBitmap
80
+ };
81
+ }
82
+ finally {
83
+ videoFrame?.close();
84
+ sample.close();
85
+ }
86
+ },
87
+ async releaseComponent(componentId) {
88
+ if (componentId === options.componentId)
89
+ await dispose();
90
+ },
91
+ destroy: dispose
92
+ };
93
+ }
94
+ catch (error) {
95
+ input.dispose();
96
+ throw error;
97
+ }
98
+ }
99
+ /** Decode staged media tracks, align each track's first decoded timestamp to startAt, then mix. */
100
+ export async function mixMediabunnyAudioTracks(options) {
101
+ assertPositiveFinite(options.duration, 'Audio mix duration');
102
+ const sampleRate = options.sampleRate ?? 48_000;
103
+ const channels = options.channels ?? 2;
104
+ assertPositiveFinite(sampleRate, 'Audio mix sampleRate');
105
+ if (!Number.isInteger(channels) || channels <= 0) {
106
+ throw new Error('Audio mix channels must be a positive integer');
107
+ }
108
+ const context = new OfflineAudioContext(channels, Math.ceil(options.duration * sampleRate), sampleRate);
109
+ for (const trackOptions of options.tracks) {
110
+ if (trackOptions.muted || (trackOptions.volume ?? 1) <= 0)
111
+ continue;
112
+ if (!Number.isFinite(trackOptions.startAt) || trackOptions.startAt < 0) {
113
+ throw new Error('Audio track startAt must be a non-negative finite number');
114
+ }
115
+ const endAt = Math.min(trackOptions.endAt ?? options.duration, options.duration);
116
+ if (!Number.isFinite(endAt) || endAt <= trackOptions.startAt)
117
+ continue;
118
+ const input = openInput(trackOptions.url);
119
+ try {
120
+ const track = await input.getPrimaryAudioTrack();
121
+ if (!track)
122
+ throw new Error(`No audio track found at ${trackOptions.url}`);
123
+ if (!(await track.canDecode())) {
124
+ throw new Error(`Primary audio track cannot be decoded by this browser: ${trackOptions.url}`);
125
+ }
126
+ const sink = new AudioBufferSink(track);
127
+ const gain = context.createGain();
128
+ gain.gain.value = trackOptions.volume ?? 1;
129
+ gain.connect(context.destination);
130
+ let firstTimestamp;
131
+ const activeDuration = endAt - trackOptions.startAt;
132
+ for await (const wrapped of sink.buffers()) {
133
+ firstTimestamp ??= wrapped.timestamp;
134
+ const relativeTimestamp = wrapped.timestamp - firstTimestamp;
135
+ if (relativeTimestamp >= activeDuration)
136
+ break;
137
+ const remaining = activeDuration - relativeTimestamp;
138
+ const playDuration = Math.min(wrapped.duration, wrapped.buffer.duration, remaining);
139
+ if (playDuration <= 0)
140
+ continue;
141
+ const source = context.createBufferSource();
142
+ source.buffer = wrapped.buffer;
143
+ source.connect(gain);
144
+ source.start(trackOptions.startAt + relativeTimestamp, 0, playDuration);
145
+ }
146
+ }
147
+ finally {
148
+ input.dispose();
149
+ }
150
+ }
151
+ return await context.startRendering();
152
+ }
@@ -21,6 +21,8 @@ type PixiComponentCradle = {
21
21
  htmlToCanvasHook: ComponentHook;
22
22
  mediaSeekingHook: ComponentHook;
23
23
  deterministicMediaFrameHook: ComponentHook;
24
+ pixiVisualTransformHook: ComponentHook;
25
+ canvasFillHook: ComponentHook;
24
26
  };
25
27
  export declare class PixiComponentBuilder implements IComponentBuilder {
26
28
  private component;
@@ -43,8 +45,11 @@ export declare class PixiComponentBuilder implements IComponentBuilder {
43
45
  private animationHook;
44
46
  private mediaSeekingHook;
45
47
  private deterministicMediaFrameHook;
48
+ private pixiVisualTransformHook;
49
+ private canvasFillHook;
46
50
  constructor(cradle: PixiComponentCradle);
47
51
  withCanvasShape(): this;
52
+ withCanvasFill(): this;
48
53
  withProgressShape(): this;
49
54
  withMedia(): this;
50
55
  withMediaSeeking(): this;
@@ -56,6 +61,7 @@ export declare class PixiComponentBuilder implements IComponentBuilder {
56
61
  withSplitScreen(): this;
57
62
  withHtmlText(): this;
58
63
  withAnimation(): this;
64
+ withPixiAnimationTarget(): this;
59
65
  withHtmlAnimation(): this;
60
66
  withHtmlToCanvasHook(): this;
61
67
  withSubtitles(): this;
@@ -20,6 +20,8 @@ export class PixiComponentBuilder {
20
20
  animationHook;
21
21
  mediaSeekingHook;
22
22
  deterministicMediaFrameHook;
23
+ pixiVisualTransformHook;
24
+ canvasFillHook;
23
25
  constructor(cradle) {
24
26
  this.component = cradle.component;
25
27
  this.mediaHook = cradle.mediaHook;
@@ -41,11 +43,17 @@ export class PixiComponentBuilder {
41
43
  this.htmlToCanvasHook = cradle.htmlToCanvasHook;
42
44
  this.mediaSeekingHook = cradle.mediaSeekingHook;
43
45
  this.deterministicMediaFrameHook = cradle.deterministicMediaFrameHook;
46
+ this.pixiVisualTransformHook = cradle.pixiVisualTransformHook;
47
+ this.canvasFillHook = cradle.canvasFillHook;
44
48
  }
45
49
  withCanvasShape() {
46
50
  this.component.addHook(this.canvasShapeHook);
47
51
  return this;
48
52
  }
53
+ withCanvasFill() {
54
+ this.component.addHook(this.canvasFillHook);
55
+ return this;
56
+ }
49
57
  withProgressShape() {
50
58
  this.component.addHook(this.pixiProgressShapeHook);
51
59
  return this;
@@ -92,6 +100,10 @@ export class PixiComponentBuilder {
92
100
  this.component.addHook(this.animationHook);
93
101
  return this;
94
102
  }
103
+ withPixiAnimationTarget() {
104
+ this.component.addHook(this.pixiVisualTransformHook);
105
+ return this;
106
+ }
95
107
  withHtmlAnimation() {
96
108
  this.component.addHook(this.htmlAnimationHook);
97
109
  return this;
@@ -23,7 +23,7 @@ export declare class ComponentState implements ComponentProps {
23
23
  set end_at(time: number);
24
24
  set name(name: string);
25
25
  get order(): number;
26
- get visible(): true;
26
+ get visible(): boolean;
27
27
  get duration(): number;
28
28
  get asset_id(): string;
29
29
  get timeline(): {
@@ -429,15 +429,17 @@ export declare class ComponentState implements ComponentProps {
429
429
  dashArray?: number[] | undefined;
430
430
  }>;
431
431
  };
432
+ get sourceUrl(): string | undefined;
433
+ get sourceStartAt(): number | undefined;
432
434
  get checksum(): string;
433
435
  getData(): ComponentData;
434
436
  setData(data: ComponentData): void;
435
437
  setStart(start: number): void;
436
438
  setEnd(end: number): void;
437
439
  setStreamPath(path: string): void;
438
- updateText(text: string): Promise<void>;
440
+ updateText(text: string): void;
439
441
  update(data: Partial<AppearanceInput>): void;
440
- updateAppearance(appearance: Partial<AppearanceInput>): Promise<void>;
441
- setVisible(visible: boolean): Promise<void>;
442
- setOrder(order: number): Promise<void>;
442
+ updateAppearance(appearance: Partial<AppearanceInput>): void;
443
+ setVisible(visible: boolean): void;
444
+ setOrder(order: number): void;
443
445
  }
@@ -15,14 +15,15 @@ export class ComponentState {
15
15
  setRefreshCallback(callback) {
16
16
  this.refreshCallback = callback;
17
17
  }
18
- async maybeAutoRefresh() {
18
+ maybeAutoRefresh() {
19
19
  if (this.refreshCallback) {
20
20
  try {
21
- await this.refreshCallback();
21
+ this.refreshCallback().catch((error) => {
22
+ console.warn('Auto-refresh callback failed:', error);
23
+ });
22
24
  }
23
25
  catch (error) {
24
26
  console.warn('Auto-refresh callback failed:', error);
25
- // Don't re-throw to avoid breaking the update operation
26
27
  }
27
28
  }
28
29
  }
@@ -55,7 +56,7 @@ export class ComponentState {
55
56
  return this.#data.order || 1;
56
57
  }
57
58
  get visible() {
58
- return this.#data.visible ? this.#data.visible : true;
59
+ return this.#data.visible ?? true;
59
60
  }
60
61
  get duration() {
61
62
  return this.#data.timeline.endAt - this.#data.timeline.startAt;
@@ -77,6 +78,18 @@ export class ComponentState {
77
78
  get effects() {
78
79
  return this.#data.effects ?? {};
79
80
  }
81
+ get sourceUrl() {
82
+ if ('source' in this.#data && this.#data.source && 'url' in this.#data.source) {
83
+ return this.#data.source.url ?? undefined;
84
+ }
85
+ return undefined;
86
+ }
87
+ get sourceStartAt() {
88
+ if ('source' in this.#data && this.#data.source && 'startAt' in this.#data.source) {
89
+ return this.#data.source.startAt ?? undefined;
90
+ }
91
+ return undefined;
92
+ }
80
93
  get checksum() {
81
94
  return (this.id +
82
95
  '-' +
@@ -114,6 +127,9 @@ export class ComponentState {
114
127
  source = { startAt: 0 };
115
128
  this.#data.source = source;
116
129
  }
130
+ if (source.startAt === undefined || source.startAt === null) {
131
+ source.startAt = 0;
132
+ }
117
133
  if (source.startAt !== undefined && source.startAt !== null) {
118
134
  source.startAt += diff;
119
135
  source.startAt = Math.max(0, source.startAt);
@@ -158,7 +174,7 @@ export class ComponentState {
158
174
  if (this.type === 'VIDEO' || this.type === 'AUDIO') {
159
175
  this.#changeVideoEnd(diff);
160
176
  }
161
- this.#data.timeline.endAt = this.sceneState.transformTime(end);
177
+ this.#data.timeline.endAt = newEnd;
162
178
  this.#emitChange();
163
179
  }
164
180
  }
@@ -168,11 +184,11 @@ export class ComponentState {
168
184
  // this.#emitChange();
169
185
  // }
170
186
  }
171
- async updateText(text) {
187
+ updateText(text) {
172
188
  if (this.type === 'TEXT') {
173
189
  this.#data.text = text;
174
190
  this.#emitChange();
175
- await this.maybeAutoRefresh();
191
+ this.maybeAutoRefresh();
176
192
  }
177
193
  }
178
194
  update(data) {
@@ -189,28 +205,26 @@ export class ComponentState {
189
205
  // console.error('Error updating component data', res.error);
190
206
  // }
191
207
  }
192
- async updateAppearance(appearance) {
208
+ updateAppearance(appearance) {
193
209
  // Use $state.snapshot() to properly extract all properties from the reactive proxy
194
210
  const currentData = $state.snapshot(this.#data);
195
211
  const mergedAppearance = merge({}, currentData.appearance, appearance);
196
212
  this.#data = { ...currentData, appearance: mergedAppearance };
197
213
  this.#emitChange();
198
- await this.maybeAutoRefresh();
214
+ this.maybeAutoRefresh();
199
215
  }
200
- async setVisible(visible) {
216
+ setVisible(visible) {
201
217
  if (this.#data.visible !== visible) {
202
218
  this.#data.visible = visible;
203
219
  this.#emitChange();
204
- await this.maybeAutoRefresh();
220
+ this.maybeAutoRefresh();
205
221
  }
206
222
  }
207
- async setOrder(order) {
223
+ setOrder(order) {
208
224
  if (this.#data.order !== order) {
209
225
  this.#data.order = order;
210
- // Note: Emitting change here might trigger frequent updates if order changes often.
211
- // Consider if the parent manager should handle order changes and emit less frequently.
212
226
  this.#emitChange();
213
- await this.maybeAutoRefresh();
227
+ this.maybeAutoRefresh();
214
228
  }
215
229
  }
216
230
  }