storyshot 0.7.3 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -2
- package/bin/storyshot.mjs +27 -2
- package/package.json +19 -4
- package/skills/storyshot/SKILL.md +356 -3
- package/src/audio.mjs +45 -2
- package/src/build.mjs +15 -1
- package/src/check.mjs +42 -5
- package/src/compose.mjs +39 -10
- package/src/config.mjs +41 -4
- package/src/hyperframes.mjs +56 -8
- package/src/init.mjs +84 -0
- package/src/mcp.mjs +43 -5
- package/src/outputs.mjs +17 -6
- package/src/record.mjs +1 -0
- package/src/scene-kit/kit3d.js +340 -0
- package/src/scene-tools.mjs +151 -0
- package/src/scenes.mjs +254 -0
- package/src/voice.mjs +114 -5
package/src/audio.mjs
CHANGED
|
@@ -42,6 +42,49 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
|
|
|
42
42
|
'-f', 'f32le', '-ac', '2', '-ar', `${SR}`, 'pipe:1'], { maxBuffer: 1 << 30 });
|
|
43
43
|
const f = new Float32Array(pcm.buffer, pcm.byteOffset, pcm.length / 4);
|
|
44
44
|
for (let i = 0; i < Math.min(N, f.length / 2); i++) { music[0][i] = f[2 * i] * 0.5; music[1][i] = f[2 * i + 1] * 0.5; }
|
|
45
|
+
} else if (cfg.music?.style === 'calm') {
|
|
46
|
+
// calm (explainers): slow pad chords vi–IV–I–V in C, two bars each, a soft bass under them and a
|
|
47
|
+
// sparse bell; no drums, nothing on the beat that would pull at the voice.
|
|
48
|
+
// The film has to *end*, not stop. Truncating the loop wherever the video happens to finish leaves
|
|
49
|
+
// the last chord hanging (and faded down, which a listener hears as "turned off", not "over"), so
|
|
50
|
+
// the loop is phased to reach IV one chord before the end, the tonic lands there — under the last
|
|
51
|
+
// spoken line — and rings out through the tail: a plagal cadence, the gentlest full stop there is.
|
|
52
|
+
const BEAT = 60 / cfg.music.bpm, LEN = BEAT * 8;
|
|
53
|
+
const chords = [[57, 60, 64], [53, 57, 60], [48, 52, 55], [55, 59, 62]];
|
|
54
|
+
const CAD = Math.max(0, events.total - LEN); // where the tonic lands (a clip shorter than one
|
|
55
|
+
// chord is all cadence, which is the right answer)
|
|
56
|
+
const before = Math.ceil(CAD / LEN); // chords before it...
|
|
57
|
+
const phase = ((2 - before) % 4 + 4) % 4; // ...phased so the last of them is IV
|
|
58
|
+
const pad = (ch, t0, len, gain = 1) => {
|
|
59
|
+
for (const [k, note] of [...ch, ch[0] + 12].entries()) {
|
|
60
|
+
for (const det of [-0.06, 0.06]) {
|
|
61
|
+
const f = hz(note + det), filt = svf();
|
|
62
|
+
add(music, t0, len, (t) => {
|
|
63
|
+
const env = Math.min(1, t / 1.4) * Math.min(1, Math.max(0, len - t) / 1.6);
|
|
64
|
+
return filt(saw(f * t) * 0.6 + Math.sin(2 * Math.PI * f * t) * 0.4, 650 + 250 * Math.sin(t * 0.4 + k)).lp * env;
|
|
65
|
+
}, 0.024 * gain, (k - 1.5) * 0.35 + det * 2);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const fb = hz(ch[0] - 12);
|
|
69
|
+
add(music, t0, len, (t) => Math.sin(2 * Math.PI * fb * t) * Math.min(1, t / 0.6) * Math.min(1, Math.max(0, len - t) / 1.2), 0.09 * gain);
|
|
70
|
+
};
|
|
71
|
+
for (let b = 0, t0 = 0; t0 < CAD; b++, t0 = b * LEN) {
|
|
72
|
+
const ch = chords[(b + phase) % chords.length], len = Math.min(LEN, CAD - t0) + 1.2; // rings into the next chord
|
|
73
|
+
pad(ch, t0, len);
|
|
74
|
+
if (b % 2 === 0) for (const [j, note] of [ch[2] + 12, ch[1] + 24].entries()) {
|
|
75
|
+
const f = hz(note);
|
|
76
|
+
add(music, t0 + j * BEAT * 2, 2.5, (t) => (Math.sin(2 * Math.PI * f * t) + 0.15 * Math.sin(4 * Math.PI * f * t)) * Math.exp(-t * 1.8) * Math.min(1, t / 0.01), 0.03, j ? 0.3 : -0.3);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
// the cadence: the tonic swells a little louder than the loop ever did, with two bells over it
|
|
80
|
+
const RING = events.total - CAD, TONIC = [48, 52, 55];
|
|
81
|
+
pad(TONIC, CAD, RING, 1.15);
|
|
82
|
+
for (const [j, note] of [TONIC[2] + 12, TONIC[0] + 24].entries()) {
|
|
83
|
+
const f = hz(note);
|
|
84
|
+
add(music, CAD + j * BEAT, RING, (t) => (Math.sin(2 * Math.PI * f * t) + 0.2 * Math.sin(4 * Math.PI * f * t)) * Math.exp(-t * 0.9) * Math.min(1, t / 0.01), 0.035, j ? 0.3 : -0.3);
|
|
85
|
+
}
|
|
86
|
+
// fade in over 2 s; the cadence is what ends the music, so the fade out only guards the cut
|
|
87
|
+
for (const c of music) for (let i = 0; i < N; i++) c[i] *= Math.min(1, i / SR / 2) * Math.min(1, (N - i) / SR / 1.2);
|
|
45
88
|
} else if (cfg.music) {
|
|
46
89
|
// I–V–vi–IV in C at `bpm`, resolving on the end card (or fading at the end)
|
|
47
90
|
const BEAT = 60 / cfg.music.bpm, BAR = BEAT * 4;
|
|
@@ -134,7 +177,7 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
|
|
|
134
177
|
if (cfg.voice && events.voice?.length) {
|
|
135
178
|
let prevEnd = 0;
|
|
136
179
|
for (const line of events.voice) {
|
|
137
|
-
let { pcm, parts } = await speak(cfg, line.text, { log });
|
|
180
|
+
let { pcm, parts } = await speak(cfg, line.text, { log, chunk: !!line.chunk });
|
|
138
181
|
let dur = pcm.length / 2 / SR;
|
|
139
182
|
const start = Math.max(line.start, prevEnd + 0.12), room = line.until - start - 0.1;
|
|
140
183
|
let sped = 1;
|
|
@@ -198,7 +241,7 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
|
|
|
198
241
|
// so re-encoding by video sites doesn't clip
|
|
199
242
|
execFileSync('ffmpeg', ['-y', '-loglevel', 'error', '-i', silentVideo, '-i', wavFile,
|
|
200
243
|
'-map', '0:v', '-map', '1:a', '-c:v', 'copy', '-af', `loudnorm=I=-14:TP=-1.5:LRA=11,aresample=${SR},alimiter=limit=0.7:level=false:attack=1:release=50`, '-ar', `${SR}`, '-c:a', 'aac', '-b:a', '192k', '-shortest', '-movflags', '+faststart', cfg.out]);
|
|
201
|
-
log(`soundtrack: ${cfg.music ? (cfg.music.file ? 'music file' : `synth ${cfg.music.bpm} bpm`) : 'no music'}, ${events.clicks.length} clicks, ${cfg.sfx?.transitions ? events.whooshes.length : 0} whooshes` +
|
|
244
|
+
log(`soundtrack: ${cfg.music ? (cfg.music.file ? 'music file' : `synth${cfg.music.style ? ` ${cfg.music.style}` : ''} ${cfg.music.bpm} bpm`) : 'no music'}, ${events.clicks.length} clicks, ${cfg.sfx?.transitions ? events.whooshes.length : 0} whooshes` +
|
|
202
245
|
(cfg.voice ? `, ${events.voice.length} narration lines (${cfg.voice.provider}/${cfg.voice.voice})` : ''));
|
|
203
246
|
return { out: cfg.out, warnings, narration };
|
|
204
247
|
}
|
package/src/build.mjs
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
// record → compose → soundtrack, shared by the CLI and the MCP server.
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { execFileSync } from 'node:child_process';
|
|
2
5
|
import { record } from './record.mjs';
|
|
3
6
|
import { compose } from './compose.mjs';
|
|
4
7
|
import { soundtrack } from './audio.mjs';
|
|
@@ -7,7 +10,9 @@ import { writeOutputs, burnSubtitles } from './outputs.mjs';
|
|
|
7
10
|
|
|
8
11
|
export async function build(cfg, mode = 'render', { log = console.log } = {}) {
|
|
9
12
|
const t0 = Date.now();
|
|
10
|
-
|
|
13
|
+
// a storyboard of scenes only (an explainer) has no web app to record: an empty recording stands in
|
|
14
|
+
if (!cfg.url) noRecording(cfg);
|
|
15
|
+
else if (mode !== 'compose') await record(cfg, { log });
|
|
11
16
|
if (mode === 'record') return { warnings: [] };
|
|
12
17
|
const voiceLens = await measureVoice(cfg, { log });
|
|
13
18
|
const { silent, clean, events, preview, warnings, captions, cards, segs } = await compose(cfg, { log, voiceLens });
|
|
@@ -16,3 +21,12 @@ export async function build(cfg, mode = 'render', { log = console.log } = {}) {
|
|
|
16
21
|
const extras = await writeOutputs(cfg, { captions, cards, segs, clean, narration: audio.narration, total: events.total }, { log });
|
|
17
22
|
return { out: audio.out, extras, preview, warnings: [...warnings, ...audio.warnings], duration: events.total, seconds: Math.round((Date.now() - t0) / 1000) };
|
|
18
23
|
}
|
|
24
|
+
|
|
25
|
+
// The recording of a storyboard with nothing to record: one black frame, every step empty.
|
|
26
|
+
export function noRecording(cfg) {
|
|
27
|
+
fs.mkdirSync(path.join(cfg.work, 'raw'), { recursive: true });
|
|
28
|
+
const frame = path.join(cfg.work, 'raw', 'blank.jpg');
|
|
29
|
+
if (!fs.existsSync(frame)) execFileSync('ffmpeg', ['-y', '-loglevel', 'error', '-f', 'lavfi', '-i', `color=c=black:s=${cfg.output.width}x${cfg.output.height}`, '-frames:v', '1', frame]);
|
|
30
|
+
const steps = cfg.steps.map((s) => ({ start: 0, end: 0, hidden: !!s.hidden, zoomBox: null, zoomTarget: null, waits: [], holds: 0, marks: [] }));
|
|
31
|
+
fs.writeFileSync(path.join(cfg.work, 'timeline.json'), JSON.stringify({ frames: [{ t: 0, file: 'blank.jpg' }], steps, clicks: [], navs: [], gotos: [], urls: [], app: null, introLogo: null, scale: cfg.scale }, null, 1));
|
|
32
|
+
}
|
package/src/check.mjs
CHANGED
|
@@ -6,6 +6,8 @@ import { gitIgnored } from './auth.mjs';
|
|
|
6
6
|
import { isZhVoice, zhVoiceStatus } from './kokoro-zh.mjs';
|
|
7
7
|
import { qwenStatus } from './qwen-tts.mjs';
|
|
8
8
|
import { sentences } from './voice.mjs';
|
|
9
|
+
import { sceneInfo, filmGround, contrast } from './scenes.mjs';
|
|
10
|
+
import path from 'node:path';
|
|
9
11
|
|
|
10
12
|
// Phrases that mark narration as machine-written, and sentences too long to say in one breath.
|
|
11
13
|
const TELLS = [
|
|
@@ -34,9 +36,10 @@ export function narrationLint(text) {
|
|
|
34
36
|
return { tells, long };
|
|
35
37
|
}
|
|
36
38
|
|
|
37
|
-
const TOP = ['name', 'url', 'out', 'purpose', 'viewport', 'output', 'theme', 'music', 'voice', 'sfx', 'storageState', 'ignoreHTTPSErrors', 'intro', 'frame', 'pace', 'follow', 'article', 'cards', 'lang', 'colorScheme',
|
|
38
|
-
'scrollOffset', 'captionPosition', 'captions', 'highlight', 'stepNumbers', 'subtitles', 'chapters', 'clips', 'maxWait', 'setup', 'steps', 'end_card', 'hyperframes'];
|
|
39
|
-
const STEP = ['caption', 'say', 'sayAt', 'duration', 'zoom', 'actions', 'hidden', 'sfx', 'highlight', 'captionPosition', 'card', 'follow', 'doc'];
|
|
39
|
+
const TOP = ['name', 'url', 'out', 'purpose', 'viewport', 'output', 'theme', 'music', 'voice', 'sfx', 'storageState', 'ignoreHTTPSErrors', 'intro', 'frame', 'pace', 'follow', 'article', 'cards', 'ground', 'lang', 'colorScheme',
|
|
40
|
+
'scrollOffset', 'captionPosition', 'captions', 'highlight', 'stepNumbers', 'subtitles', 'chapters', 'clips', 'maxWait', 'setup', 'steps', 'end_card', 'hyperframes', 'scenes'];
|
|
41
|
+
const STEP = ['caption', 'say', 'sayAt', 'duration', 'zoom', 'actions', 'hidden', 'sfx', 'highlight', 'captionPosition', 'card', 'follow', 'doc', 'scene'];
|
|
42
|
+
const SCENE = ['file', 'title', 'lead', 'tail'];
|
|
40
43
|
const ZOOM = ['target', 'scale', 'start', 'end', 'ramp', 'offset'];
|
|
41
44
|
const END = ['logo', 'title', 'subtitle', 'cta', 'duration', 'say', 'kicker', 'bullets', 'image', 'think', 'emblem'];
|
|
42
45
|
const HYPERFRAMES = ['template', 'band', 'bandColors', 'emblem', 'hand', 'backdrop'];
|
|
@@ -103,9 +106,14 @@ export function check(file, { draft = false } = {}) {
|
|
|
103
106
|
if (!Array.isArray(s.actions ?? [])) errors.push(`${where}: actions must be a list`);
|
|
104
107
|
for (const [j, a] of (s.actions ?? []).entries()) checkAction(a, `${where} action ${j + 1}`);
|
|
105
108
|
if (s.card) unknown(s.card, END, `${where} card`);
|
|
109
|
+
if (s.scene) {
|
|
110
|
+
if (typeof s.scene === 'object') unknown(s.scene, SCENE, `${where} scene`);
|
|
111
|
+
for (const k of ['caption', 'zoom', 'sayAt', 'follow', 'highlight']) if (s[k] != null) warnings.push(`${where}: ${k} does nothing on a scene (a scene draws its own picture and text)`);
|
|
112
|
+
for (const k of ['lead', 'tail']) if (s.scene?.[k] != null && !(typeof s.scene[k] === 'number' && s.scene[k] >= 0)) errors.push(`${where} scene: ${k} is seconds (≥ 0)`);
|
|
113
|
+
}
|
|
106
114
|
// tutorials: one thing per step, so the viewer can follow (and the narration can explain it)
|
|
107
115
|
const purpose = raw.purpose ?? 'pitch';
|
|
108
|
-
if ((purpose === 'tutorial' || purpose === 'course') && !s.hidden) {
|
|
116
|
+
if ((purpose === 'tutorial' || purpose === 'course') && !s.hidden && !s.scene) {
|
|
109
117
|
const doing = (s.actions ?? []).filter((a) => a && ['click', 'dblclick', 'type', 'drag', 'press', 'goto', 'select', 'slide', 'paste'].includes(Object.keys(a)[0]) && !(Object.keys(a)[0] === 'press' && /^(Enter|Tab)$/.test(String(Object.values(a)[0]))));
|
|
110
118
|
if (doing.length > 2) warnings.push(`${where}: does ${doing.length} things (${doing.map((a) => Object.keys(a)[0]).join(', ')}) in one step — in a ${purpose}, split it so each action gets its own explanation and the viewer can keep up`);
|
|
111
119
|
if (doing.length && !s.say) warnings.push(`${where}: does something but has no say: line — in a ${purpose}, tell the viewer what and why`);
|
|
@@ -168,6 +176,34 @@ export function check(file, { draft = false } = {}) {
|
|
|
168
176
|
}
|
|
169
177
|
let summary = '';
|
|
170
178
|
if (cfg) {
|
|
179
|
+
// scenes: the file is there and keeps the HyperFrames contract (the full lint runs in `storyshot scene`)
|
|
180
|
+
for (const [i, s] of cfg.steps.entries()) {
|
|
181
|
+
if (!s.scene || s.hidden) continue;
|
|
182
|
+
const r = sceneInfo(s.scene.file, { ...cfg.output, three: !!cfg.sceneKit?.three });
|
|
183
|
+
errors.push(...r.errors.map((e) => `step ${i + 1}: ${e}`));
|
|
184
|
+
warnings.push(...r.warnings.map((w) => `step ${i + 1}: ${w}`));
|
|
185
|
+
if (!cfg.voice && !(typeof s.duration === 'number')) errors.push(`step ${i + 1}: a scene is as long as its narration, and there is no voice: — add a voice: block, or give the scene a duration`);
|
|
186
|
+
}
|
|
187
|
+
if (raw.scenes) unknown(raw.scenes, ['css', 'js', 'three'], 'scenes');
|
|
188
|
+
for (const f of [...cfg.sceneKit.css, ...cfg.sceneKit.js]) if (!fs.existsSync(f)) errors.push(`scenes: ${f} does not exist`);
|
|
189
|
+
for (const f of cfg.sceneKit.css) if (!/\.css$/i.test(f)) errors.push(`scenes.css: ${path.basename(f)} is not a .css file`);
|
|
190
|
+
for (const f of cfg.sceneKit.js) if (!/\.m?js$/i.test(f)) errors.push(`scenes.js: ${path.basename(f)} is not a .js file`);
|
|
191
|
+
const ids = cfg.steps.filter((s) => s.scene && !s.hidden).map((s) => sceneInfo(s.scene.file).id).filter(Boolean);
|
|
192
|
+
for (const id of new Set(ids.filter((x, k) => ids.indexOf(x) !== k))) errors.push(`two scenes share the composition id "${id}" — give each scene its own id (and prefix its element ids with it)`);
|
|
193
|
+
if (cfg.purpose === 'explainer' && !cfg.steps.some((s) => s.scene)) warnings.push('an explainer is made of scenes: give its steps a scene: file');
|
|
194
|
+
// burned-in subtitles follow the film's ground: ink letters on a light film, white on a dark one,
|
|
195
|
+
// with the word being spoken in the accent either way, so the accent has to carry a large text's
|
|
196
|
+
// 3:1 against that ground
|
|
197
|
+
if (cfg.subtitles === 'burn') {
|
|
198
|
+
const ground = filmGround(cfg), against = ground === 'light' ? '#ffffff' : '#000000';
|
|
199
|
+
const ratio = contrast(cfg.theme.accent, against);
|
|
200
|
+
if (ground && ratio != null && ratio < 3) {
|
|
201
|
+
warnings.push(`theme.accent (${cfg.theme.accent}) is only ${ratio}:1 against ${ground === 'light' ? 'white' : 'black'}, and the burned-in subtitles paint the word being spoken in the accent: on this ${ground} film it would barely read — ${ground === 'light' ? 'darken' : 'lighten'} the accent (large text needs 3:1)`);
|
|
202
|
+
}
|
|
203
|
+
if (ground == null && cfg.steps.some((s) => s.scene && !s.hidden)) {
|
|
204
|
+
warnings.push("the scenes' ground could not be read (a variable, an image, or a rule this check cannot follow), so the burned-in subtitles stay white, which needs something dark under it — say which the film is with `ground: light` or `ground: dark`");
|
|
205
|
+
}
|
|
206
|
+
}
|
|
171
207
|
const visible = cfg.steps.filter((s) => !s.hidden);
|
|
172
208
|
const says = visible.filter((s) => s.say).length + (cfg.endCard?.say ? 1 : 0) + (cfg.intro?.say ? 1 : 0);
|
|
173
209
|
if (cfg.voice && !says) warnings.push('voice is set but no step has a `say:` line');
|
|
@@ -230,7 +266,8 @@ export function check(file, { draft = false } = {}) {
|
|
|
230
266
|
const ec = cfg.endCard ? (cfg.endCard.duration === 'auto' ? null : cfg.endCard.duration) : 0;
|
|
231
267
|
const length = auto ? `${fixed ? `${fixed.toFixed(1)}s fixed + ` : ''}${auto} steps timed to narration${cfg.endCard ? ' + end card' : ''}${cfg.intro ? ' + intro' : ''}`
|
|
232
268
|
: `${fixed.toFixed(1)}s of steps${cfg.endCard ? ` + ${ec ?? 'auto'}s end card` : ''}${ec != null ? ` = ${(fixed + ec).toFixed(1)}s total` : ''}`;
|
|
233
|
-
|
|
269
|
+
const nScenes = visible.filter((s) => s.scene).length;
|
|
270
|
+
summary = `${cfg.name}: purpose ${cfg.purpose}, ${visible.length} visible steps${nScenes ? ` (${nScenes} scene${nScenes > 1 ? 's' : ''})` : ''}, ${length}` +
|
|
234
271
|
(cfg.voice ? `, voice ${cfg.voice.provider}` : '') + ` → ${cfg.out}`;
|
|
235
272
|
}
|
|
236
273
|
return { ok: !errors.length, errors, warnings, summary };
|
package/src/compose.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { execFileSync } from 'node:child_process';
|
|
|
5
5
|
import fs from 'node:fs';
|
|
6
6
|
import path from 'node:path';
|
|
7
7
|
import { renderHyperframesCards } from './hyperframes.mjs';
|
|
8
|
+
import { sceneSeconds, filmGround } from './scenes.mjs';
|
|
8
9
|
|
|
9
10
|
const esc = (s) => String(s).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
|
10
11
|
const md = (s, accent) => esc(s).replace(/\*\*(.+?)\*\*/g, `<b style="color:${accent};font-weight:700">$1</b>`);
|
|
@@ -89,6 +90,16 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
89
90
|
cards.push({ i, t0: acc, t1: acc + C, card: s.card, thinkAt: C - (s.card.think ?? 0) - 0.3 });
|
|
90
91
|
acc += C;
|
|
91
92
|
}
|
|
93
|
+
if (s.scene) {
|
|
94
|
+
// a scene is its own picture (the last frame shown holds under it) for as long as its narration needs
|
|
95
|
+
const C = sceneSeconds(s, voiceLens.steps[i]);
|
|
96
|
+
const prev = pieces.at(-1)?.raw[1] ?? r.start;
|
|
97
|
+
pieces.push({ raw: [prev, prev], o0: acc, o1: acc + C });
|
|
98
|
+
cards.push({ i, t0: acc, t1: acc + C, card: { title: s.scene.title ?? '' }, scene: s.scene, say: s.say, lead: s.scene.lead, cues: voiceLens.cues?.[i] ?? [] });
|
|
99
|
+
acc += C;
|
|
100
|
+
afterHidden = false;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
92
103
|
const g = { i, step: s, o0: acc, o1: acc + D, sayAt, zoomBox: r.zoomBox, afterHidden, hold,
|
|
93
104
|
motionSpeed: M && motionBudget ? M / motionBudget : 1, waitSpeed: Wt && waitBudget ? Wt / waitBudget : 1 };
|
|
94
105
|
for (const [a, b, w] of parts) {
|
|
@@ -185,7 +196,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
185
196
|
const report = (to) => {
|
|
186
197
|
if (from != null && to - from >= 3) {
|
|
187
198
|
const ns = segs.filter((x) => x.o1 > from + 1e-6 && x.o0 < to - 1e-6).map((x) => x.i + 1);
|
|
188
|
-
warn(`${ns.length > 1 ? `steps ${ns[0]}–${ns.at(-1)}` : `step ${ns[0] ?? segs.at(-1)
|
|
199
|
+
warn(`${ns.length > 1 ? `steps ${ns[0]}–${ns.at(-1)}` : `step ${ns[0] ?? (segs.at(-1)?.i ?? cfg.steps.length - 1) + 1}`}: ${(to - from).toFixed(1)}s of dead air at ${from.toFixed(1)}–${to.toFixed(1)}s (nothing moves and nothing is said) — shorten the wait or pause, hide the load, or narrate what is happening`);
|
|
189
200
|
}
|
|
190
201
|
from = null;
|
|
191
202
|
};
|
|
@@ -373,9 +384,10 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
373
384
|
// Intro and end cards are short animations (logo pops, title rises, the highlight sweeps, a command
|
|
374
385
|
// CTA types itself out), rendered frame by frame by seeking the page's CSS animations.
|
|
375
386
|
const u = H / 1080;
|
|
376
|
-
// cards follow the
|
|
377
|
-
// cut to a white card; `
|
|
378
|
-
|
|
387
|
+
// cards follow the film's ground (see filmGround): dark backdrops (aurora, dark, blur) and scenes that
|
|
388
|
+
// draw on a dark ground get dark cards, so a dark film doesn't cut to a white card; `ground: light |
|
|
389
|
+
// dark` says it outright and `cards: light | dark` overrides the cards alone
|
|
390
|
+
const darkCards = cfg.cards === 'dark' || (cfg.cards !== 'light' && filmGround(cfg) === 'dark');
|
|
379
391
|
const CK = darkCards
|
|
380
392
|
? { bg: `radial-gradient(${W * 0.62}px ${H * 0.65}px at 50% 40%,#1e2a44 0%,#111830 60%,#0b1020 100%)`, title: '#f1f5f9', sub: '#cbd5e1', muted: '#94a3b8', item: '#e2e8f0', track: '#26314d',
|
|
381
393
|
pillBg: '#1a2238', pillFg: '#e2e8f0', pillLine: '1px solid #33405f', ctaBg: accent, ctaFg: ink, shadow: 'rgba(0,0,0,.45)' }
|
|
@@ -485,7 +497,10 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
485
497
|
// what each HyperFrames card is about: its text, when its narration plays (card seconds), and
|
|
486
498
|
// pictures from the recording (the page it leads into, blurred, as a backdrop; for a launch intro, the
|
|
487
499
|
// first page sharp, and where the app shows the intro's logo, so the logo can land there)
|
|
500
|
+
// scenes: rendered with HyperFrames whatever `hyperframes:` says (they are HyperFrames compositions);
|
|
501
|
+
// the cards only when it is on
|
|
488
502
|
function hfJobs() {
|
|
503
|
+
if (!cfg.hyperframes) return cards.flatMap((c, k) => (c.scene ? [sceneJob(c, k)] : []));
|
|
489
504
|
const launch = cfg.hyperframes.template === 'launch';
|
|
490
505
|
const seqAt = (t) => path.join(SEQ, `${String(Math.min(nFrames - 1, Math.max(0, Math.round(t * FPS)))).padStart(5, '0')}.jpg`);
|
|
491
506
|
const blurred = (t, name) => {
|
|
@@ -522,6 +537,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
522
537
|
page: launch && nFrames ? seqAt(INTRO) : null, backdrop: launch ? null : blurred(INTRO + 0.05, 'intro') });
|
|
523
538
|
}
|
|
524
539
|
for (const [k, c] of cards.entries()) {
|
|
540
|
+
if (c.scene) { jobs.push(sceneJob(c, k)); continue; }
|
|
525
541
|
jobs.push({ name: `card${k}`, kind: 'section', seconds: c.t1 - c.t0, card: c.card, voice: voice(c.card.say, 0.15, voiceLens.cards?.[c.i]),
|
|
526
542
|
index: k, of: cards.length, thinkAt: c.thinkAt, image: c.card.imageFile, backdrop: blurred(c.t1 + 0.05, `card${k}`) });
|
|
527
543
|
}
|
|
@@ -534,10 +550,13 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
534
550
|
return jobs;
|
|
535
551
|
}
|
|
536
552
|
|
|
553
|
+
function sceneJob(c, k) {
|
|
554
|
+
return { name: `card${k}`, kind: 'scene', seconds: c.t1 - c.t0, scene: c.scene, say: c.say, lead: c.lead, cues: c.cues, step: c.i, kit: cfg.sceneKit };
|
|
555
|
+
}
|
|
537
556
|
// the cards: HyperFrames compositions when asked for (`hyperframes:`), else (or for any card that
|
|
538
557
|
// doesn't render) the built-in ones. A launch outro is drawn with transparency over the last frame,
|
|
539
558
|
// which the ffmpeg pass below zooms into the page.
|
|
540
|
-
const hf = cfg.hyperframes ? await renderHyperframesCards(hfJobs(), {
|
|
559
|
+
const hf = cfg.hyperframes || cards.some((c) => c.scene) ? await renderHyperframesCards(hfJobs(), {
|
|
541
560
|
W, H, FPS, dark: darkCards, accent, ink, font, lang: cfg.lang, labels: cfg.labels, opts: cfg.hyperframes, work: cfg.work, subs: cfg.subtitles === 'burn',
|
|
542
561
|
frame: { FX, FY, FW, FH, ...Object.fromEntries([['frameBg', 'frame-bg.png'], ['chrome', 'frame-chrome.png'], ['url', 'frame-url0.png']]
|
|
543
562
|
.map(([k, f]) => [k, cfg.frame && fs.existsSync(path.join(cfg.work, f)) ? path.join(cfg.work, f) : null])) },
|
|
@@ -555,6 +574,10 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
555
574
|
}
|
|
556
575
|
for (const [k, c] of cards.entries()) {
|
|
557
576
|
if (hf[`card${k}`]) place(`card${k}`, c.t1 - c.t0);
|
|
577
|
+
else if (c.scene) {
|
|
578
|
+
await browser.close();
|
|
579
|
+
throw new Error(`step ${c.i + 1}: the scene ${path.relative(process.cwd(), c.scene.file)} did not render${hf._errors?.[`card${k}`] ? `: ${hf._errors[`card${k}`]}` : ''} — preview it with \`storyshot scene <storyboard> --step ${c.i + 1}\` (MCP: preview_scene), fix it, and render again`);
|
|
580
|
+
}
|
|
558
581
|
else await animate(card(c.card, c.card.logoFile ?? '', { thinkAt: c.thinkAt }), c.t1 - c.t0, path.join(cfg.work, `card${k}`));
|
|
559
582
|
}
|
|
560
583
|
if (in0) {
|
|
@@ -607,7 +630,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
607
630
|
const lead = in0 && Math.abs(c.t0 - INTRO) < 0.02 ? Math.min(0.2, INTRO) : 0, s0 = c.t0 - lead;
|
|
608
631
|
inputs.push('-framerate', `${FPS}`, '-i', path.join(cfg.work, `card${k}`, '%04d.jpg'));
|
|
609
632
|
graph += `[${idx}:v]format=rgba,${lead ? `tpad=start_duration=${lead.toFixed(3)}:start_mode=clone,` : ''}setpts=PTS-STARTPTS+${s0.toFixed(3)}/TB,` +
|
|
610
|
-
`${lead ? '' : `fade=in:st=${c.t0.toFixed(3)}:d=0.15:alpha=1,`}fade=out:st=${Math.max(c.t0, c.t1 - 0.15).toFixed(3)}:d=0.15:alpha=1[sc${k}];\n` +
|
|
633
|
+
`${lead || c.scene ? '' : `fade=in:st=${c.t0.toFixed(3)}:d=0.15:alpha=1,`}${c.scene ? 'null' : `fade=out:st=${Math.max(c.t0, c.t1 - 0.15).toFixed(3)}:d=0.15:alpha=1`}[sc${k}];\n` +
|
|
611
634
|
`[${last}][sc${k}]overlay=0:0:eof_action=pass:enable='between(t,${s0.toFixed(3)},${c.t1.toFixed(3)})'[v${++idx}];\n`;
|
|
612
635
|
last = `v${idx}`;
|
|
613
636
|
}
|
|
@@ -636,7 +659,12 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
636
659
|
`[${last}][in]overlay=0:0:eof_action=pass[v${++idx}];\n`;
|
|
637
660
|
last = `v${idx}`;
|
|
638
661
|
}
|
|
639
|
-
|
|
662
|
+
// The picture needs a full stop too. Without one the file simply ceases on whatever was on screen —
|
|
663
|
+
// which in an explainer is the recap, the busiest frame of the film, held still and then cut. An end
|
|
664
|
+
// card is already an ending; otherwise settle to the film's own ground over the last of the tail.
|
|
665
|
+
const OUTRO = ec ? 0 : Math.min(0.9, TOTAL * 0.02);
|
|
666
|
+
const outro = OUTRO > 0.05 ? `,fade=out:st=${(TOTAL - OUTRO).toFixed(3)}:d=${OUTRO.toFixed(3)}:color=${filmGround(cfg) === 'light' ? 'white' : 'black'}` : '';
|
|
667
|
+
graph += `[${last}]trim=duration=${TOTAL},format=yuv420p${outro}[out]`;
|
|
640
668
|
if (clean) graph += `;\n[vclean0]trim=duration=${TOTAL},format=yuv420p[clean]`;
|
|
641
669
|
const graphFile = path.join(cfg.work, 'graph.txt');
|
|
642
670
|
fs.writeFileSync(graphFile, graph);
|
|
@@ -652,7 +680,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
652
680
|
// --- 6. preview contact sheet: one frame per step (at its zoom peak) + end card ------------
|
|
653
681
|
// zoomed steps: at the zoom peak; others: late in the step, where its actions' result is visible
|
|
654
682
|
const shots = segs.map((g) => ({ t: Math.min(g.o1 - 0.05, g.zoomPeak ?? g.o0 + (g.o1 - g.o0) * 0.8), label: `step ${g.i + 1} · ${g.o0.toFixed(1)}–${g.o1.toFixed(1)}s${g.step.zoom ? ' · zoom' : ''}`, caption: g.step.caption ?? '' }));
|
|
655
|
-
for (const c of cards) shots.push({ t: c.t0 + (c.t1 - c.t0) * 0.85, label: `card · ${c.t0.toFixed(1)}–${c.t1.toFixed(1)}s`, caption: c.card.title
|
|
683
|
+
for (const c of cards) shots.push({ t: c.t0 + (c.t1 - c.t0) * 0.85, label: `${c.scene ? `step ${c.i + 1} scene` : 'card'} · ${c.t0.toFixed(1)}–${c.t1.toFixed(1)}s`, caption: c.card.title || (c.scene ? path.basename(c.scene.file) : '') });
|
|
656
684
|
shots.sort((a, b) => a.t - b.t);
|
|
657
685
|
if (in0) shots.unshift({ t: INTRO * 0.8, label: `intro · 0.0–${INTRO.toFixed(1)}s`, caption: '' });
|
|
658
686
|
if (ec) shots.push({ t: VIDEO + END / 2, label: `end card · ${VIDEO.toFixed(1)}–${TOTAL.toFixed(1)}s`, caption: '' });
|
|
@@ -685,7 +713,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
685
713
|
.map(toOut).filter((t) => t != null && t > 0.2),
|
|
686
714
|
...segs.filter((g) => g.afterHidden).map((g) => g.o0),
|
|
687
715
|
...(in0 ? [INTRO - 0.2] : []),
|
|
688
|
-
...cards.flatMap((c) => [c.t0, c.t1 - 0.15]),
|
|
716
|
+
...cards.filter((c) => !c.scene).flatMap((c) => [c.t0, c.t1 - 0.15]),
|
|
689
717
|
...(ec ? [VIDEO - 0.25] : []),
|
|
690
718
|
]),
|
|
691
719
|
risers: segs.filter((g) => g.step.sfx === 'riser').map((g) => [g.o0, g.o1]),
|
|
@@ -694,6 +722,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
694
722
|
voice: [
|
|
695
723
|
...(in0?.say ? [{ text: in0.say, start: 0.15, where: 'intro' }] : []),
|
|
696
724
|
...cards.filter((c) => c.card.say).map((c) => ({ text: c.card.say, start: c.t0 + 0.15, where: `step ${c.i + 1} card` })),
|
|
725
|
+
...cards.filter((c) => c.scene && c.say).map((c) => ({ text: c.say, start: c.t0 + c.lead, where: `step ${c.i + 1} scene`, chunk: true })),
|
|
697
726
|
...segs.filter((g) => g.step.say).map((g) => ({ text: g.step.say, start: g.o0 + g.sayAt, where: `step ${g.i + 1}` })),
|
|
698
727
|
...(ec?.say ? [{ text: ec.say, start: VIDEO + 0.05, where: 'end card' }] : []),
|
|
699
728
|
],
|
|
@@ -716,5 +745,5 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
716
745
|
let cx = (x0 + x1) / 2, cy = (y0 + y1) / 2;
|
|
717
746
|
g.crop = { x: even(Math.max(0, Math.min(W - w, cx - w / 2))), y: even(Math.max(0, Math.min(H - h, cy - h / 2))), w: even(w), h: even(h) };
|
|
718
747
|
}
|
|
719
|
-
return { silent, clean, events, preview, warnings, captions, cards: cards.map((c) => ({ t: c.t0, title: [c.card.kicker, c.card.title].filter(Boolean).join(' · ') })), segs: segs.map((g) => ({ i: g.i, o0: g.o0, o1: g.o1, caption: g.step.caption, doc: g.step.doc ?? g.step.say, crop: g.crop })) };
|
|
748
|
+
return { silent, clean, events, preview, warnings, captions, cards: cards.filter((c) => !c.scene || c.card.title).map((c) => ({ t: c.t0, title: [c.card.kicker, c.card.title].filter(Boolean).join(' · ') })), segs: segs.map((g) => ({ i: g.i, o0: g.o0, o1: g.o1, caption: g.step.caption, doc: g.step.doc ?? g.step.say, crop: g.crop })) };
|
|
720
749
|
}
|
package/src/config.mjs
CHANGED
|
@@ -4,6 +4,7 @@ import path from 'node:path';
|
|
|
4
4
|
import YAML from 'yaml';
|
|
5
5
|
import { VOICE_DEFAULTS, kokoroVoiceFor, qwenVoiceFor } from './voice.mjs';
|
|
6
6
|
import { QWEN_VOICES, qwenSupported, qwenVoice } from './qwen-tts.mjs';
|
|
7
|
+
import { SCENE_DEFAULTS } from './scenes.mjs';
|
|
7
8
|
|
|
8
9
|
// Built-in labels drawn into the video, by `lang` (e.g. es, es-MX); other languages get English
|
|
9
10
|
export const LABELS = {
|
|
@@ -35,6 +36,8 @@ export const PURPOSES = {
|
|
|
35
36
|
tutorial: { duration: 'auto', music: { bpm: 96, volume: 0.35 }, sfx: 'clicks', highlight: true, stepNumbers: true, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true, hyperframes: true },
|
|
36
37
|
course: { duration: 'auto', music: false, sfx: 'clicks', highlight: true, stepNumbers: false, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true, hyperframes: true },
|
|
37
38
|
docs: { duration: 'auto', music: false, sfx: false, highlight: true, stepNumbers: true, subtitles: false, chapters: false, clips: ['gif', 'png'], frame: false, pace: 'normal', follow: false, hyperframes: false },
|
|
39
|
+
// a concept explained with animated scenes the agent writes (no web app to record)
|
|
40
|
+
explainer: { duration: 'auto', music: { style: 'calm', bpm: 64, volume: 0.3 }, sfx: false, highlight: false, stepNumbers: false, subtitles: true, chapters: true, clips: null, frame: false, pace: 'normal', follow: false, hyperframes: true },
|
|
38
41
|
};
|
|
39
42
|
|
|
40
43
|
// Where storyboards and their videos go, and how they're named: <site>-<kind>[-<topic>].yaml →
|
|
@@ -52,7 +55,7 @@ export const PACES = {
|
|
|
52
55
|
export const FRAME_STYLES = ['aurora', 'studio', 'dark', 'pastel', 'blur'];
|
|
53
56
|
|
|
54
57
|
export const VIDEO_DIR = 'videos';
|
|
55
|
-
export const KINDS = { pitch: 'promo', tutorial: 'tutorial', course: 'lesson', docs: 'docs' };
|
|
58
|
+
export const KINDS = { pitch: 'promo', tutorial: 'tutorial', course: 'lesson', docs: 'docs', explainer: 'explainer' };
|
|
56
59
|
// pxgenius.ai → pxgenius; an IP or a port is kept whole so two internal apps don't share a name
|
|
57
60
|
// (http://10.1.2.3:8080 → 10-1-2-3-8080, http://localhost:3000 → localhost-3000)
|
|
58
61
|
export const siteOf = (url) => {
|
|
@@ -70,7 +73,8 @@ export const certError = (e, url) => (/ERR_CERT|ERR_SSL/.test(e?.message ?? '')
|
|
|
70
73
|
const slug = (s) => String(s).toLowerCase().normalize('NFKD').replace(/[^\p{L}\p{N}]+/gu, '-').replace(/^-+|-+$/g, '').slice(0, 40);
|
|
71
74
|
|
|
72
75
|
export function defaultStoryboard(url, purpose = 'pitch', topic) {
|
|
73
|
-
|
|
76
|
+
// an explainer has no site: it is named by its topic
|
|
77
|
+
const base = (url ? [siteOf(url), KINDS[purpose] ?? purpose, topic && slug(topic)] : [topic && slug(topic), KINDS[purpose] ?? purpose]).filter(Boolean).join('-');
|
|
74
78
|
for (let i = 1; ; i++) {
|
|
75
79
|
const file = path.join(VIDEO_DIR, `${base}${i > 1 ? `-${i}` : ''}.yaml`);
|
|
76
80
|
if (!fs.existsSync(file)) return file;
|
|
@@ -86,8 +90,9 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
86
90
|
const raw = YAML.parse(fs.readFileSync(abs, 'utf8'));
|
|
87
91
|
const dir = path.dirname(abs);
|
|
88
92
|
const name = raw.name ?? path.basename(abs).replace(/\.ya?ml$/, '');
|
|
89
|
-
if (!raw.url) throw new Error('demo.yaml: `url` is required');
|
|
90
93
|
if (!Array.isArray(raw.steps) || !raw.steps.length) throw new Error('demo.yaml: `steps` must be a non-empty list');
|
|
94
|
+
// only a storyboard made entirely of scenes has nothing to record
|
|
95
|
+
if (!raw.url && !raw.steps.every((s) => s?.scene || s?.hidden)) throw new Error('demo.yaml: `url` is required (only a storyboard whose steps are all scenes can leave it out)');
|
|
91
96
|
|
|
92
97
|
const viewport = { width: 1600, height: 900, ...raw.viewport };
|
|
93
98
|
const output = { width: 1920, height: 1080, fps: 30, ...raw.output };
|
|
@@ -103,6 +108,17 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
103
108
|
if (!PACES[pick('pace')]) throw new Error(`pace must be one of ${Object.keys(PACES).join(', ')}`);
|
|
104
109
|
|
|
105
110
|
const steps = raw.steps.map((s, i) => {
|
|
111
|
+
// scene: an animation the agent wrote (see scenes.mjs), timed to the step's narration
|
|
112
|
+
let scene;
|
|
113
|
+
if (s.scene) {
|
|
114
|
+
const sc = typeof s.scene === 'string' ? { file: s.scene } : s.scene;
|
|
115
|
+
if (!sc.file) throw new Error(`step ${i + 1}: scene needs a file: \`scene: scenes/<name>.html\``);
|
|
116
|
+
if (s.actions?.length) throw new Error(`step ${i + 1}: a scene plays its own animation instead of the web app, so it has no actions — move them into a step of their own`);
|
|
117
|
+
if (!s.say && typeof s.duration !== 'number') throw new Error(`step ${i + 1}: a scene without a say: line needs a duration in seconds`);
|
|
118
|
+
scene = { ...SCENE_DEFAULTS, ...sc, file: path.resolve(dir, sc.file) };
|
|
119
|
+
const card = s.card ? { ...s.card, ...(s.card.image ? { imageFile: path.resolve(dir, s.card.image) } : {}) } : undefined;
|
|
120
|
+
return { ...s, actions: [], duration: typeof s.duration === 'number' ? s.duration : 'auto', card, scene };
|
|
121
|
+
}
|
|
106
122
|
const duration = s.duration ?? P.duration;
|
|
107
123
|
if (!s.hidden && duration !== 'auto' && !(duration > 0)) {
|
|
108
124
|
throw new Error(`step ${i + 1}: visible steps need a positive \`duration\` in seconds, or \`duration: auto\``);
|
|
@@ -136,6 +152,7 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
136
152
|
const frame = frame0 ? { background: null, window: 'browser', ...(typeof frame0 === 'object' ? frame0 : {}), style: styleOf(frame0) } : null;
|
|
137
153
|
if (frame && !FRAME_STYLES.includes(frame.style)) throw new Error(`frame style must be one of ${FRAME_STYLES.join(', ')}`);
|
|
138
154
|
if (frame && !['browser', 'app'].includes(frame.window)) throw new Error('frame.window must be browser or app');
|
|
155
|
+
if (!['auto', 'light', 'dark'].includes(raw.ground ?? 'auto')) throw new Error('ground must be light or dark (leave it out and auto reads it from the scenes, or from the frame)');
|
|
139
156
|
if (frame?.icon) frame.iconFile = path.resolve(dir, frame.icon);
|
|
140
157
|
|
|
141
158
|
// hyperframes: true | { template: lesson | launch, band, bandColors, emblem, hand, backdrop } — cards
|
|
@@ -156,6 +173,9 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
156
173
|
const byLang = { kokoro: kokoroVoice, qwen: qwenVoiceFor(raw) }[provider];
|
|
157
174
|
voice = { speed: pace0.voiceSpeed, volume: 1, pause: pace0.sentencePause, ...VOICE_DEFAULTS[provider], ...(byLang ? { voice: byLang } : {}), ...(raw.lang ? { lang: raw.lang } : {}), ...raw.voice, provider };
|
|
158
175
|
if (!(typeof voice.pause === 'number' && voice.pause >= 0 && voice.pause <= 3)) throw new Error('voice.pause is the shortest pause between the sentences of a line, in seconds, 0–3 (0 leaves the voice as it is)');
|
|
176
|
+
// explainers: a scene's narration keeps moving (no pause in a line longer than this)
|
|
177
|
+
if (voice.maxPause === undefined && purpose === 'explainer') voice.maxPause = 0.55;
|
|
178
|
+
if (voice.maxPause != null && !(typeof voice.maxPause === 'number' && voice.maxPause >= voice.pause)) throw new Error(`voice.maxPause is the longest pause inside a line, in seconds, at least voice.pause (${voice.pause})`);
|
|
159
179
|
if (provider === 'qwen') {
|
|
160
180
|
const v = qwenVoice(voice.voice);
|
|
161
181
|
if (!v) throw new Error(`voice: qwen has no voice "${voice.voice}" (one of ${QWEN_VOICES.join(', ')})`);
|
|
@@ -178,10 +198,22 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
178
198
|
const clips = [].concat(pick('clips') ?? []).filter(Boolean);
|
|
179
199
|
for (const c of clips) if (!['gif', 'webm', 'mp4', 'png'].includes(c)) throw new Error('clips must be gif, webm, mp4 or png (or a list of them)');
|
|
180
200
|
if (music?.file) music.file = path.resolve(dir, music.file);
|
|
201
|
+
if (music?.style && !['calm'].includes(music.style)) throw new Error('music.style must be calm (or leave it out for the upbeat synth)');
|
|
181
202
|
|
|
203
|
+
// scenes: { css: [files], js: [files] } shared by every scene (a palette, drawing helpers), loaded before each one
|
|
204
|
+
const sk = raw.scenes ?? {};
|
|
205
|
+
// `three: true` adds storyshot's own drawn-3D helpers (three.js + Kit3) before the storyboard's
|
|
206
|
+
// own kit files, for scenes whose subject is an object rather than a quantity (guide: "3D scenes").
|
|
207
|
+
if (sk.three != null && typeof sk.three !== 'boolean') throw new Error('scenes.three is true or false (it loads three.js and storyshot\'s Kit3 helpers into every scene)');
|
|
208
|
+
const sceneKit = {
|
|
209
|
+
css: [].concat(sk.css ?? []).map((f) => path.resolve(dir, f)),
|
|
210
|
+
js: [].concat(sk.js ?? []).map((f) => path.resolve(dir, f)),
|
|
211
|
+
three: sk.three === true,
|
|
212
|
+
};
|
|
182
213
|
return {
|
|
183
214
|
name,
|
|
184
215
|
purpose,
|
|
216
|
+
sceneKit,
|
|
185
217
|
highlight: pick('highlight'),
|
|
186
218
|
stepNumbers: pick('stepNumbers'),
|
|
187
219
|
subtitles: pick('subtitles'),
|
|
@@ -189,7 +221,7 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
189
221
|
clips,
|
|
190
222
|
pace: { name: pick('pace'), ...PACES[pick('pace')] },
|
|
191
223
|
maxWait: raw.maxWait ?? PACES[pick('pace')].maxWait,
|
|
192
|
-
url: raw.url,
|
|
224
|
+
url: raw.url ?? null,
|
|
193
225
|
viewport,
|
|
194
226
|
output,
|
|
195
227
|
scale,
|
|
@@ -209,6 +241,11 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
209
241
|
// e.g. when burned-in subtitles already carry the words
|
|
210
242
|
captions: raw.captions !== false,
|
|
211
243
|
cards: raw.cards ?? 'auto',
|
|
244
|
+
// ground: light | dark — is the film light or dark overall? It decides the colour of the burned-in
|
|
245
|
+
// subtitles (ink letters on a light film, white on a dark one) and the cards' ground. `auto` reads
|
|
246
|
+
// it from the scenes, or from the frame's backdrop; say it outright when the scenes take their
|
|
247
|
+
// ground from the shared scene kit, which `auto` cannot always see through.
|
|
248
|
+
ground: raw.ground ?? 'auto',
|
|
212
249
|
// language of the built-in labels ("Pause and think", "Introduction"): `lang`, else guessed from the captions
|
|
213
250
|
lang: raw.lang ?? (/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/.test(JSON.stringify(raw.steps ?? [])) ? 'zh' : 'en'),
|
|
214
251
|
labels: LABELS[String(raw.lang ?? '').toLowerCase().split(/[-_]/)[0]] ?? (/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/.test(JSON.stringify(raw.steps ?? [])) ? LABELS.zh : LABELS.en),
|
package/src/hyperframes.mjs
CHANGED
|
@@ -16,9 +16,11 @@ import path from 'node:path';
|
|
|
16
16
|
import { createRequire } from 'node:module';
|
|
17
17
|
import os from 'node:os';
|
|
18
18
|
import { fileURLToPath } from 'node:url';
|
|
19
|
+
import { writeSceneProject, sceneAssets, fileHash } from './scenes.mjs';
|
|
19
20
|
|
|
20
21
|
export const HF_VERSION = '0.8.217';
|
|
21
22
|
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
23
|
+
const KIT3D = path.join(HERE, 'scene-kit', 'kit3d.js');
|
|
22
24
|
// families HyperFrames resolves by name; any other needs an @font-face
|
|
23
25
|
const BUNDLED = ['Inter', 'Montserrat', 'Outfit', 'Nunito', 'Oswald', 'League Gothic', 'Archivo Black', 'Space Mono', 'IBM Plex Mono', 'JetBrains Mono',
|
|
24
26
|
'EB Garamond', 'Playfair Display', 'Source Code Pro', 'Noto Sans JP', 'Roboto', 'Open Sans', 'Lato', 'Poppins'];
|
|
@@ -66,6 +68,22 @@ const run = (cli, args) => new Promise((resolve, reject) => {
|
|
|
66
68
|
p.on('close', (code) => (code === 0 ? resolve(out) : reject(new Error(out.trim().split('\n').filter(Boolean).slice(-3).join(' | ') || `exit ${code}`))));
|
|
67
69
|
});
|
|
68
70
|
|
|
71
|
+
// Run the HyperFrames CLI (installing the pinned version on first use); resolves with its exit code and output.
|
|
72
|
+
export async function hyperframesRun(args, { log = console.log } = {}) {
|
|
73
|
+
const c = hyperframesCli();
|
|
74
|
+
if (c.error) throw new Error(`HyperFrames: ${c.error}`);
|
|
75
|
+
const cli = await ensureCli(log);
|
|
76
|
+
return new Promise((resolve, reject) => {
|
|
77
|
+
const p = spawn(cli.cmd, [...cli.args, ...args], { env: { ...process.env, HYPERFRAMES_NO_TELEMETRY: '1', DO_NOT_TRACK: '1' }, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
78
|
+
let out = '', err = '';
|
|
79
|
+
p.stdout.on('data', (d) => { out += d; });
|
|
80
|
+
p.stderr.on('data', (d) => { err += d; });
|
|
81
|
+
p.on('error', reject);
|
|
82
|
+
p.on('close', (code) => resolve({ code, out, err }));
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
export const sceneFontFaces = (font) => fontFaces(font);
|
|
86
|
+
|
|
69
87
|
// --- narration cues --------------------------------------------------------------------------------
|
|
70
88
|
// When the narration names each item (a bullet, the call to action): the item's earliest content word
|
|
71
89
|
// found in the line after the previous item's, placed by its character position (speech runs at a
|
|
@@ -574,7 +592,10 @@ blurIn('#try', ${n(ctaAt + 0.85)}, { d: 0.25 });` : ''}
|
|
|
574
592
|
// --- rendering ---------------------------------------------------------------------------------------
|
|
575
593
|
const hashOf = (...parts) => {
|
|
576
594
|
const h = crypto.createHash('sha1');
|
|
577
|
-
|
|
595
|
+
// JSON.stringify(undefined) is undefined, not a string, and Hash.update throws on it. A scene step
|
|
596
|
+
// with a `duration:` and no `say:` line is legal and hashes `job.say` as undefined, so keep a name
|
|
597
|
+
// for the empty ones rather than crashing the whole render on them.
|
|
598
|
+
for (const p of parts) h.update(typeof p === 'string' ? p : JSON.stringify(p) ?? String(p));
|
|
578
599
|
return h.digest('hex').slice(0, 16);
|
|
579
600
|
};
|
|
580
601
|
|
|
@@ -582,16 +603,40 @@ const hashOf = (...parts) => {
|
|
|
582
603
|
// backdrop?, page?, logo?, logoSize?, logoTarget?, image?, alpha? }]
|
|
583
604
|
// ctx: { W, H, FPS, dark, accent, ink, font, lang, labels, opts, frame: {FX,FY,FW,FH,frameBg,chrome,url}, work, subs (burned-in subtitles) }
|
|
584
605
|
// Returns { [name]: { dir, ext, alpha } } for the cards that rendered; the rest fall back.
|
|
606
|
+
// Scene jobs ({ kind: 'scene', scene, say, lead, cues }) have no fallback: why one failed is in out._errors.
|
|
585
607
|
export async function renderHyperframesCards(jobs, ctx, { log = console.log, warn = log } = {}) {
|
|
586
|
-
|
|
608
|
+
const errors = {};
|
|
609
|
+
const out = {};
|
|
610
|
+
Object.defineProperty(out, '_errors', { value: errors, enumerable: false });
|
|
611
|
+
if (!jobs.length) return out;
|
|
587
612
|
let cli = hyperframesCli();
|
|
588
|
-
if (cli.error) {
|
|
589
|
-
|
|
613
|
+
if (cli.error) {
|
|
614
|
+
for (const j of jobs) if (j.kind === 'scene') errors[j.name] = cli.error;
|
|
615
|
+
warn(`hyperframes: ${cli.error} — using the built-in cards`);
|
|
616
|
+
return out;
|
|
617
|
+
}
|
|
618
|
+
const template = ctx.opts?.template;
|
|
590
619
|
const root = path.join(ctx.work, 'hf');
|
|
591
620
|
fs.mkdirSync(root, { recursive: true });
|
|
592
|
-
const out = {};
|
|
593
621
|
const todo = [];
|
|
594
622
|
for (const job of jobs) {
|
|
623
|
+
if (job.kind === 'scene') {
|
|
624
|
+
// the agent's composition, wrapped in a project as long as the scene (see scenes.mjs)
|
|
625
|
+
const sctx = { ...ctx, fontFaces: fontFaces(ctx.font) };
|
|
626
|
+
const files = sceneAssets(job.scene.file);
|
|
627
|
+
const key = hashOf(HF_VERSION, ctx.FPS, job.seconds.toFixed(3), job.lead, job.cues, job.say, ctx.W, ctx.H, ctx.accent, ctx.ink, ctx.font, ctx.lang, ctx.subs,
|
|
628
|
+
fs.existsSync(job.scene.file) ? fileHash(job.scene.file) : 'missing', ...Object.entries(files).map(([k, f]) => [k, fileHash(f)]),
|
|
629
|
+
// the shipped 3D helpers are part of the picture too: upgrading storyshot has to invalidate
|
|
630
|
+
// the cached frames of every 3D scene, or a fixed engine renders the old frames back
|
|
631
|
+
job.kit?.three ? ['kit3d', fileHash(KIT3D)] : 'no-three',
|
|
632
|
+
...[...(job.kit?.css ?? []), ...(job.kit?.js ?? [])].map((f) => [f, fs.existsSync(f) ? fileHash(f) : 'missing']));
|
|
633
|
+
const dir = path.join(root, `${job.name}-${key}`), frames = path.join(dir, 'frames');
|
|
634
|
+
if (fs.existsSync(path.join(dir, 'done'))) { out[job.name] = { dir: frames, ext: 'jpg', alpha: false }; continue; }
|
|
635
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
636
|
+
try { writeSceneProject(job, sctx, dir); } catch (e) { errors[job.name] = e.message; continue; }
|
|
637
|
+
todo.push({ job, dir, frames, ext: 'jpg', alpha: false });
|
|
638
|
+
continue;
|
|
639
|
+
}
|
|
595
640
|
const launchCard = template === 'launch' && job.kind !== 'section';
|
|
596
641
|
if (launchCard) {
|
|
597
642
|
try { await import('roughjs'); } catch { warn('hyperframes: the launch template needs the roughjs package (npm i roughjs) — using the built-in cards'); return {}; }
|
|
@@ -627,10 +672,12 @@ export async function renderHyperframesCards(jobs, ctx, { log = console.log, war
|
|
|
627
672
|
if (!todo.length) return out;
|
|
628
673
|
|
|
629
674
|
try { cli = await ensureCli(log); } catch (e) {
|
|
675
|
+
for (const t of todo) if (t.job.kind === 'scene') errors[t.job.name] = `could not install the HyperFrames CLI: ${String(e.message).slice(0, 200)}`;
|
|
630
676
|
warn(`hyperframes: could not install the CLI (${String(e.message).slice(0, 200)}) — using the built-in cards`);
|
|
631
677
|
return out;
|
|
632
678
|
}
|
|
633
|
-
|
|
679
|
+
const nScenes = todo.filter((t) => t.job.kind === 'scene').length, nCards = todo.length - nScenes;
|
|
680
|
+
log(`hyperframes: rendering ${[nScenes && `${nScenes} scene${nScenes > 1 ? 's' : ''}`, nCards && `${nCards} card${nCards > 1 ? 's' : ''}`].filter(Boolean).join(' and ')} (${cli.how})`);
|
|
634
681
|
const one = async (t) => {
|
|
635
682
|
const t0 = Date.now();
|
|
636
683
|
try {
|
|
@@ -649,9 +696,10 @@ export async function renderHyperframesCards(jobs, ctx, { log = console.log, war
|
|
|
649
696
|
}
|
|
650
697
|
fs.writeFileSync(path.join(t.dir, 'done'), '');
|
|
651
698
|
out[t.job.name] = { dir: t.frames, ext: t.ext, alpha: t.alpha };
|
|
652
|
-
log(` ${t.job.name}: ${t.job.seconds.toFixed(1)}s card in ${((Date.now() - t0) / 1000).toFixed(0)}s`);
|
|
699
|
+
log(` ${t.job.kind === 'scene' ? `step ${t.job.step + 1} scene` : t.job.name}: ${t.job.seconds.toFixed(1)}s ${t.job.kind === 'scene' ? path.basename(t.job.scene.file) : 'card'} in ${((Date.now() - t0) / 1000).toFixed(0)}s`);
|
|
653
700
|
} catch (e) {
|
|
654
|
-
|
|
701
|
+
if (t.job.kind === 'scene') errors[t.job.name] = String(e.message).slice(0, 600);
|
|
702
|
+
else warn(`hyperframes: the ${t.job.name} card did not render (${String(e.message).slice(0, 300)}) — using the built-in one`);
|
|
655
703
|
}
|
|
656
704
|
};
|
|
657
705
|
// two renders at a time (each one drives its own browser)
|