storyshot 0.6.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -26
- package/package.json +1 -2
- package/skills/storyshot/SKILL.md +93 -14
- package/src/audio.mjs +9 -4
- package/src/check.mjs +43 -3
- package/src/compose.mjs +33 -9
- package/src/config.mjs +15 -12
- package/src/doctor.mjs +2 -2
- package/src/hyperframes.mjs +155 -35
- package/src/init.mjs +11 -2
- package/src/inspect.mjs +63 -19
- package/src/qwen-tts.mjs +40 -15
- package/src/qwen_tts.py +144 -13
- package/src/record.mjs +208 -64
- package/src/voice.mjs +107 -11
package/README.md
CHANGED
|
@@ -153,7 +153,7 @@ Rendering is free and runs locally. Only the voiceover can cost money:
|
|
|
153
153
|
|---|---|---|
|
|
154
154
|
| none | free | captions only |
|
|
155
155
|
| `kokoro` | free | local, English and Mandarin Chinese (`zf_001`); downloads a ~90 MB model on first use (Chinese: ~130 MB, and needs Python 3.10–3.12) |
|
|
156
|
-
| `qwen` | free | local, Apple Silicon Macs only; the most natural free Chinese voice (`dylan`, `serena`);
|
|
156
|
+
| `qwen` | free | local, Apple Silicon Macs only; the most natural free Chinese voice (`dylan`, `serena`); every take is checked by a speech recognizer and made again if it comes out wrong; downloads ~3.3 GB of models and needs Python 3.10–3.12 |
|
|
157
157
|
| ElevenLabs | your credits | the most natural voice; many languages including Chinese. Put `ELEVENLABS_API_KEY` in `.env`. |
|
|
158
158
|
| OpenAI | your credits | `OPENAI_API_KEY` |
|
|
159
159
|
|
|
@@ -191,7 +191,7 @@ from `storyshot guide`.
|
|
|
191
191
|
| `inspect_page` | Verified click targets and a screenshot, for a URL or for the state after step N of a storyboard. Reports login walls. |
|
|
192
192
|
| `login` | Opens a browser window where the **user** logs in, and saves the session. |
|
|
193
193
|
| `init_storyboard` | Writes a starter storyboard with verified targets. |
|
|
194
|
-
| `check_storyboard` | Validates in about a second: typos with "did you mean", action and target shapes, `TODO`s, keys, the session file. |
|
|
194
|
+
| `check_storyboard` | Validates in about a second: typos with "did you mean", action and target shapes, `TODO`s, keys, the session file, and narration that sounds machine-written ("not just X, but Y", "seamless", 赋能…) or runs on without a breath. |
|
|
195
195
|
| `render_video` / `render_status` | Renders in the background and returns the MP4 path, warnings, a per-step speed log and the preview image. If the render isn't done within 45 s, it returns a `job_id`, so client tool timeouts never kill a render. |
|
|
196
196
|
|
|
197
197
|
Any other MCP client: run `npx -y storyshot mcp` (stdio), and point the agent at the playbook (`storyshot guide`). When run from a git
|
|
@@ -211,7 +211,7 @@ Everything the agent does is also a command:
|
|
|
211
211
|
| `storyshot init <url> [--purpose tutorial] [--topic export-pdf] [--storage videos/<site>-login.json]` | Write `videos/<site>-<kind>[-<topic>].yaml` with verified targets; the video gets the same name. |
|
|
212
212
|
| `storyshot login <url> [--save videos/<site>-login.json]` | Log in once in a browser window and save the session. |
|
|
213
213
|
| `storyshot inspect <url> [--storage auth.json]` | List verified targets and save screenshots. |
|
|
214
|
-
| `storyshot inspect videos/acme-promo.yaml --after 3` | The same, for the page after steps 1–3. |
|
|
214
|
+
| `storyshot inspect videos/acme-promo.yaml --after 3` | The same, for the page after steps 1–3. What step 3 opened is marked ★ and listed first. |
|
|
215
215
|
| `storyshot check videos/acme-promo.yaml` | Validate without opening a browser. |
|
|
216
216
|
| `storyshot render videos/acme-promo.yaml [--draft]` | Record, then compose the video and soundtrack. |
|
|
217
217
|
| `storyshot compose videos/acme-promo.yaml [--draft]` | Rebuild from the last recording, in seconds. |
|
|
@@ -248,7 +248,7 @@ viewport: { width: 1600, height: 900 } # CSS px the site is laid out at
|
|
|
248
248
|
output: { width: 1920, height: 1080, fps: 30 } # same aspect ratio; use 1080x1920 with a 900x1600 viewport for vertical
|
|
249
249
|
theme: { accent: "#d9f75b", ink: "#0f172a", font: Inter }
|
|
250
250
|
music: { bpm: 120, volume: 1 } # or { file: track.mp3 }, or false
|
|
251
|
-
sfx: true # click / whoosh / riser sounds; clicks = no whooshes on cards and page changes (tutorial and course default); false = none
|
|
251
|
+
sfx: true # click / whoosh / riser sounds; clicks = no whooshes on cards and page changes, and no rising sound under a riser step, only its chime (tutorial and course default); false = none
|
|
252
252
|
voice: { provider: elevenlabs, voice: hpp4J3VqNfWAUOO0d1Us } # optional voiceover, see below
|
|
253
253
|
storageState: auth.json # optional Playwright login state
|
|
254
254
|
ignoreHTTPSErrors: true # internal site with a self-signed HTTPS certificate (plain http:// needs nothing)
|
|
@@ -287,7 +287,7 @@ steps:
|
|
|
287
287
|
- click: { role: button, name: Try it }
|
|
288
288
|
- caption: AI does the work
|
|
289
289
|
duration: 1.3
|
|
290
|
-
sfx: riser # rising sound + chime (or `chime`)
|
|
290
|
+
sfx: riser # rising sound + chime (or `chime`); a tutorial or lesson plays only the chime
|
|
291
291
|
actions:
|
|
292
292
|
- waitFor: { text: Done, timeout: 120000 } # waiting is compressed automatically
|
|
293
293
|
- card: { kicker: Part 2, title: Read the results } # a section card before this step (tutorials); becomes a chapter
|
|
@@ -326,21 +326,21 @@ You can add `exact: true`, `last: true` or `nth: 2` to any of them.
|
|
|
326
326
|
| `move` / `hover` | `move: { text: "$0" }` or `move: { x: 0.5, y: 0.4, ms: 1200 }` |
|
|
327
327
|
| `type` | `type: { target: { label: Email }, text: hi@acme.com, delay: 60 }`, or `type: hello` to type into whatever has focus. `clear: true` empties a pre-filled field first |
|
|
328
328
|
| `drag` | `drag: { from: { css: ".slider" }, by: [{ x: -300 }, { x: 450 }] }` (several moves while held), or `to: <target or point>` |
|
|
329
|
-
| `press` | `press: Enter`, `press: "^"`, `press: { key: Meta+K, show: true }`. When highlighting, shortcuts and special keys show an on-screen badge; plain characters don't. `show: false` hides it. |
|
|
329
|
+
| `press` | `press: Enter`, `press: "^"`, `press: { key: Meta+K, show: true }`. When highlighting, shortcuts and special keys show an on-screen badge next to the cursor; plain characters don't. `show: false` hides it. |
|
|
330
330
|
| `pause` | `pause: 700` (ms) |
|
|
331
331
|
| `waitFor` | `waitFor: { role: button, name: Done, timeout: 120000 }` |
|
|
332
332
|
| `waitForUrl` | `waitForUrl: "**/dashboard"` |
|
|
333
333
|
| `networkIdle` | `networkIdle: true` |
|
|
334
334
|
| `goto` | `goto: /pricing` |
|
|
335
|
-
| `scrollTo` | `scrollTo: { css: "#pricing", ms: 1200 }` |
|
|
335
|
+
| `scrollTo` | `scrollTo: { css: "#pricing", ms: 1200 }` scrolls the target to the top of the window, below a sticky header |
|
|
336
336
|
| `scroll` | `scroll: 600` |
|
|
337
337
|
| `dismiss` | `dismiss: { role: button, name: Close }` (clicks only if visible) |
|
|
338
338
|
| `hide` | `hide: '[id*="QSI"], .chat-widget'` hides pop-ups, survey prompts, feedback tabs and chat widgets for the whole video, including ones that appear late (use it in `setup`) |
|
|
339
|
-
| `mark` | `mark: "automatic zoom, captions"` sweeps a highlighter over that phrase, even inside a paragraph (scrolling to it if
|
|
339
|
+
| `mark` | `mark: "automatic zoom, captions"` sweeps a highlighter over that phrase, even inside a paragraph (scrolling to it only if it's off screen, and below a sticky header). `mark: { css: "table tr:nth-child(3)" }` marks any target with a box. Options: `ms` (sweep time, default 700), `color` (default: theme accent), `box: true/false`. Only text the viewer can see counts: a copy that is hidden, transparent, collapsed or covered is skipped. Marks stay until the step ends, and a mark whose text scrolls out of view (out of its panel, under a header) disappears with it. |
|
|
340
340
|
| `unmark` | `unmark: true` clears the marks (and notes) earlier |
|
|
341
|
-
| `note` | `note: { target: { label: Database }, text: Curated proteins only }` puts a short label with an arrow next to the element, on the side with the most room (or `side: left/right/top/bottom`). It stays until the step ends and gets reading time (about 3 words a second, or `ms`). |
|
|
341
|
+
| `note` | `note: { target: { label: Database }, text: Curated proteins only }` puts a short label with an arrow next to the element, on the side with the most room (or `side: left/right/top/bottom`). It stays until the step ends and gets reading time (about 3 words or 5 Chinese/Japanese/Korean characters a second, or `ms`). |
|
|
342
342
|
| `paste` | `paste: { target: { label: JSON }, text: '{"a": 1}' }` pastes text in one go through the real clipboard (with a ⌘V badge), so editors react as to a real paste; `paste: "..."` pastes into whatever has focus |
|
|
343
|
-
| `slide` | `slide: { target: { label: Quality }, to: 0.2 }` drags a slider to a position along its track (0 = start, 1 = end), found from the element, not pixels |
|
|
343
|
+
| `slide` | `slide: { target: { label: Quality }, to: 0.2 }` drags a slider to a position along its track (0 = start, 1 = end), found from the element, not pixels. `value: 3` drags it to that value instead, when the slider reports its range (an `<input type=range>`, or an ARIA slider such as Desmos's; `inspect` shows the range). The target can be the slider or its knob |
|
|
344
344
|
| `select` | `select: { target: { label: Database }, option: "UniProtKB/Swiss-Prot" }` picks an option in a dropdown (by its visible text, or part of it). `inspect` lists each dropdown's options. |
|
|
345
345
|
|
|
346
346
|
`ms` sets how long the cursor takes to travel to the target (`dragMs` for each
|
|
@@ -375,7 +375,7 @@ In-page anchor links (`href="#..."`) scroll smoothly instead of jumping.
|
|
|
375
375
|
|---|---|
|
|
376
376
|
| `target` | What to center on: a target, or a point `{ x: 0.5, y: 0.3 }`. If a step action used the same target, its position at that moment is used (so you can zoom on a button that disappears after the click). Otherwise it is measured at the start of the step when the zoom begins in the first half, or at the end of the step otherwise. |
|
|
377
377
|
| `scale` | Zoom factor (default 1.3). 1.2–1.5 works well. |
|
|
378
|
-
| `start`, `end` | Seconds from the start of the step. The camera eases in from 1× at `start` and is back at 1× at `end` (default: the step's end). `end`
|
|
378
|
+
| `start`, `end` | Seconds from the start of the step. The camera eases in from 1× at `start` and is back at 1× at `end` (default: the step's end). A zoom belongs to its step: an `end` past the step's end stops there, with a warning. |
|
|
379
379
|
| `ramp` | Ease-in / ease-out seconds: `0.5` or `[in, out]`. |
|
|
380
380
|
| `offset` | Moves the camera center from the target's center, in viewport CSS px. `{ y: -120 }` centers 120 px above the target. |
|
|
381
381
|
|
|
@@ -386,8 +386,10 @@ rather than cut off.
|
|
|
386
386
|
### Voiceover
|
|
387
387
|
|
|
388
388
|
Add a `voice` block, then give steps a `say:` line (and optionally
|
|
389
|
-
`end_card.say`). Each line starts as its step begins (`sayAt`
|
|
390
|
-
|
|
389
|
+
`end_card.say`). Each line starts as its step begins (`sayAt: 0.5` starts it
|
|
390
|
+
half a second in; `sayAt: after` starts it, and the step's caption, once the
|
|
391
|
+
step's actions are done, for a line about their result) and may run until the
|
|
392
|
+
next line. A line that is up to 20% too long is sped up slightly.
|
|
391
393
|
A line longer than that is queued after the previous line, and you get a
|
|
392
394
|
warning telling you to shorten it. Music ducks under the voice, and the final
|
|
393
395
|
mix is normalized to -14 LUFS.
|
|
@@ -398,7 +400,7 @@ mix is normalized to -14 LUFS.
|
|
|
398
400
|
| `qwen` | none | Local and free, Apple Silicon Macs only: Qwen3-TTS, the most natural free Chinese voice here (`dylan`, `serena`, see below). |
|
|
399
401
|
| `elevenlabs` | `ELEVENLABS_API_KEY` | Most natural, many languages including Chinese. `voice` is a voice ID; `model` defaults to `eleven_multilingual_v2`. The key needs Text to Speech access, plus Voices → Read to list voices. |
|
|
400
402
|
| `openai` | `OPENAI_API_KEY` | `voice: alloy`, `model: gpt-4o-mini-tts`; optional `instructions` for tone. |
|
|
401
|
-
| `say` | none | macOS built-in voices, robotic;
|
|
403
|
+
| `say` | none | macOS built-in voices, robotic; for drafts only: Apple's license allows them for personal, non-commercial use, and not in anything you publish or share. For Chinese, use `qwen` (Apple Silicon) or `kokoro` with `zf_001` rather than `Tingting`. |
|
|
402
404
|
|
|
403
405
|
**Chinese narration.** `voice: { provider: kokoro, voice: zf_001 }` speaks Mandarin with Kokoro
|
|
404
406
|
v1.1-zh, free and local, and is what `kokoro` uses when the `say:` lines are Chinese and no voice is
|
|
@@ -419,14 +421,22 @@ for Chinese tutorials, lessons and docs), `serena` (female, warm; the default fo
|
|
|
419
421
|
`vivian` (female, bright), `uncle_fu` (male, mellow), `eric` (male, Sichuan accent), and `ryan`, `aiden` (English), `ono_anna` (Japanese), `sohee` (Korean). On
|
|
420
422
|
first use Storyshot creates a venv in `~/.cache/storyshot/qwen-venv` (Python 3.10–3.12, about 400 MB)
|
|
421
423
|
and downloads the ~2.3 GB model into the Hugging Face cache. It speaks at about 2x real time on an M4 Pro
|
|
422
|
-
and peaks at about 8 GB of memory. The model samples its speech, so a
|
|
423
|
-
invented words or stops short. Storyshot
|
|
424
|
-
|
|
425
|
-
|
|
424
|
+
and peaks at about 8 GB of memory. The model samples its speech, so a take now and then garbles a few
|
|
425
|
+
words, drops one, runs on into invented words or stops short. Storyshot listens to every take with
|
|
426
|
+
[Qwen3-ASR 0.6B](https://huggingface.co/Qwen/Qwen3-ASR-0.6B) (Apache-2.0, a ~1 GB model downloaded on
|
|
427
|
+
first use) and compares what it hears with the line: Chinese by sound, so a homophone isn't a mistake, and
|
|
428
|
+
numbers digit by digit. A take that comes out wrong, or far too long or short, is made again with another
|
|
429
|
+
seed (up to 4 tries), and Storyshot warns, quoting what it heard, if a line still doesn't come out right.
|
|
430
|
+
This adds about 0.2 s per line. A scientific number like `2e-105` is spelled out in Chinese, because it sets the
|
|
431
|
+
model babbling. Lines in Latin letters are read in the storyboard's `lang` when Qwen3-TTS knows it
|
|
432
|
+
(`en`, `es`, `fr`, `de`, `it`, `pt`), so a Spanish lesson with `lang: es` gets a Spanish reading, checked
|
|
433
|
+
by ear in Spanish. On other machines the same storyboard renders with `kokoro` (`zf_001`, or `draftVoice`),
|
|
426
434
|
with a note.
|
|
427
435
|
|
|
428
|
-
Other fields: `speed` (default 1), `volume` (default 1),
|
|
429
|
-
|
|
436
|
+
Other fields: `speed` (default 1), `volume` (default 1), `pause` (the shortest pause between the
|
|
437
|
+
sentences of a line, in seconds: 0.3 for promos, 0.6 for tutorials and lessons; a voice's own longer pause
|
|
438
|
+
stays, and `0` leaves the voice as it is),
|
|
439
|
+
and `pronounce`, a map of word → spoken form used only for TTS, e.g.
|
|
430
440
|
`pronounce: { pxGenius: P X Genius, PPTX: P P T X }`. Subtitles keep the real
|
|
431
441
|
spelling. Synthesized lines
|
|
432
442
|
are cached in `.storyshot/tts-cache/`, so `compose` only pays for lines you
|
|
@@ -447,7 +457,7 @@ usually shouldn't.
|
|
|
447
457
|
|
|
448
458
|
| field | meaning |
|
|
449
459
|
|---|---|
|
|
450
|
-
| `logo` | A target on the page (captured as an image, e.g. `{ alt: Acme logo }`) or a path to
|
|
460
|
+
| `logo` | A target on the page (captured as an image, e.g. `{ alt: Acme logo }`; an inline SVG logo is `{ css: "header a svg" }` or similar) or a path to a PNG, JPG or SVG file. With a `title` too, the title shows under the logo; if the logo already contains the wordmark, leave out `title`. |
|
|
451
461
|
| `title` | Large product name next to the logo. |
|
|
452
462
|
| `kicker` | Small label above the title, e.g. `Part 2`. |
|
|
453
463
|
| `subtitle` | One line; `**bold**` gets an accent highlight. |
|
|
@@ -457,13 +467,14 @@ usually shouldn't.
|
|
|
457
467
|
| `cta` | Dark call-to-action pill, e.g. `Start free at acme.com`. |
|
|
458
468
|
| `duration` | Seconds. Default: fits the `say` line, else 1.0 (end card) or 2.2 (intro). |
|
|
459
469
|
| `say` | Voiceover line played over the end card. |
|
|
460
|
-
| `emblem` | With `hyperframes`: one to three short monospace lines over the title (an alignment, a formula, a line of code); `false` for none. |
|
|
470
|
+
| `emblem` | With `hyperframes`: one to three short monospace lines over the title (an alignment, a formula, a line of code); `false` for none. On HyperFrames cards, text too long for the frame (a title, subtitle, emblem or call to action) wraps and then shrinks until it fits. |
|
|
461
471
|
|
|
462
|
-
### HyperFrames cards
|
|
472
|
+
### HyperFrames cards
|
|
463
473
|
|
|
464
|
-
|
|
474
|
+
Tutorials and lessons render their intro, section and end cards with
|
|
465
475
|
[HyperFrames](https://hyperframes.heygen.com) (HeyGen's open-source HTML-to-video
|
|
466
|
-
renderer, Apache-2.0) for more motion
|
|
476
|
+
renderer, Apache-2.0) for more motion; `hyperframes: false` keeps the built-in
|
|
477
|
+
cards, and `hyperframes: true` turns them on for a promo. Two templates, picked by `purpose`:
|
|
467
478
|
|
|
468
479
|
- **lesson** (tutorials, courses, docs): the title lands word by word over a
|
|
469
480
|
slowly drifting, blurred view of the page the section leads into; section
|
|
@@ -485,7 +496,7 @@ hyperframes:
|
|
|
485
496
|
backdrop: false # lesson: plain background instead of the blurred page
|
|
486
497
|
```
|
|
487
498
|
|
|
488
|
-
It needs Node 22 or newer. On first use storyshot installs the pinned
|
|
499
|
+
It needs Node 22 or newer; on older Node, tutorials and lessons use the built-in cards. On first use storyshot installs the pinned
|
|
489
500
|
HyperFrames version into `~/.cache/storyshot/hyperframes/` (a local install
|
|
490
501
|
of the `hyperframes` package wins); its telemetry is switched off. Cards are
|
|
491
502
|
cached, so `compose` only re-renders the ones that changed (about 10–25 s
|
|
@@ -493,6 +504,29 @@ each). Any card that can't be rendered falls back to the built-in one with a
|
|
|
493
504
|
warning. The Virgil font is © Ellinor Rapp, SIL Open Font License 1.1
|
|
494
505
|
(`src/fonts/Virgil-LICENSE.md`).
|
|
495
506
|
|
|
507
|
+
## Licenses of what Storyshot uses
|
|
508
|
+
|
|
509
|
+
Storyshot is MIT. What it installs and downloads:
|
|
510
|
+
|
|
511
|
+
- **npm packages** are MIT, BSD, ISC, Apache-2.0 or BlueOak, with two exceptions, both under the optional
|
|
512
|
+
`kokoro-js` (the `kokoro` voice):
|
|
513
|
+
- `phonemizer` says Apache-2.0, but it bundles [eSpeak NG](https://github.com/espeak-ng/espeak-ng),
|
|
514
|
+
which is GPL-3.0-or-later, compiled to WebAssembly. If you ship Storyshot inside a closed-source
|
|
515
|
+
product, account for that, or install without it (`npm install --omit=optional`) and narrate with
|
|
516
|
+
`qwen` or a cloud voice.
|
|
517
|
+
- `sharp` loads libvips (LGPL-3.0) as a separate library.
|
|
518
|
+
- **Downloaded on first use:** Chromium (through Playwright), HyperFrames (Apache-2.0), the voice and
|
|
519
|
+
speech models (Kokoro-82M, Qwen3-TTS, Qwen3-ASR: all Apache-2.0) and their Python packages
|
|
520
|
+
(mlx-audio: MIT, pypinyin: MIT). ffmpeg is your own install; Storyshot runs it as a separate program.
|
|
521
|
+
- **The cards** animate with the browser's own Web Animations API. The Virgil font is under the SIL Open
|
|
522
|
+
Font License 1.1. The music and sound effects are synthesized by Storyshot.
|
|
523
|
+
- **Your videos:** narration made with `kokoro` or `qwen` comes from Apache-2.0 models, so you can use it
|
|
524
|
+
in commercial work. The macOS `say` voices are for drafts only: Apple's license allows them for
|
|
525
|
+
personal, non-commercial use, and not in anything published or shared. ElevenLabs and OpenAI voices
|
|
526
|
+
follow those services' terms.
|
|
527
|
+
|
|
528
|
+
This is a summary, not legal advice.
|
|
529
|
+
|
|
496
530
|
## License
|
|
497
531
|
|
|
498
532
|
MIT
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "storyshot",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.2",
|
|
4
4
|
"description": "Turn a short YAML storyboard into a polished product demo video of your web app — smooth cursor, auto zoom, captions, music.",
|
|
5
5
|
"author": "boheling <boheling@gmail.com>",
|
|
6
6
|
"repository": {
|
|
@@ -39,7 +39,6 @@
|
|
|
39
39
|
"zod": "^4.6.5"
|
|
40
40
|
},
|
|
41
41
|
"optionalDependencies": {
|
|
42
|
-
"gsap": "^3.14.2",
|
|
43
42
|
"kokoro-js": "^1.2.1",
|
|
44
43
|
"roughjs": "^4.6.6"
|
|
45
44
|
}
|
|
@@ -92,9 +92,15 @@ frames or stock footage presented as a recording of their app.
|
|
|
92
92
|
best free Chinese voice is `voice: { provider: qwen }` (Qwen3-TTS: `dylan`,
|
|
93
93
|
a male teacher's voice, for tutorials and lessons; `serena`, female, for
|
|
94
94
|
promos; those are the defaults): clearer than
|
|
95
|
-
zf_001 with English words and letters, about 2x real time,
|
|
96
|
-
on first use.
|
|
97
|
-
|
|
95
|
+
zf_001 with English words and letters, about 2x real time, ~3.3 GB of
|
|
96
|
+
models on first use. Storyshot listens to each take and makes it again if
|
|
97
|
+
it comes out garbled; a `⚠ qwen: … sounds like "…"` warning means a line
|
|
98
|
+
still came out wrong after 4 tries: rephrase it (spell out a number, say
|
|
99
|
+
`一格` for `1 格`). Elsewhere a `qwen` storyboard renders with kokoro.
|
|
100
|
+
`qwen` also reads Spanish, French, German, Italian and Portuguese: set the
|
|
101
|
+
storyboard's `lang` (`lang: es`). For other languages, use ElevenLabs.
|
|
102
|
+
Never publish a video narrated with macOS `say`: Apple's license allows its
|
|
103
|
+
voices only for personal, non-commercial use, so keep it for drafts.
|
|
98
104
|
3. **Login.** If the flow needs a logged-in page:
|
|
99
105
|
- Tell the user a browser window will open and that they should log in
|
|
100
106
|
there, preferably with a demo account (the video shows whatever is on
|
|
@@ -124,7 +130,13 @@ frames or stock footage presented as a recording of their app.
|
|
|
124
130
|
3. For states you can only reach by clicking (an editor after an upload, an
|
|
125
131
|
open menu, a modal), write the steps up to that point, then call
|
|
126
132
|
`inspect_page` with `storyboard` + `after_step: N`. It runs steps 1..N fast
|
|
127
|
-
(about 10 s) and lists the targets on the page at that moment.
|
|
133
|
+
(about 10 s) and lists the targets on the page at that moment. What step N
|
|
134
|
+
opened (a menu, a dialog, a panel) is marked ★ and listed first. If a step
|
|
135
|
+
fails, the result lists the targets on the page at that moment.
|
|
136
|
+
- Inspect lists up to 150 targets. When it says "… and N more not listed",
|
|
137
|
+
open the panel you need with a step and inspect after that step.
|
|
138
|
+
- `(covered by something on top right now)` means a dialog or banner sits over
|
|
139
|
+
that element: a click lands on the cover. Close or `hide` it first.
|
|
128
140
|
4. Copy targets exactly as printed: `{ role: "button", name: "Convert", exact: true }`.
|
|
129
141
|
- `name` is the accessible name. It can differ from the label on screen;
|
|
130
142
|
inspect shows `(on screen: "Export Image")` when it does. Never write a
|
|
@@ -158,6 +170,12 @@ small controls are readable. Your job is the structure:
|
|
|
158
170
|
its own `say:` explaining **what** it does and **why this value**. Never
|
|
159
171
|
"fill in the form and submit" in one line. `check` warns when a tutorial
|
|
160
172
|
step does more than two things.
|
|
173
|
+
- **Say it when it happens.** A line starts with its step, so the viewer hears
|
|
174
|
+
"now the line is flat" before the drag has flattened it. When a line is about
|
|
175
|
+
what the step's actions produce, give that step `sayAt: after`: its line and
|
|
176
|
+
caption start once the actions are done (needs `duration: auto`, the default
|
|
177
|
+
for tutorials and lessons). Lines that say what you're about to do ("click
|
|
178
|
+
Send") stay as they are.
|
|
161
179
|
- **Show, then explain the result.** After an action that produces output
|
|
162
180
|
(a results table, a number, a plot, a status), add a step that only looks:
|
|
163
181
|
`mark` the value or row you're talking about (`mark: "E value"`,
|
|
@@ -184,14 +202,18 @@ small controls are readable. Your job is the structure:
|
|
|
184
202
|
`note` the field the reader must not miss. Set the article's heading and
|
|
185
203
|
first paragraph with `article: { title: …, intro: … }`. Downloads show a
|
|
186
204
|
"Downloaded" bubble; `paste` and copy-to-clipboard work.
|
|
187
|
-
- **
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
205
|
+
- **Animated cards that help explain.** Tutorials and lessons render their
|
|
206
|
+
cards with HyperFrames by default (Node 22+; it installs itself on first
|
|
207
|
+
use): titles land word by word, and each bullet lights up when the `say`
|
|
208
|
+
line names it, so write the bullets in the narration's own words. Use them:
|
|
209
|
+
a concept card for every idea that isn't on screen (the idea as the title,
|
|
210
|
+
its 2–4 steps as bullets, a formula or a line of code as `emblem`), a
|
|
211
|
+
question card for each check, and a recap end card. For a subject with its
|
|
212
|
+
own material, give every title a backdrop from it:
|
|
191
213
|
`hyperframes: { band: <a sequence or formula>, emblem: [<1–3 short lines>] }`.
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
214
|
+
For a promo, `hyperframes: true` gives the launch template with hand-drawn
|
|
215
|
+
accents (`hand: true` for hand lettering). `hyperframes: false` keeps the
|
|
216
|
+
built-in cards.
|
|
195
217
|
- **End with a recap**: `end_card` with `bullets` listing the steps or rules
|
|
196
218
|
to remember, and a `say` that repeats them.
|
|
197
219
|
- **Narrating real results.** You can't know the numbers (scores, counts,
|
|
@@ -206,6 +228,59 @@ small controls are readable. Your job is the structure:
|
|
|
206
228
|
don't pad: if a part is long, split it into more parts rather than
|
|
207
229
|
speeding it up.
|
|
208
230
|
|
|
231
|
+
### Writing the narration
|
|
232
|
+
|
|
233
|
+
Viewers notice three things first: lines that run together, phrasing nobody
|
|
234
|
+
says out loud, and an explanation that jumps around. Write the `say:` lines as
|
|
235
|
+
a script before the steps around them, then read them in order on their own.
|
|
236
|
+
They should sound like one person talking a friend through it.
|
|
237
|
+
|
|
238
|
+
**Say it like a person.**
|
|
239
|
+
- Short sentences, one idea each: about 15 words at most in English, 25
|
|
240
|
+
characters in Chinese. If you would run out of breath, split it.
|
|
241
|
+
- Spoken words: contractions ("you'll", "it's"), "you", present tense, active
|
|
242
|
+
voice. In Chinese, 口语 rather than 书面语: "点一下这里", not "进行点击操作".
|
|
243
|
+
- Use the exact labels on screen, and the same term every time (no synonyms
|
|
244
|
+
for variety).
|
|
245
|
+
- Each sentence gets a breath after it: Storyshot widens the pause between
|
|
246
|
+
sentences to at least `voice.pause` (0.3 s for `pitch`, 0.6 s for tutorials
|
|
247
|
+
and lessons). Commas get no extra pause, so split long lines into sentences
|
|
248
|
+
rather than chaining clauses with commas.
|
|
249
|
+
- For fixed durations, budget about 2.5 words a second in English and 4
|
|
250
|
+
characters a second in Chinese, and leave silence while the screen is busy.
|
|
251
|
+
|
|
252
|
+
**Cut the phrases that mark text as machine-written** (`check` flags the
|
|
253
|
+
common ones):
|
|
254
|
+
- Contrasts that only add weight: "not just X, but Y", "it's not X, it's Y".
|
|
255
|
+
- Inflated words: seamless, effortless, powerful, robust, unlock, elevate,
|
|
256
|
+
supercharge, revolutionize, leverage, cutting-edge, game-changer, delve;
|
|
257
|
+
"in today's fast-paced world". In Chinese: 赋能, 助力, 一站式, 全方位, 极致,
|
|
258
|
+
无缝, 轻松搞定, 不仅……更……, 让我们一起.
|
|
259
|
+
- Openers and closers that say nothing: "Let's dive in", "Ever wondered…?",
|
|
260
|
+
"And that's it!", "Simple as that."
|
|
261
|
+
- Three adjectives in a row, line after line ("fast, simple and powerful").
|
|
262
|
+
- Filler that talks down to a stuck viewer: basically, simply, just, easily.
|
|
263
|
+
- Dashes (—) in `say:` lines. Voices read them as a run-on; use a period.
|
|
264
|
+
|
|
265
|
+
**Explain in an order the viewer can follow** (tutorials and lessons):
|
|
266
|
+
1. The question or the goal ("How does b move the line?").
|
|
267
|
+
2. A concrete case on screen first: change the value, show the result.
|
|
268
|
+
3. Then name the idea, in plain words, the first time it appears.
|
|
269
|
+
4. Why it works, in one or two sentences, with a concept card if it isn't on
|
|
270
|
+
screen (its bullets light up as you name them; a formula goes in `emblem`).
|
|
271
|
+
5. A check: a question card, then the step that reveals the answer.
|
|
272
|
+
6. The recap, in the same words as before.
|
|
273
|
+
|
|
274
|
+
Every step answers the question the step before it raised. Link them out loud:
|
|
275
|
+
"Now that b has moved the line up, let's see what m does." When the camera
|
|
276
|
+
has to go somewhere else on the page, say why first ("The results are further
|
|
277
|
+
down, in the table."). If a line only makes sense with the video, or the
|
|
278
|
+
script reads as a list of disconnected facts, reorder the steps before you
|
|
279
|
+
render.
|
|
280
|
+
|
|
281
|
+
(These rules borrow from [humanizer](https://github.com/blader/humanizer) and
|
|
282
|
+
[stop-slop](https://github.com/hardikpandya/stop-slop), both MIT.)
|
|
283
|
+
|
|
209
284
|
Rules of thumb from real renders (promos and tutorials):
|
|
210
285
|
|
|
211
286
|
- **One idea per step.**
|
|
@@ -225,11 +300,15 @@ Rules of thumb from real renders (promos and tutorials):
|
|
|
225
300
|
`voice.pronounce` (e.g. `pxGenius: P X Genius`, `AI: A I`) so subtitles stay
|
|
226
301
|
correct. For fixed durations, budget about 2.5 words per second.
|
|
227
302
|
- **Isolate long waits** (AI processing, uploads) in their own step: a
|
|
228
|
-
`waitFor` the finished state
|
|
229
|
-
`duration: 1.0–1.5`.
|
|
303
|
+
`waitFor` the finished state. In `pitch` mode add `sfx: riser` and
|
|
304
|
+
`duration: 1.0–1.5`. In tutorials and lessons, say what is happening
|
|
305
|
+
instead; at most `sfx: chime` when the result arrives (a riser there plays
|
|
306
|
+
only its chime). Waiting is compressed automatically.
|
|
230
307
|
- **Zoom sparingly**, on 2–4 payoff moments (headline, clicked button, result,
|
|
231
308
|
price), with `scale: 1.2–1.5`.
|
|
232
|
-
- **Show the signature interaction.** Use `
|
|
309
|
+
- **Show the signature interaction.** Use `slide` for sliders:
|
|
310
|
+
`slide: { target: <the slider inspect lists>, value: 3 }` lands on that value
|
|
311
|
+
(`to: 0..1` for sliders that don't report a range). Use `drag` for before/after
|
|
233
312
|
views (`drag: { from: { x: 0.57, y: 0.5 }, by: [{ x: -300 }, { x: 400 }] }`;
|
|
234
313
|
`from` can be a target or a point). Use `type` for inputs: `type: y=x^2`
|
|
235
314
|
types into whatever has focus (for editors and canvases with hidden inputs),
|
package/src/audio.mjs
CHANGED
|
@@ -108,7 +108,8 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
|
|
|
108
108
|
return filt(rand(), 400 + 5000 * Math.sin(Math.PI * p), 1.6).bp * Math.pow(Math.sin(Math.PI * p), 2);
|
|
109
109
|
}, 0.5);
|
|
110
110
|
}
|
|
111
|
-
|
|
111
|
+
// the rising sound under a wait is for promos; tutorials and lessons (sfx: clicks) keep only its chime
|
|
112
|
+
for (const [a, b] of cfg.sfx.transitions ? events.risers : []) {
|
|
112
113
|
const len = b - a, filt = svf();
|
|
113
114
|
let ph = 0;
|
|
114
115
|
add(sfx, a, len, (t) => {
|
|
@@ -133,12 +134,15 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
|
|
|
133
134
|
if (cfg.voice && events.voice?.length) {
|
|
134
135
|
let prevEnd = 0;
|
|
135
136
|
for (const line of events.voice) {
|
|
136
|
-
let pcm = await speak(cfg, line.text, { log });
|
|
137
|
+
let { pcm, parts } = await speak(cfg, line.text, { log });
|
|
137
138
|
let dur = pcm.length / 2 / SR;
|
|
138
139
|
const start = Math.max(line.start, prevEnd + 0.12), room = line.until - start - 0.1;
|
|
139
140
|
let sped = 1;
|
|
140
141
|
// a line pushed past its window by the one before has no room left (room <= 0): it plays late, at normal speed
|
|
141
|
-
if (dur > room && room > 0 && dur / room <= 1.2) {
|
|
142
|
+
if (dur > room && room > 0 && dur / room <= 1.2) {
|
|
143
|
+
sped = dur / room; pcm = tempo(pcm, sped); dur = pcm.length / 2 / SR;
|
|
144
|
+
parts = parts.map((p) => ({ ...p, start: p.start / sped, end: p.end / sped }));
|
|
145
|
+
}
|
|
142
146
|
else if (dur > room) warn(room > 0
|
|
143
147
|
? `${line.where}: narration is ${dur.toFixed(1)}s but only ${room.toFixed(1)}s until ${line === events.voice.at(-1) ? 'the video ends' : 'the next line'} — shorten "${line.text}" or give the step more duration`
|
|
144
148
|
: `${line.where}: the narration before it runs ${(start - line.start).toFixed(1)}s past where this line should start, so "${line.text}" plays late — shorten the earlier lines or give their steps more duration`);
|
|
@@ -154,7 +158,8 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
|
|
|
154
158
|
voiceOn[s0 + i] = 1;
|
|
155
159
|
}
|
|
156
160
|
prevEnd = start + dur;
|
|
157
|
-
|
|
161
|
+
// subtitles per sentence, so the pauses between them don't throw off the word timing
|
|
162
|
+
for (const p of parts) narration.push({ start: start + p.start, end: Math.min(start + p.end, start + dur), text: p.text });
|
|
158
163
|
}
|
|
159
164
|
}
|
|
160
165
|
|
package/src/check.mjs
CHANGED
|
@@ -5,6 +5,34 @@ import { loadConfig } from './config.mjs';
|
|
|
5
5
|
import { gitIgnored } from './auth.mjs';
|
|
6
6
|
import { isZhVoice, zhVoiceStatus } from './kokoro-zh.mjs';
|
|
7
7
|
import { qwenStatus } from './qwen-tts.mjs';
|
|
8
|
+
import { sentences } from './voice.mjs';
|
|
9
|
+
|
|
10
|
+
// Phrases that mark narration as machine-written, and sentences too long to say in one breath.
|
|
11
|
+
const TELLS = [
|
|
12
|
+
/\bnot (?:just|only|merely) [^.,;!?]{1,40}?,? but(?: also)?\b/i, // "not just X, but Y"
|
|
13
|
+
/\bit'?s not (?:about )?[^.,;!?]{1,30}[,;—–-]+ it'?s\b/i, // "it's not X, it's Y"
|
|
14
|
+
/\b(?:seamless(?:ly)?|effortless(?:ly)?|supercharg\w*|revolutioni[sz]\w*|game[- ]chang\w*|cutting[- ]edge|elevat(?:e|es|ed|ing)|leverag(?:e|es|ed|ing)|delv(?:e|es|ing))\b/i,
|
|
15
|
+
/\bin today'?s (?:fast-paced|digital|modern)\b/i,
|
|
16
|
+
/\blet'?s dive in\b|\bever wondered\b|\bsimple as that\b|\band that'?s it\b/i,
|
|
17
|
+
/\b(?:basically|simply)\b/i, // ("easily" is too often literal: "binds more easily")
|
|
18
|
+
/[—–]/,
|
|
19
|
+
/赋能|助力|一站式|全方位|极致|无缝|轻松搞定|让我们一起|不仅[^。!?]{1,20}更/,
|
|
20
|
+
];
|
|
21
|
+
export function narrationLint(text) {
|
|
22
|
+
const t = String(text).replace(/\*\*/g, ''), tells = [];
|
|
23
|
+
for (const re of TELLS) {
|
|
24
|
+
const g = new RegExp(re.source, re.flags.includes('g') ? re.flags : `${re.flags}g`);
|
|
25
|
+
for (const m of t.matchAll(g)) if (!tells.includes(m[0].trim())) tells.push(m[0].trim());
|
|
26
|
+
}
|
|
27
|
+
let long = null;
|
|
28
|
+
for (const s of sentences(t)) {
|
|
29
|
+
const cjk = (s.match(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g) ?? []).length;
|
|
30
|
+
const words = s.replace(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g, ' ').split(/\s+/).filter((w) => /\w/.test(w)).length;
|
|
31
|
+
if (cjk > 40 && !long) long = `${cjk}-character`;
|
|
32
|
+
else if (!cjk && words > 25 && !long) long = `${words}-word`;
|
|
33
|
+
}
|
|
34
|
+
return { tells, long };
|
|
35
|
+
}
|
|
8
36
|
|
|
9
37
|
const TOP = ['name', 'url', 'out', 'purpose', 'viewport', 'output', 'theme', 'music', 'voice', 'sfx', 'storageState', 'ignoreHTTPSErrors', 'intro', 'frame', 'pace', 'follow', 'article', 'cards', 'lang', 'colorScheme',
|
|
10
38
|
'scrollOffset', 'captionPosition', 'captions', 'highlight', 'stepNumbers', 'subtitles', 'chapters', 'clips', 'maxWait', 'setup', 'steps', 'end_card', 'hyperframes'];
|
|
@@ -70,6 +98,8 @@ export function check(file, { draft = false } = {}) {
|
|
|
70
98
|
else target(s.zoom.target, `${where} zoom.target`);
|
|
71
99
|
}
|
|
72
100
|
if (s.hidden && (s.caption || s.say)) warnings.push(`${where}: hidden steps are cut from the video, so caption/say are ignored`);
|
|
101
|
+
if (s.sayAt != null && s.sayAt !== 'after' && !(typeof s.sayAt === 'number' && s.sayAt >= 0)) errors.push(`${where}: sayAt must be seconds into the step (e.g. 0.5) or "after" (once the step's actions are done)`);
|
|
102
|
+
if (s.sayAt === 'after' && typeof (s.duration ?? raw.duration) === 'number') warnings.push(`${where}: sayAt: after needs duration: auto (a fixed-length step has no time after its actions)`);
|
|
73
103
|
if (!Array.isArray(s.actions ?? [])) errors.push(`${where}: actions must be a list`);
|
|
74
104
|
for (const [j, a] of (s.actions ?? []).entries()) checkAction(a, `${where} action ${j + 1}`);
|
|
75
105
|
if (s.card) unknown(s.card, END, `${where} card`);
|
|
@@ -107,7 +137,7 @@ export function check(file, { draft = false } = {}) {
|
|
|
107
137
|
if (type === 'target') target(v, `${where} (${kind})`);
|
|
108
138
|
if (type === 'mark' && typeof v !== 'string') target(v, `${where} (mark)`); // a string is a phrase to highlight
|
|
109
139
|
if (type === 'slide') {
|
|
110
|
-
if (!v?.target || typeof v?.to !== 'number' || v.to < 0 || v.to > 1) errors.push(`${where}: slide needs { target: <slider>, to: 0..1 } (0 = start of the track, 1 = end)`);
|
|
140
|
+
if (!v?.target || (typeof v?.value !== 'number' && (typeof v?.to !== 'number' || v.to < 0 || v.to > 1))) errors.push(`${where}: slide needs { target: <slider>, to: 0..1 } (0 = start of the track, 1 = end) or { target: <slider>, value: <number> }`);
|
|
111
141
|
else target(v.target, `${where} (slide.target)`);
|
|
112
142
|
}
|
|
113
143
|
if (type === 'select') {
|
|
@@ -146,18 +176,28 @@ export function check(file, { draft = false } = {}) {
|
|
|
146
176
|
// estimate from its length; the render measures the real voice)
|
|
147
177
|
if (cfg.voice) {
|
|
148
178
|
const words = (t) => { const x = String(t).replace(/\*\*/g, ''); const c = (x.match(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g) ?? []).length; return { c, w: x.replace(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g, ' ').split(/\s+/).filter(Boolean).length }; };
|
|
149
|
-
const secs = (t) => { const { c, w } = words(t); return (c / 4.5 + w / 2.8) / (cfg.voice.speed ?? 1); };
|
|
179
|
+
const secs = (t) => { const { c, w } = words(t); return (c / 4.5 + w / 2.8) / (cfg.voice.speed ?? 1) + (sentences(t).length - 1) * (cfg.voice.pause ?? 0); };
|
|
150
180
|
const vis = visible.map((s) => ({ s, i: cfg.steps.indexOf(s) }));
|
|
151
181
|
for (const [k, { s, i }] of vis.entries()) {
|
|
152
182
|
if (!s.say) continue;
|
|
153
183
|
const nextK = vis.findIndex((v, j) => j > k && v.s.say);
|
|
154
184
|
const span = vis.slice(k, nextK < 0 ? undefined : nextK).map((v) => v.s.duration);
|
|
155
185
|
if (!span.every((d) => typeof d === 'number')) continue;
|
|
156
|
-
const room = span.reduce((a, d) => a + d, 0) - (s.sayAt
|
|
186
|
+
const room = span.reduce((a, d) => a + d, 0) - (typeof s.sayAt === 'number' ? s.sayAt : 0.15) + (nextK < 0 && cfg.endCard ? 2.5 : 0);
|
|
157
187
|
const need = secs(s.say);
|
|
158
188
|
if (need > room * 1.4 && need - room > 1) warnings.push(`step ${i + 1}: the say: line takes about ${need.toFixed(1)}s to speak but has ${room.toFixed(1)}s before ${nextK < 0 ? 'the video ends' : 'the next line'} — shorten it or raise duration (the lines after it would play late)`);
|
|
159
189
|
}
|
|
160
190
|
}
|
|
191
|
+
// spoken lines that read as machine-written, or run on without a breath (the skill's "Writing the narration")
|
|
192
|
+
if (cfg.voice) {
|
|
193
|
+
const lines = [...visible.flatMap((s) => [[s.say, `step ${cfg.steps.indexOf(s) + 1}`], [s.card?.say, `step ${cfg.steps.indexOf(s) + 1} card`]]),
|
|
194
|
+
[cfg.intro?.say, 'intro'], [cfg.endCard?.say, 'end card']].filter(([t]) => t);
|
|
195
|
+
for (const [text, where] of lines) {
|
|
196
|
+
const { tells, long } = narrationLint(text);
|
|
197
|
+
if (tells.length) warnings.push(`${where}: the narration sounds machine-written (${tells.map((x) => `"${x}"`).join(', ')}) — say it plainly, as you would out loud`);
|
|
198
|
+
if (long) warnings.push(`${where}: a ${long} sentence — split it, so the voice gets a breath between the parts`);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
161
201
|
// the Chinese Kokoro voice needs a Python for its G2P: say so now, not after the recording
|
|
162
202
|
const v = draft && cfg.voice ? loadConfig(file, { draft }).voice : cfg.voice;
|
|
163
203
|
if (v?.provider === 'kokoro' && isZhVoice(v.voice)) {
|
package/src/compose.mjs
CHANGED
|
@@ -25,7 +25,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
25
25
|
const cardLen = (c, voice, min) => (c.duration === 'auto' || c.duration == null ? Math.max(min, (voice ?? 0) + 0.9, readTime(c)) + (c.think ?? 0) : c.duration);
|
|
26
26
|
// the end card carries the call to action: long enough to read it (and for a command to type itself out)
|
|
27
27
|
const ctaTime = (c) => (c?.cta ? 0.8 + String(c.cta).length * 0.045 + 1.6 : 0);
|
|
28
|
-
const END = !ec0 ? 0 : ec0.duration === 'auto' ? Math.max(2.5, (voiceLens.endCard ?? 0) + 0.6, readTime(ec0), ctaTime(ec0)) : ec0.duration;
|
|
28
|
+
const END = !ec0 ? 0 : ec0.duration === 'auto' ? Math.max(2.5, (voiceLens.endCard ?? 0) + 0.6 + cfg.pace.tail * 0.6, readTime(ec0), ctaTime(ec0)) : ec0.duration;
|
|
29
29
|
if (ec0 && END < 2.5) warn(`the end card shows for only ${END.toFixed(1)}s: too short to read its call to action — use duration: auto, or at least 2.5`);
|
|
30
30
|
const in0 = cfg.intro;
|
|
31
31
|
const INTRO = !in0 ? 0 : in0.duration === 'auto' ? Math.max(cfg.hyperframes?.template === 'launch' ? 2.8 : 1.8, (voiceLens.intro ?? 0) + 0.8, readTime(in0)) : in0.duration;
|
|
@@ -56,21 +56,27 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
56
56
|
const M = parts.filter((p) => !p[2]).reduce((x, p) => x + p[1] - p[0], 0);
|
|
57
57
|
const Wt = parts.filter((p) => p[2]).reduce((x, p) => x + p[1] - p[0], 0);
|
|
58
58
|
let D, waitBudget, motionBudget, hold = 0;
|
|
59
|
+
// when the line starts: `sayAt: after` once the step's actions are done (a line about their result:
|
|
60
|
+
// "now the line is flat"), else this many seconds into the step
|
|
61
|
+
let sayAt = typeof s.sayAt === 'number' ? s.sayAt : 0.15;
|
|
59
62
|
if (s.duration === 'auto') {
|
|
60
|
-
const voiceNeed = voiceLens.steps[i] != null ? voiceLens.steps[i] + (s.sayAt ?? 0.15) + cfg.pace.tail : 0;
|
|
61
63
|
waitBudget = Math.min(Wt, cfg.maxWait);
|
|
62
64
|
motionBudget = M;
|
|
65
|
+
if (s.sayAt === 'after') sayAt = M + waitBudget + 0.2;
|
|
66
|
+
const voiceNeed = voiceLens.steps[i] != null ? voiceLens.steps[i] + sayAt + cfg.pace.tail : 0;
|
|
63
67
|
// a silent step holds its result too (docs GIFs: long enough to see it before the loop restarts)
|
|
64
68
|
const quiet = s.say ? 0 : cfg.clips.length ? 1.5 : cfg.pace.tail;
|
|
65
69
|
D = Math.max(M + waitBudget + quiet, voiceNeed, 0.8);
|
|
66
70
|
let extra = D - M - waitBudget;
|
|
67
|
-
|
|
71
|
+
// more of the waiting first, then hold the last frame (after the line about the result: only hold)
|
|
72
|
+
const stretch = s.sayAt === 'after' ? 0 : Math.min(extra, Wt - waitBudget);
|
|
68
73
|
waitBudget += stretch; extra -= stretch;
|
|
69
74
|
hold = extra;
|
|
70
75
|
} else {
|
|
71
76
|
D = s.duration;
|
|
72
77
|
waitBudget = M + Wt <= D ? Wt * D / (M + Wt) : M === 0 ? D : Math.min(Wt, Math.max(D - M, 0.15 * D));
|
|
73
78
|
motionBudget = D - waitBudget;
|
|
79
|
+
if (s.sayAt === 'after') warn(`step ${i + 1}: sayAt: after needs duration: auto (a fixed-length step has no time after its actions) — the line starts with the step`);
|
|
74
80
|
}
|
|
75
81
|
if (s.card) {
|
|
76
82
|
// a section card plays over the step's first frame, before the step
|
|
@@ -83,7 +89,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
83
89
|
cards.push({ i, t0: acc, t1: acc + C, card: s.card, thinkAt: C - (s.card.think ?? 0) - 0.3 });
|
|
84
90
|
acc += C;
|
|
85
91
|
}
|
|
86
|
-
const g = { i, step: s, o0: acc, o1: acc + D, zoomBox: r.zoomBox, afterHidden, hold,
|
|
92
|
+
const g = { i, step: s, o0: acc, o1: acc + D, sayAt, zoomBox: r.zoomBox, afterHidden, hold,
|
|
87
93
|
motionSpeed: M && motionBudget ? M / motionBudget : 1, waitSpeed: Wt && waitBudget ? Wt / waitBudget : 1 };
|
|
88
94
|
for (const [a, b, w] of parts) {
|
|
89
95
|
const len = (b - a) / (w ? g.waitSpeed : g.motionSpeed);
|
|
@@ -173,7 +179,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
173
179
|
const spoken = [
|
|
174
180
|
...(in0 ? [[0, INTRO]] : []), // cards are their own pictures (the frames under them don't show)
|
|
175
181
|
...cards.map((c) => [c.t0, c.t1]),
|
|
176
|
-
...segs.filter((g) => voiceLens.steps[g.i] != null).map((g) => [g.o0 +
|
|
182
|
+
...segs.filter((g) => voiceLens.steps[g.i] != null).map((g) => [g.o0 + g.sayAt, g.o0 + g.sayAt + voiceLens.steps[g.i]]),
|
|
177
183
|
];
|
|
178
184
|
let from = null;
|
|
179
185
|
const report = (to) => {
|
|
@@ -235,6 +241,11 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
235
241
|
g.zoomBox = null;
|
|
236
242
|
} else if (z && !g.zoomBox) warn(`step ${g.i + 1}: zoom target not found on the page, zoom skipped`);
|
|
237
243
|
if (!z || !g.zoomBox) continue;
|
|
244
|
+
{
|
|
245
|
+
// measured off screen (e.g. before the step scrolled to it): the camera can only frame what's in the window
|
|
246
|
+
const b = g.zoomBox, cx = b.x + b.width / 2, cy = b.y + b.height / 2, { width: VW, height: VH } = cfg.viewport;
|
|
247
|
+
if (cx < 0 || cx > VW || cy < 0 || cy > VH) warn(`step ${g.i + 1}: the zoom target was off screen when it was measured (${cy > VH ? `${Math.round(cy - VH)}px below` : cy < 0 ? `${Math.round(-cy)}px above` : 'beside'} the window), so the zoom frames the edge of the page instead — zoom on something an action in this step moves to, or use a point target`);
|
|
248
|
+
}
|
|
238
249
|
g.zoomPeak = g.o0 + (z.start ?? 0) + (Array.isArray(z.ramp) ? z.ramp[0] : z.ramp ?? 0.5) + 0.1;
|
|
239
250
|
const b = g.zoomBox, off = z.offset ?? {};
|
|
240
251
|
const c = [(b.x + b.width / 2 + (off.x ?? 0)) * K, (b.y + b.height / 2 + (off.y ?? 0)) * K];
|
|
@@ -355,7 +366,8 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
355
366
|
warn(`step ${g.i + 1}: the ${want} caption would cover ${hit.what}, so it was moved to the ${other}` + (explicit ? ` (despite captionPosition: ${explicit})` : '') + ` — set captionPosition: ${other} to make this explicit`);
|
|
356
367
|
} else if (hit) warn(`step ${g.i + 1}: the caption covers ${hit.what} at both top and bottom — shorten the caption, or change the zoom`);
|
|
357
368
|
if (prev && n === prev.n && prev.pos === pos) prev.t[1] = g.o1;
|
|
358
|
-
|
|
369
|
+
// a line about the step's result (sayAt: after) shows its caption with it, not before the result is there
|
|
370
|
+
else captions.push({ text: g.step.caption, t: [g.step.sayAt === 'after' ? g.o0 + Math.max(0, g.sayAt - 0.1) : g.o0, g.o1], pos, n, h });
|
|
359
371
|
}
|
|
360
372
|
for (const [i, c] of (cfg.captions ? captions : []).entries()) await shot(bubble(c.text, c.n, c.pos), path.join(cfg.work, `cap${i}.png`), true);
|
|
361
373
|
// Intro and end cards are short animations (logo pops, title rises, the highlight sweeps, a command
|
|
@@ -483,10 +495,22 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
483
495
|
return f;
|
|
484
496
|
};
|
|
485
497
|
const sizeOf = (f) => {
|
|
498
|
+
if (/\.svg$/i.test(f)) {
|
|
499
|
+
// ffprobe can't read SVG: its shape from width/height or the viewBox, as large as the cards draw a logo
|
|
500
|
+
const tag = fs.readFileSync(f, 'utf8').match(/<svg\b[^>]*>/i)?.[0] ?? '';
|
|
501
|
+
const attr = (a) => parseFloat(tag.match(new RegExp(`\\s${a}\\s*=\\s*["']?([\\d.]+)(px)?["'\\s>]`, 'i'))?.[1]);
|
|
502
|
+
const vb = tag.match(/viewBox\s*=\s*["']\s*[-\d.]+[\s,]+[-\d.]+[\s,]+([\d.]+)[\s,]+([\d.]+)/i);
|
|
503
|
+
const [w, h] = attr('width') > 0 && attr('height') > 0 ? [attr('width'), attr('height')] : vb ? [+vb[1], +vb[2]] : [0, 0];
|
|
504
|
+
if (!(w > 0 && h > 0)) { warn(`the logo ${path.basename(f)} has no width/height or viewBox, so the cards leave it out — add a viewBox, or use a PNG`); return null; }
|
|
505
|
+
const k = Math.min((W * 0.42) / w, (H * 0.17) / h) / 3.2;
|
|
506
|
+
return { w: w * k, h: h * k };
|
|
507
|
+
}
|
|
486
508
|
try {
|
|
487
509
|
const [w, h] = execFileSync('ffprobe', ['-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=width,height', '-of', 'csv=p=0', f]).toString().trim().split(',').map(Number);
|
|
488
|
-
|
|
489
|
-
} catch {
|
|
510
|
+
if (w && h) return { w, h };
|
|
511
|
+
} catch {}
|
|
512
|
+
warn(`could not read the size of the logo ${path.basename(f)}, so the cards leave it out — use a PNG, JPG or SVG`);
|
|
513
|
+
return null;
|
|
490
514
|
};
|
|
491
515
|
const voice = (text, start, len) => (text && len ? { text, start, len } : null);
|
|
492
516
|
const jobs = [];
|
|
@@ -670,7 +694,7 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
670
694
|
voice: [
|
|
671
695
|
...(in0?.say ? [{ text: in0.say, start: 0.15, where: 'intro' }] : []),
|
|
672
696
|
...cards.filter((c) => c.card.say).map((c) => ({ text: c.card.say, start: c.t0 + 0.15, where: `step ${c.i + 1} card` })),
|
|
673
|
-
...segs.filter((g) => g.step.say).map((g) => ({ text: g.step.say, start: g.o0 +
|
|
697
|
+
...segs.filter((g) => g.step.say).map((g) => ({ text: g.step.say, start: g.o0 + g.sayAt, where: `step ${g.i + 1}` })),
|
|
674
698
|
...(ec?.say ? [{ text: ec.say, start: VIDEO + 0.05, where: 'end card' }] : []),
|
|
675
699
|
],
|
|
676
700
|
};
|