storyshot 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -153,7 +153,7 @@ Rendering is free and runs locally. Only the voiceover can cost money:
153
153
  |---|---|---|
154
154
  | none | free | captions only |
155
155
  | `kokoro` | free | local, English and Mandarin Chinese (`zf_001`); downloads a ~90 MB model on first use (Chinese: ~130 MB, and needs Python 3.10–3.12) |
156
- | `qwen` | free | local, Apple Silicon Macs only; the most natural free Chinese voice (`dylan`, `serena`); downloads a ~2.3 GB model and needs Python 3.10–3.12 |
156
+ | `qwen` | free | local, Apple Silicon Macs only; the most natural free Chinese voice (`dylan`, `serena`); every take is checked by a speech recognizer and made again if it comes out wrong; downloads ~3.3 GB of models and needs Python 3.10–3.12 |
157
157
  | ElevenLabs | your credits | the most natural voice; many languages including Chinese. Put `ELEVENLABS_API_KEY` in `.env`. |
158
158
  | OpenAI | your credits | `OPENAI_API_KEY` |
159
159
 
@@ -191,7 +191,7 @@ from `storyshot guide`.
191
191
  | `inspect_page` | Verified click targets and a screenshot, for a URL or for the state after step N of a storyboard. Reports login walls. |
192
192
  | `login` | Opens a browser window where the **user** logs in, and saves the session. |
193
193
  | `init_storyboard` | Writes a starter storyboard with verified targets. |
194
- | `check_storyboard` | Validates in about a second: typos with "did you mean", action and target shapes, `TODO`s, keys, the session file. |
194
+ | `check_storyboard` | Validates in about a second: typos with "did you mean", action and target shapes, `TODO`s, keys, the session file, and narration that sounds machine-written ("not just X, but Y", "seamless", 赋能…) or runs on without a breath. |
195
195
  | `render_video` / `render_status` | Renders in the background and returns the MP4 path, warnings, a per-step speed log and the preview image. If the render isn't done within 45 s, it returns a `job_id`, so client tool timeouts never kill a render. |
196
196
 
197
197
  Any other MCP client: run `npx -y storyshot mcp` (stdio), and point the agent at the playbook (`storyshot guide`). When run from a git
@@ -248,7 +248,7 @@ viewport: { width: 1600, height: 900 } # CSS px the site is laid out at
248
248
  output: { width: 1920, height: 1080, fps: 30 } # same aspect ratio; use 1080x1920 with a 900x1600 viewport for vertical
249
249
  theme: { accent: "#d9f75b", ink: "#0f172a", font: Inter }
250
250
  music: { bpm: 120, volume: 1 } # or { file: track.mp3 }, or false
251
- sfx: true # click / whoosh / riser sounds; clicks = no whooshes on cards and page changes (tutorial and course default); false = none
251
+ sfx: true # click / whoosh / riser sounds; clicks = no whooshes on cards and page changes, and no rising sound under a riser step, only its chime (tutorial and course default); false = none
252
252
  voice: { provider: elevenlabs, voice: hpp4J3VqNfWAUOO0d1Us } # optional voiceover, see below
253
253
  storageState: auth.json # optional Playwright login state
254
254
  ignoreHTTPSErrors: true # internal site with a self-signed HTTPS certificate (plain http:// needs nothing)
@@ -287,7 +287,7 @@ steps:
287
287
  - click: { role: button, name: Try it }
288
288
  - caption: AI does the work
289
289
  duration: 1.3
290
- sfx: riser # rising sound + chime (or `chime`)
290
+ sfx: riser # rising sound + chime (or `chime`); a tutorial or lesson plays only the chime
291
291
  actions:
292
292
  - waitFor: { text: Done, timeout: 120000 } # waiting is compressed automatically
293
293
  - card: { kicker: Part 2, title: Read the results } # a section card before this step (tutorials); becomes a chapter
@@ -332,15 +332,15 @@ You can add `exact: true`, `last: true` or `nth: 2` to any of them.
332
332
  | `waitForUrl` | `waitForUrl: "**/dashboard"` |
333
333
  | `networkIdle` | `networkIdle: true` |
334
334
  | `goto` | `goto: /pricing` |
335
- | `scrollTo` | `scrollTo: { css: "#pricing", ms: 1200 }` |
335
+ | `scrollTo` | `scrollTo: { css: "#pricing", ms: 1200 }` scrolls the target to the top of the window, below a sticky header |
336
336
  | `scroll` | `scroll: 600` |
337
337
  | `dismiss` | `dismiss: { role: button, name: Close }` (clicks only if visible) |
338
338
  | `hide` | `hide: '[id*="QSI"], .chat-widget'` hides pop-ups, survey prompts, feedback tabs and chat widgets for the whole video, including ones that appear late (use it in `setup`) |
339
- | `mark` | `mark: "automatic zoom, captions"` sweeps a highlighter over that phrase, even inside a paragraph (scrolling to it if needed). `mark: { css: "table tr:nth-child(3)" }` marks any target with a box. Options: `ms` (sweep time, default 700), `color` (default: theme accent), `box: true/false`. Marks stay until the step ends. |
339
+ | `mark` | `mark: "automatic zoom, captions"` sweeps a highlighter over that phrase, even inside a paragraph (scrolling to it only if it's off screen, and below a sticky header). `mark: { css: "table tr:nth-child(3)" }` marks any target with a box. Options: `ms` (sweep time, default 700), `color` (default: theme accent), `box: true/false`. Marks stay until the step ends. |
340
340
  | `unmark` | `unmark: true` clears the marks (and notes) earlier |
341
341
  | `note` | `note: { target: { label: Database }, text: Curated proteins only }` puts a short label with an arrow next to the element, on the side with the most room (or `side: left/right/top/bottom`). It stays until the step ends and gets reading time (about 3 words a second, or `ms`). |
342
342
  | `paste` | `paste: { target: { label: JSON }, text: '{"a": 1}' }` pastes text in one go through the real clipboard (with a ⌘V badge), so editors react as to a real paste; `paste: "..."` pastes into whatever has focus |
343
- | `slide` | `slide: { target: { label: Quality }, to: 0.2 }` drags a slider to a position along its track (0 = start, 1 = end), found from the element, not pixels |
343
+ | `slide` | `slide: { target: { label: Quality }, to: 0.2 }` drags a slider to a position along its track (0 = start, 1 = end), found from the element, not pixels. `value: 3` drags it to that value instead, when the slider reports its range (an `<input type=range>`, or an ARIA slider such as Desmos's; `inspect` shows the range). The target can be the slider or its knob |
344
344
  | `select` | `select: { target: { label: Database }, option: "UniProtKB/Swiss-Prot" }` picks an option in a dropdown (by its visible text, or part of it). `inspect` lists each dropdown's options. |
345
345
 
346
346
  `ms` sets how long the cursor takes to travel to the target (`dragMs` for each
@@ -375,7 +375,7 @@ In-page anchor links (`href="#..."`) scroll smoothly instead of jumping.
375
375
  |---|---|
376
376
  | `target` | What to center on: a target, or a point `{ x: 0.5, y: 0.3 }`. If a step action used the same target, its position at that moment is used (so you can zoom on a button that disappears after the click). Otherwise it is measured at the start of the step when the zoom begins in the first half, or at the end of the step otherwise. |
377
377
  | `scale` | Zoom factor (default 1.3). 1.2–1.5 works well. |
378
- | `start`, `end` | Seconds from the start of the step. The camera eases in from 1× at `start` and is back at 1× at `end` (default: the step's end). `end` may run past the step, e.g. to hold the zoom through a click. |
378
+ | `start`, `end` | Seconds from the start of the step. The camera eases in from 1× at `start` and is back at 1× at `end` (default: the step's end). A zoom belongs to its step: an `end` past the step's end stops there, with a warning. |
379
379
  | `ramp` | Ease-in / ease-out seconds: `0.5` or `[in, out]`. |
380
380
  | `offset` | Moves the camera center from the target's center, in viewport CSS px. `{ y: -120 }` centers 120 px above the target. |
381
381
 
@@ -419,14 +419,20 @@ for Chinese tutorials, lessons and docs), `serena` (female, warm; the default fo
419
419
  `vivian` (female, bright), `uncle_fu` (male, mellow), `eric` (male, Sichuan accent), and `ryan`, `aiden` (English), `ono_anna` (Japanese), `sohee` (Korean). On
420
420
  first use Storyshot creates a venv in `~/.cache/storyshot/qwen-venv` (Python 3.10–3.12, about 400 MB)
421
421
  and downloads the ~2.3 GB model into the Hugging Face cache. It speaks at about 2x real time on an M4 Pro
422
- and peaks at about 8 GB of memory. The model samples its speech, so a line now and then runs on into
423
- invented words or stops short. Storyshot regenerates such a take (it judges by length) and warns if
424
- one still looks off. A scientific number like `2e-105` is spelled out in Chinese, because it sets the
422
+ and peaks at about 8 GB of memory. The model samples its speech, so a take now and then garbles a few
423
+ words, drops one, runs on into invented words or stops short. Storyshot listens to every take with
424
+ [Qwen3-ASR 0.6B](https://huggingface.co/Qwen/Qwen3-ASR-0.6B) (Apache-2.0, a ~1 GB model downloaded on
425
+ first use) and compares what it hears with the line: Chinese by sound, so a homophone isn't a mistake, and
426
+ numbers digit by digit. A take that comes out wrong, or far too long or short, is made again with another
427
+ seed (up to 4 tries), and Storyshot warns, quoting what it heard, if a line still doesn't come out right.
428
+ This adds about 0.2 s per line. A scientific number like `2e-105` is spelled out in Chinese, because it sets the
425
429
  model babbling. On other machines the same storyboard renders with `kokoro` (`zf_001`, or `draftVoice`),
426
430
  with a note.
427
431
 
428
- Other fields: `speed` (default 1), `volume` (default 1), and `pronounce`, a
429
- map of word → spoken form used only for TTS, e.g.
432
+ Other fields: `speed` (default 1), `volume` (default 1), `pause` (the shortest pause between the
433
+ sentences of a line, in seconds: 0.3 for promos, 0.6 for tutorials and lessons; a voice's own longer pause
434
+ stays, and `0` leaves the voice as it is),
435
+ and `pronounce`, a map of word → spoken form used only for TTS, e.g.
430
436
  `pronounce: { pxGenius: P X Genius, PPTX: P P T X }`. Subtitles keep the real
431
437
  spelling. Synthesized lines
432
438
  are cached in `.storyshot/tts-cache/`, so `compose` only pays for lines you
@@ -459,11 +465,12 @@ usually shouldn't.
459
465
  | `say` | Voiceover line played over the end card. |
460
466
  | `emblem` | With `hyperframes`: one to three short monospace lines over the title (an alignment, a formula, a line of code); `false` for none. |
461
467
 
462
- ### HyperFrames cards (optional)
468
+ ### HyperFrames cards
463
469
 
464
- `hyperframes: true` renders the intro, section and end cards with
470
+ Tutorials and lessons render their intro, section and end cards with
465
471
  [HyperFrames](https://hyperframes.heygen.com) (HeyGen's open-source HTML-to-video
466
- renderer, Apache-2.0) for more motion. Two templates, picked by `purpose`:
472
+ renderer, Apache-2.0) for more motion; `hyperframes: false` keeps the built-in
473
+ cards, and `hyperframes: true` turns them on for a promo. Two templates, picked by `purpose`:
467
474
 
468
475
  - **lesson** (tutorials, courses, docs): the title lands word by word over a
469
476
  slowly drifting, blurred view of the page the section leads into; section
@@ -485,7 +492,7 @@ hyperframes:
485
492
  backdrop: false # lesson: plain background instead of the blurred page
486
493
  ```
487
494
 
488
- It needs Node 22 or newer. On first use storyshot installs the pinned
495
+ It needs Node 22 or newer; on older Node, tutorials and lessons use the built-in cards. On first use storyshot installs the pinned
489
496
  HyperFrames version into `~/.cache/storyshot/hyperframes/` (a local install
490
497
  of the `hyperframes` package wins); its telemetry is switched off. Cards are
491
498
  cached, so `compose` only re-renders the ones that changed (about 10–25 s
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "storyshot",
3
- "version": "0.6.0",
3
+ "version": "0.7.0",
4
4
  "description": "Turn a short YAML storyboard into a polished product demo video of your web app — smooth cursor, auto zoom, captions, music.",
5
5
  "author": "boheling <boheling@gmail.com>",
6
6
  "repository": {
@@ -92,8 +92,11 @@ frames or stock footage presented as a recording of their app.
92
92
  best free Chinese voice is `voice: { provider: qwen }` (Qwen3-TTS: `dylan`,
93
93
  a male teacher's voice, for tutorials and lessons; `serena`, female, for
94
94
  promos; those are the defaults): clearer than
95
- zf_001 with English words and letters, about 2x real time, a ~2.3 GB model
96
- on first use. Elsewhere a `qwen` storyboard renders with kokoro. For other
95
+ zf_001 with English words and letters, about 2x real time, ~3.3 GB of
96
+ models on first use. Storyshot listens to each take and makes it again if
97
+ it comes out garbled; a `⚠ qwen: … sounds like "…"` warning means a line
98
+ still came out wrong after 4 tries: rephrase it (spell out a number, say
99
+ `一格` for `1 格`). Elsewhere a `qwen` storyboard renders with kokoro. For other
97
100
  languages, use ElevenLabs.
98
101
  3. **Login.** If the flow needs a logged-in page:
99
102
  - Tell the user a browser window will open and that they should log in
@@ -184,14 +187,18 @@ small controls are readable. Your job is the structure:
184
187
  `note` the field the reader must not miss. Set the article's heading and
185
188
  first paragraph with `article: { title: …, intro: … }`. Downloads show a
186
189
  "Downloaded" bubble; `paste` and copy-to-clipboard work.
187
- - **More motion on the cards** (when the user asks for livelier titles, or for
188
- a launch video): `hyperframes: true` renders the intro, section and end
189
- cards with HyperFrames (needs Node 22+; installs itself on first use). For a
190
- subject with its own material, give the titles a backdrop from it:
190
+ - **Animated cards that help explain.** Tutorials and lessons render their
191
+ cards with HyperFrames by default (Node 22+; it installs itself on first
192
+ use): titles land word by word, and each bullet lights up when the `say`
193
+ line names it, so write the bullets in the narration's own words. Use them:
194
+ a concept card for every idea that isn't on screen (the idea as the title,
195
+ its 2–4 steps as bullets, a formula or a line of code as `emblem`), a
196
+ question card for each check, and a recap end card. For a subject with its
197
+ own material, give every title a backdrop from it:
191
198
  `hyperframes: { band: <a sequence or formula>, emblem: [<1–3 short lines>] }`.
192
- Promos get hand-drawn accents; `hand: true` uses hand lettering. Write the
193
- intro's `bullets` and the end card's recap with the same words the `say`
194
- line uses: each point lights up when the narration names it.
199
+ For a promo, `hyperframes: true` gives the launch template with hand-drawn
200
+ accents (`hand: true` for hand lettering). `hyperframes: false` keeps the
201
+ built-in cards.
195
202
  - **End with a recap**: `end_card` with `bullets` listing the steps or rules
196
203
  to remember, and a `say` that repeats them.
197
204
  - **Narrating real results.** You can't know the numbers (scores, counts,
@@ -206,6 +213,59 @@ small controls are readable. Your job is the structure:
206
213
  don't pad: if a part is long, split it into more parts rather than
207
214
  speeding it up.
208
215
 
216
+ ### Writing the narration
217
+
218
+ Viewers notice three things first: lines that run together, phrasing nobody
219
+ says out loud, and an explanation that jumps around. Write the `say:` lines as
220
+ a script before the steps around them, then read them in order on their own.
221
+ They should sound like one person talking a friend through it.
222
+
223
+ **Say it like a person.**
224
+ - Short sentences, one idea each: about 15 words at most in English, 25
225
+ characters in Chinese. If you would run out of breath, split it.
226
+ - Spoken words: contractions ("you'll", "it's"), "you", present tense, active
227
+ voice. In Chinese, 口语 rather than 书面语: "点一下这里", not "进行点击操作".
228
+ - Use the exact labels on screen, and the same term every time (no synonyms
229
+ for variety).
230
+ - Each sentence gets a breath after it: Storyshot widens the pause between
231
+ sentences to at least `voice.pause` (0.3 s for `pitch`, 0.6 s for tutorials
232
+ and lessons). Commas get no extra pause, so split long lines into sentences
233
+ rather than chaining clauses with commas.
234
+ - For fixed durations, budget about 2.5 words a second in English and 4
235
+ characters a second in Chinese, and leave silence while the screen is busy.
236
+
237
+ **Cut the phrases that mark text as machine-written** (`check` flags the
238
+ common ones):
239
+ - Contrasts that only add weight: "not just X, but Y", "it's not X, it's Y".
240
+ - Inflated words: seamless, effortless, powerful, robust, unlock, elevate,
241
+ supercharge, revolutionize, leverage, cutting-edge, game-changer, delve;
242
+ "in today's fast-paced world". In Chinese: 赋能, 助力, 一站式, 全方位, 极致,
243
+ 无缝, 轻松搞定, 不仅……更……, 让我们一起.
244
+ - Openers and closers that say nothing: "Let's dive in", "Ever wondered…?",
245
+ "And that's it!", "Simple as that."
246
+ - Three adjectives in a row, line after line ("fast, simple and powerful").
247
+ - Filler that talks down to a stuck viewer: basically, simply, just, easily.
248
+ - Dashes (—) in `say:` lines. Voices read them as a run-on; use a period.
249
+
250
+ **Explain in an order the viewer can follow** (tutorials and lessons):
251
+ 1. The question or the goal ("How does b move the line?").
252
+ 2. A concrete case on screen first: change the value, show the result.
253
+ 3. Then name the idea, in plain words, the first time it appears.
254
+ 4. Why it works, in one or two sentences, with a concept card if it isn't on
255
+ screen (its bullets light up as you name them; a formula goes in `emblem`).
256
+ 5. A check: a question card, then the step that reveals the answer.
257
+ 6. The recap, in the same words as before.
258
+
259
+ Every step answers the question the step before it raised. Link them out loud:
260
+ "Now that b has moved the line up, let's see what m does." When the camera
261
+ has to go somewhere else on the page, say why first ("The results are further
262
+ down, in the table."). If a line only makes sense with the video, or the
263
+ script reads as a list of disconnected facts, reorder the steps before you
264
+ render.
265
+
266
+ (These rules borrow from [humanizer](https://github.com/blader/humanizer) and
267
+ [stop-slop](https://github.com/hardikpandya/stop-slop), both MIT.)
268
+
209
269
  Rules of thumb from real renders (promos and tutorials):
210
270
 
211
271
  - **One idea per step.**
@@ -225,11 +285,15 @@ Rules of thumb from real renders (promos and tutorials):
225
285
  `voice.pronounce` (e.g. `pxGenius: P X Genius`, `AI: A I`) so subtitles stay
226
286
  correct. For fixed durations, budget about 2.5 words per second.
227
287
  - **Isolate long waits** (AI processing, uploads) in their own step: a
228
- `waitFor` the finished state, `sfx: riser`, and in `pitch` mode
229
- `duration: 1.0–1.5`. Waiting is compressed automatically.
288
+ `waitFor` the finished state. In `pitch` mode add `sfx: riser` and
289
+ `duration: 1.0–1.5`. In tutorials and lessons, say what is happening
290
+ instead; at most `sfx: chime` when the result arrives (a riser there plays
291
+ only its chime). Waiting is compressed automatically.
230
292
  - **Zoom sparingly**, on 2–4 payoff moments (headline, clicked button, result,
231
293
  price), with `scale: 1.2–1.5`.
232
- - **Show the signature interaction.** Use `drag` for sliders and before/after
294
+ - **Show the signature interaction.** Use `slide` for sliders:
295
+ `slide: { target: <the slider inspect lists>, value: 3 }` lands on that value
296
+ (`to: 0..1` for sliders that don't report a range). Use `drag` for before/after
233
297
  views (`drag: { from: { x: 0.57, y: 0.5 }, by: [{ x: -300 }, { x: 400 }] }`;
234
298
  `from` can be a target or a point). Use `type` for inputs: `type: y=x^2`
235
299
  types into whatever has focus (for editors and canvases with hidden inputs),
package/src/audio.mjs CHANGED
@@ -108,7 +108,8 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
108
108
  return filt(rand(), 400 + 5000 * Math.sin(Math.PI * p), 1.6).bp * Math.pow(Math.sin(Math.PI * p), 2);
109
109
  }, 0.5);
110
110
  }
111
- for (const [a, b] of events.risers) {
111
+ // the rising sound under a wait is for promos; tutorials and lessons (sfx: clicks) keep only its chime
112
+ for (const [a, b] of cfg.sfx.transitions ? events.risers : []) {
112
113
  const len = b - a, filt = svf();
113
114
  let ph = 0;
114
115
  add(sfx, a, len, (t) => {
@@ -133,12 +134,15 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
133
134
  if (cfg.voice && events.voice?.length) {
134
135
  let prevEnd = 0;
135
136
  for (const line of events.voice) {
136
- let pcm = await speak(cfg, line.text, { log });
137
+ let { pcm, parts } = await speak(cfg, line.text, { log });
137
138
  let dur = pcm.length / 2 / SR;
138
139
  const start = Math.max(line.start, prevEnd + 0.12), room = line.until - start - 0.1;
139
140
  let sped = 1;
140
141
  // a line pushed past its window by the one before has no room left (room <= 0): it plays late, at normal speed
141
- if (dur > room && room > 0 && dur / room <= 1.2) { sped = dur / room; pcm = tempo(pcm, sped); dur = pcm.length / 2 / SR; }
142
+ if (dur > room && room > 0 && dur / room <= 1.2) {
143
+ sped = dur / room; pcm = tempo(pcm, sped); dur = pcm.length / 2 / SR;
144
+ parts = parts.map((p) => ({ ...p, start: p.start / sped, end: p.end / sped }));
145
+ }
142
146
  else if (dur > room) warn(room > 0
143
147
  ? `${line.where}: narration is ${dur.toFixed(1)}s but only ${room.toFixed(1)}s until ${line === events.voice.at(-1) ? 'the video ends' : 'the next line'} — shorten "${line.text}" or give the step more duration`
144
148
  : `${line.where}: the narration before it runs ${(start - line.start).toFixed(1)}s past where this line should start, so "${line.text}" plays late — shorten the earlier lines or give their steps more duration`);
@@ -154,7 +158,8 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
154
158
  voiceOn[s0 + i] = 1;
155
159
  }
156
160
  prevEnd = start + dur;
157
- narration.push({ start, end: start + dur, text: line.text });
161
+ // subtitles per sentence, so the pauses between them don't throw off the word timing
162
+ for (const p of parts) narration.push({ start: start + p.start, end: Math.min(start + p.end, start + dur), text: p.text });
158
163
  }
159
164
  }
160
165
 
package/src/check.mjs CHANGED
@@ -5,6 +5,34 @@ import { loadConfig } from './config.mjs';
5
5
  import { gitIgnored } from './auth.mjs';
6
6
  import { isZhVoice, zhVoiceStatus } from './kokoro-zh.mjs';
7
7
  import { qwenStatus } from './qwen-tts.mjs';
8
+ import { sentences } from './voice.mjs';
9
+
10
+ // Phrases that mark narration as machine-written, and sentences too long to say in one breath.
11
+ const TELLS = [
12
+ /\bnot (?:just|only|merely) [^.,;!?]{1,40}?,? but(?: also)?\b/i, // "not just X, but Y"
13
+ /\bit'?s not (?:about )?[^.,;!?]{1,30}[,;—–-]+ it'?s\b/i, // "it's not X, it's Y"
14
+ /\b(?:seamless(?:ly)?|effortless(?:ly)?|supercharg\w*|revolutioni[sz]\w*|game[- ]chang\w*|cutting[- ]edge|elevat(?:e|es|ed|ing)|leverag(?:e|es|ed|ing)|delv(?:e|es|ing))\b/i,
15
+ /\bin today'?s (?:fast-paced|digital|modern)\b/i,
16
+ /\blet'?s dive in\b|\bever wondered\b|\bsimple as that\b|\band that'?s it\b/i,
17
+ /\b(?:basically|simply)\b/i, // ("easily" is too often literal: "binds more easily")
18
+ /[—–]/,
19
+ /赋能|助力|一站式|全方位|极致|无缝|轻松搞定|让我们一起|不仅[^。!?]{1,20}更/,
20
+ ];
21
+ export function narrationLint(text) {
22
+ const t = String(text).replace(/\*\*/g, ''), tells = [];
23
+ for (const re of TELLS) {
24
+ const g = new RegExp(re.source, re.flags.includes('g') ? re.flags : `${re.flags}g`);
25
+ for (const m of t.matchAll(g)) if (!tells.includes(m[0].trim())) tells.push(m[0].trim());
26
+ }
27
+ let long = null;
28
+ for (const s of sentences(t)) {
29
+ const cjk = (s.match(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g) ?? []).length;
30
+ const words = s.replace(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g, ' ').split(/\s+/).filter((w) => /\w/.test(w)).length;
31
+ if (cjk > 40 && !long) long = `${cjk}-character`;
32
+ else if (!cjk && words > 25 && !long) long = `${words}-word`;
33
+ }
34
+ return { tells, long };
35
+ }
8
36
 
9
37
  const TOP = ['name', 'url', 'out', 'purpose', 'viewport', 'output', 'theme', 'music', 'voice', 'sfx', 'storageState', 'ignoreHTTPSErrors', 'intro', 'frame', 'pace', 'follow', 'article', 'cards', 'lang', 'colorScheme',
10
38
  'scrollOffset', 'captionPosition', 'captions', 'highlight', 'stepNumbers', 'subtitles', 'chapters', 'clips', 'maxWait', 'setup', 'steps', 'end_card', 'hyperframes'];
@@ -107,7 +135,7 @@ export function check(file, { draft = false } = {}) {
107
135
  if (type === 'target') target(v, `${where} (${kind})`);
108
136
  if (type === 'mark' && typeof v !== 'string') target(v, `${where} (mark)`); // a string is a phrase to highlight
109
137
  if (type === 'slide') {
110
- if (!v?.target || typeof v?.to !== 'number' || v.to < 0 || v.to > 1) errors.push(`${where}: slide needs { target: <slider>, to: 0..1 } (0 = start of the track, 1 = end)`);
138
+ if (!v?.target || (typeof v?.value !== 'number' && (typeof v?.to !== 'number' || v.to < 0 || v.to > 1))) errors.push(`${where}: slide needs { target: <slider>, to: 0..1 } (0 = start of the track, 1 = end) or { target: <slider>, value: <number> }`);
111
139
  else target(v.target, `${where} (slide.target)`);
112
140
  }
113
141
  if (type === 'select') {
@@ -146,7 +174,7 @@ export function check(file, { draft = false } = {}) {
146
174
  // estimate from its length; the render measures the real voice)
147
175
  if (cfg.voice) {
148
176
  const words = (t) => { const x = String(t).replace(/\*\*/g, ''); const c = (x.match(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g) ?? []).length; return { c, w: x.replace(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g, ' ').split(/\s+/).filter(Boolean).length }; };
149
- const secs = (t) => { const { c, w } = words(t); return (c / 4.5 + w / 2.8) / (cfg.voice.speed ?? 1); };
177
+ const secs = (t) => { const { c, w } = words(t); return (c / 4.5 + w / 2.8) / (cfg.voice.speed ?? 1) + (sentences(t).length - 1) * (cfg.voice.pause ?? 0); };
150
178
  const vis = visible.map((s) => ({ s, i: cfg.steps.indexOf(s) }));
151
179
  for (const [k, { s, i }] of vis.entries()) {
152
180
  if (!s.say) continue;
@@ -158,6 +186,16 @@ export function check(file, { draft = false } = {}) {
158
186
  if (need > room * 1.4 && need - room > 1) warnings.push(`step ${i + 1}: the say: line takes about ${need.toFixed(1)}s to speak but has ${room.toFixed(1)}s before ${nextK < 0 ? 'the video ends' : 'the next line'} — shorten it or raise duration (the lines after it would play late)`);
159
187
  }
160
188
  }
189
+ // spoken lines that read as machine-written, or run on without a breath (the skill's "Writing the narration")
190
+ if (cfg.voice) {
191
+ const lines = [...visible.flatMap((s) => [[s.say, `step ${cfg.steps.indexOf(s) + 1}`], [s.card?.say, `step ${cfg.steps.indexOf(s) + 1} card`]]),
192
+ [cfg.intro?.say, 'intro'], [cfg.endCard?.say, 'end card']].filter(([t]) => t);
193
+ for (const [text, where] of lines) {
194
+ const { tells, long } = narrationLint(text);
195
+ if (tells.length) warnings.push(`${where}: the narration sounds machine-written (${tells.map((x) => `"${x}"`).join(', ')}) — say it plainly, as you would out loud`);
196
+ if (long) warnings.push(`${where}: a ${long} sentence — split it, so the voice gets a breath between the parts`);
197
+ }
198
+ }
161
199
  // the Chinese Kokoro voice needs a Python for its G2P: say so now, not after the recording
162
200
  const v = draft && cfg.voice ? loadConfig(file, { draft }).voice : cfg.voice;
163
201
  if (v?.provider === 'kokoro' && isZhVoice(v.voice)) {
package/src/compose.mjs CHANGED
@@ -235,6 +235,11 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
235
235
  g.zoomBox = null;
236
236
  } else if (z && !g.zoomBox) warn(`step ${g.i + 1}: zoom target not found on the page, zoom skipped`);
237
237
  if (!z || !g.zoomBox) continue;
238
+ {
239
+ // measured off screen (e.g. before the step scrolled to it): the camera can only frame what's in the window
240
+ const b = g.zoomBox, cx = b.x + b.width / 2, cy = b.y + b.height / 2, { width: VW, height: VH } = cfg.viewport;
241
+ if (cx < 0 || cx > VW || cy < 0 || cy > VH) warn(`step ${g.i + 1}: the zoom target was off screen when it was measured (${cy > VH ? `${Math.round(cy - VH)}px below` : cy < 0 ? `${Math.round(-cy)}px above` : 'beside'} the window), so the zoom frames the edge of the page instead — zoom on something an action in this step moves to, or use a point target`);
242
+ }
238
243
  g.zoomPeak = g.o0 + (z.start ?? 0) + (Array.isArray(z.ramp) ? z.ramp[0] : z.ramp ?? 0.5) + 0.1;
239
244
  const b = g.zoomBox, off = z.offset ?? {};
240
245
  const c = [(b.x + b.width / 2 + (off.x ?? 0)) * K, (b.y + b.height / 2 + (off.y ?? 0)) * K];
package/src/config.mjs CHANGED
@@ -31,10 +31,10 @@ export function loadEnv(dir) {
31
31
 
32
32
  // What the video is for sets sensible defaults; anything in the YAML overrides them.
33
33
  export const PURPOSES = {
34
- pitch: { duration: null, music: { bpm: 120, volume: 1 }, sfx: true, highlight: false, stepNumbers: false, subtitles: false, chapters: false, clips: null, frame: 'aurora', pace: 'fast', follow: false },
35
- tutorial: { duration: 'auto', music: { bpm: 96, volume: 0.35 }, sfx: 'clicks', highlight: true, stepNumbers: true, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true },
36
- course: { duration: 'auto', music: false, sfx: 'clicks', highlight: true, stepNumbers: false, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true },
37
- docs: { duration: 'auto', music: false, sfx: false, highlight: true, stepNumbers: true, subtitles: false, chapters: false, clips: ['gif', 'png'], frame: false, pace: 'normal', follow: false },
34
+ pitch: { duration: null, music: { bpm: 120, volume: 1 }, sfx: true, highlight: false, stepNumbers: false, subtitles: false, chapters: false, clips: null, frame: 'aurora', pace: 'fast', follow: false, hyperframes: false },
35
+ tutorial: { duration: 'auto', music: { bpm: 96, volume: 0.35 }, sfx: 'clicks', highlight: true, stepNumbers: true, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true, hyperframes: true },
36
+ course: { duration: 'auto', music: false, sfx: 'clicks', highlight: true, stepNumbers: false, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true, hyperframes: true },
37
+ docs: { duration: 'auto', music: false, sfx: false, highlight: true, stepNumbers: true, subtitles: false, chapters: false, clips: ['gif', 'png'], frame: false, pace: 'normal', follow: false, hyperframes: false },
38
38
  };
39
39
 
40
40
  // Where storyboards and their videos go, and how they're named: <site>-<kind>[-<topic>].yaml →
@@ -43,9 +43,10 @@ export const PURPOSES = {
43
43
  // follow each action and to read its result before the next one starts.
44
44
  export const PACES = {
45
45
  // cursor ms typing ms/char beat after a click (ms) hold after narration (s) waits shown up to (s) voice speed
46
- fast: { cursor: 700, type: 55, beat: 0, tail: 0.4, maxWait: 1.0, voiceSpeed: 1 },
47
- normal: { cursor: 900, type: 70, beat: 350, tail: 0.8, maxWait: 1.5, voiceSpeed: 1 },
48
- slow: { cursor: 1200, type: 95, beat: 800, tail: 1.6, maxWait: 3.0, voiceSpeed: 0.92 },
46
+ // shortest pause between the sentences of one line (s)
47
+ fast: { cursor: 700, type: 55, beat: 0, tail: 0.4, maxWait: 1.0, voiceSpeed: 1, sentencePause: 0.3 },
48
+ normal: { cursor: 900, type: 70, beat: 350, tail: 0.8, maxWait: 1.5, voiceSpeed: 1, sentencePause: 0.45 },
49
+ slow: { cursor: 1200, type: 95, beat: 800, tail: 1.6, maxWait: 3.0, voiceSpeed: 0.92, sentencePause: 0.6 },
49
50
  };
50
51
 
51
52
  export const FRAME_STYLES = ['aurora', 'studio', 'dark', 'pastel', 'blur'];
@@ -138,8 +139,9 @@ export function loadConfig(file, { draft = false } = {}) {
138
139
  if (frame?.icon) frame.iconFile = path.resolve(dir, frame.icon);
139
140
 
140
141
  // hyperframes: true | { template: lesson | launch, band, bandColors, emblem, hand, backdrop } — cards
141
- // rendered by HyperFrames (see hyperframes.mjs); promos get the launch template, the rest the lesson one
142
- const hf0 = raw.hyperframes;
142
+ // rendered by HyperFrames (see hyperframes.mjs); promos get the launch template, the rest the lesson one.
143
+ // On by default for tutorials and lessons, where Node can run it (22+); else the built-in cards
144
+ const hf0 = raw.hyperframes !== undefined ? raw.hyperframes : P.hyperframes && Number(process.versions.node.split('.')[0]) >= 22;
143
145
  const hyperframes = hf0 ? { template: purpose === 'pitch' ? 'launch' : 'lesson', backdrop: true, ...(typeof hf0 === 'object' ? hf0 : {}) } : null;
144
146
  if (hyperframes && !['lesson', 'launch'].includes(hyperframes.template)) throw new Error('hyperframes.template must be lesson or launch');
145
147
  if (hyperframes?.bandColors && !['aminoacid', 'nucleotide', 'accent'].includes(hyperframes.bandColors)) throw new Error('hyperframes.bandColors must be aminoacid, nucleotide or accent');
@@ -152,7 +154,8 @@ export function loadConfig(file, { draft = false } = {}) {
152
154
  // kokoro's and qwen's default voices follow the narration's language (zf_001 / serena for Chinese)
153
155
  const kokoroVoice = kokoroVoiceFor(raw);
154
156
  const byLang = { kokoro: kokoroVoice, qwen: qwenVoiceFor(raw) }[provider];
155
- voice = { speed: pace0.voiceSpeed, volume: 1, ...VOICE_DEFAULTS[provider], ...(byLang ? { voice: byLang } : {}), ...raw.voice, provider };
157
+ voice = { speed: pace0.voiceSpeed, volume: 1, pause: pace0.sentencePause, ...VOICE_DEFAULTS[provider], ...(byLang ? { voice: byLang } : {}), ...raw.voice, provider };
158
+ if (!(typeof voice.pause === 'number' && voice.pause >= 0 && voice.pause <= 3)) throw new Error('voice.pause is the shortest pause between the sentences of a line, in seconds, 0–3 (0 leaves the voice as it is)');
156
159
  if (provider === 'qwen') {
157
160
  const v = qwenVoice(voice.voice);
158
161
  if (!v) throw new Error(`voice: qwen has no voice "${voice.voice}" (one of ${QWEN_VOICES.join(', ')})`);
@@ -214,8 +217,8 @@ export function loadConfig(file, { draft = false } = {}) {
214
217
  follow,
215
218
  music,
216
219
  voice,
217
- // sfx: true (clicks and transition whooshes), 'clicks' (no whooshes: tutorials and lessons, where a
218
- // sound at every card soon grates), or false
220
+ // sfx: true (clicks, transition whooshes and risers), 'clicks' (no whooshes or risers: tutorials and
221
+ // lessons, where a sound at every card soon grates), or false
219
222
  sfx: pick('sfx') === false ? false : { clicks: true, transitions: pick('sfx') !== 'clicks' },
220
223
  draft,
221
224
  out: path.resolve(dir, raw.out ?? `${name}.mp4`).replace(/(\.[^./]+)?$/, (ext) => (draft ? `.draft${ext || '.mp4'}` : ext)),
package/src/doctor.mjs CHANGED
@@ -81,8 +81,8 @@ export async function doctor({ log = console.log } = {}) {
81
81
  : '· Chinese voice (kokoro zf_001) needs Python 3.10–3.12 — optional: brew install python@3.12, or uv python install 3.12 (or set STORYSHOT_PYTHON)');
82
82
  const q = qwenStatus();
83
83
  lines.push(!q.supported ? '· Qwen3-TTS voice (provider: qwen) needs an Apple Silicon Mac; storyboards that use it fall back to kokoro here'
84
- : q.ready ? `✔ Qwen3-TTS voice (provider: qwen): venv in ${q.venv}${q.model ? ', model downloaded' : '; the model (~2.3 GB) downloads on first use'}`
85
- : q.python ? `· Qwen3-TTS voice (provider: qwen): Python ${q.python.version} found; storyshot sets up its venv in ${q.venv} and downloads the model (~2.3 GB) on first use`
84
+ : q.ready ? `✔ Qwen3-TTS voice (provider: qwen): venv in ${q.venv}${q.model ? ', models downloaded' : '; the models (~3.3 GB: the voice, and a speech recognizer that checks each take) download on first use'}`
85
+ : q.python ? `· Qwen3-TTS voice (provider: qwen): Python ${q.python.version} found; storyshot sets up its venv in ${q.venv} and downloads the models (~3.3 GB) on first use`
86
86
  : '· Qwen3-TTS voice (provider: qwen) needs Python 3.10–3.12 — optional: brew install python@3.12, or uv python install 3.12 (or set STORYSHOT_PYTHON)');
87
87
  const hf = hyperframesCli();
88
88
  lines.push(hf.error ? `· HyperFrames cards (hyperframes: true) unavailable — optional: ${hf.error}`
package/src/init.mjs CHANGED
@@ -7,6 +7,14 @@ import { PURPOSES, defaultStoryboard } from './config.mjs';
7
7
  import { loginAdvice } from './auth.mjs';
8
8
 
9
9
  const q = (s) => JSON.stringify(String(s));
10
+ // The product's name from the page title: its first part ("Squoosh | Image compressor" → "Squoosh"),
11
+ // or the repo on a code host ("GitHub - boheling/storyshot: Ask…" → "storyshot"). Only spaced
12
+ // separators split, so a hyphenated name (Tic-Tac-Toe) stays whole.
13
+ export function productName(title, host) {
14
+ const segs = String(title ?? '').split(/\s+[|—–·-]\s+|:\s+/).map((s) => s.trim()).filter(Boolean);
15
+ const repo = segs.find((s) => /^[\w.-]+ ?\/ ?[\w.-]+$/.test(s));
16
+ return repo ? repo.split('/')[1].trim() : segs[0] ?? host;
17
+ }
10
18
  const tgt = (t) => (typeof t === 'string' ? q(t) : `{ ${Object.entries(t).map(([k, v]) => `${k}: ${typeof v === 'string' ? q(v) : v}`).join(', ')} }`);
11
19
 
12
20
  export async function initStoryboard(url, { purpose = 'pitch', topic, file, storageState, ignoreHTTPSErrors = false, viewport = { width: 1600, height: 900 } } = {}) {
@@ -41,7 +49,7 @@ out: ${name}.mp4
41
49
  ${storageState ? `storageState: ${path.relative(path.dirname(path.resolve(file)), path.resolve(storageState)).split(path.sep).join('/')} # saved login (never commit it)\n` : ''}${ignoreHTTPSErrors ? 'ignoreHTTPSErrors: true # internal site with a self-signed certificate\n' : ''}${narrated ? 'voice: { provider: kokoro } # free, local, English. For the final: { provider: elevenlabs, voice: <voice id> } + ELEVENLABS_API_KEY' : '# voice: { provider: kokoro } # optional voiceover; add `say:` lines to steps'}
42
50
 
43
51
  ${purpose === 'pitch' ? `intro: # title card before the recording (promos; optional for tutorials/lessons)
44
- title: ${q(page.title.split(/[|—–-]/)[0].trim() || host)}
52
+ title: ${q(productName(page.title, host))}
45
53
  subtitle: TODO the one-line promise, with the **key words** in bold
46
54
  ${PURPOSES[purpose]?.frame ? '# frame: false # the recording sits in a browser window on a backdrop; false = full frame\n' : ''}
47
55
  ` : purpose === 'tutorial' || purpose === 'course' ? `# intro: { title: TODO, subtitle: TODO } # optional title card (e.g. "How to export a report")
@@ -65,7 +73,7 @@ steps:
65
73
  # run \`storyshot inspect ${file} --after <step>\` to list verified targets at that point.
66
74
 
67
75
  end_card:
68
- title: ${q(page.title.split(/[|—–-]/)[0].trim() || host)}
76
+ title: ${q(productName(page.title, host))}
69
77
  cta: ${q(`Try it at ${host}`)}${narrated ? `\n say: ${q(`Try it at ${host}.`)}` : ''}
70
78
 
71
79
  # ── Verified targets on ${page.url} (each matches exactly one visible element) ──
package/src/inspect.mjs CHANGED
@@ -44,7 +44,8 @@ function collect() {
44
44
  const clean = (s) => (s || '').replace(/\s+/g, ' ').trim();
45
45
  // a short CSS selector (tag + classes, or #id) — the last-resort target when names don't resolve
46
46
  const cssOf = (el) => {
47
- if (el.id && !/^[0-9:]|:/.test(el.id)) return `#${CSS.escape(el.id)}`;
47
+ const id = el.getAttribute('id'); // not el.id: a form with an <input name=id> has that input as its .id
48
+ if (id && !/^[0-9:]|:/.test(id)) return `#${CSS.escape(id)}`;
48
49
  const cls = [...el.classList].filter((c) => !/^(css|sc|jsx|svelte|tw)-|\d{3,}|__/.test(c)).slice(0, 2).map((c) => `.${CSS.escape(c)}`).join('');
49
50
  return cls ? `${el.tagName.toLowerCase()}${cls}` : null;
50
51
  };
@@ -68,10 +69,17 @@ function collect() {
68
69
  // a label's own text, without the links/buttons inside it (e.g. a "Clear" link)
69
70
  const labelText = (lab) => { if (!lab) return null; const c = lab.cloneNode(true); c.querySelectorAll('a,button,select,input,textarea').forEach((x) => x.remove()); return clean(c.innerText) || null; };
70
71
  // styled checkboxes and radios hide the real input and draw their label instead: list them where the label is
72
+ // a slider's range and value, when it says (an <input type=range>, or aria-valuemin/max/now)
73
+ const sliderRange = (el) => {
74
+ const n = (a) => (el.getAttribute(a) == null ? NaN : Number(el.getAttribute(a)));
75
+ const [lo, hi, now] = el.type === 'range' ? [Number(el.min || 0), Number(el.max || 100), Number(el.value)] : [n('aria-valuemin'), n('aria-valuemax'), n('aria-valuenow')];
76
+ return [lo, hi, now].every(Number.isFinite) ? { lo, hi, now } : null;
77
+ };
71
78
  const shownBy = (el) => (el.tagName === 'INPUT' && ['checkbox', 'radio'].includes(el.type) && !vis(el) ? [...(el.labels ?? [])].find(vis) : null);
72
79
  const interactive = [...document.querySelectorAll(NATIVE)]
73
80
  .filter((el) => (vis(el) || shownBy(el)) && el.tagName !== 'LABEL' && !isAd(el) && !(el.isContentEditable && el.parentElement?.isContentEditable))
74
81
  .map((el) => ({ kind: 'role', role: roleOf(el), name: (el.isContentEditable || el.tagName === 'TEXTAREA' ? editorName(el) : nameOf(el)).slice(0, 80), shows: clean(el.innerText).slice(0, 60), alt: clean(el.querySelector('img[alt]')?.alt) || null, href: el.getAttribute('href'), label: labelText(el.labels?.[0]),
82
+ range: roleOf(el) === 'slider' ? sliderRange(el) : null,
75
83
  options: el.tagName === 'SELECT' ? [...el.options].map((o) => ({ text: clean(o.text), on: o.selected })).filter((o) => o.text).slice(0, 12) : null, ...pos(shownBy(el) ?? el) }));
76
84
  // custom clickables: outermost cursor:pointer elements that aren't native controls
77
85
  const pointer = [];
@@ -85,11 +93,11 @@ function collect() {
85
93
  return {
86
94
  title: document.title,
87
95
  height: scroller ? scroller.scrollHeight : document.documentElement.scrollHeight,
88
- scroller: scroller ? (scroller.id ? `#${scroller.id}` : scroller.tagName.toLowerCase() + (scroller.className && typeof scroller.className === 'string' ? '.' + scroller.className.trim().split(/\s+/)[0] : '')) : null,
96
+ scroller: scroller ? (scroller.getAttribute('id') ? `#${scroller.getAttribute('id')}` : scroller.tagName.toLowerCase() + (scroller.className && typeof scroller.className === 'string' ? '.' + scroller.className.trim().split(/\s+/)[0] : '')) : null,
89
97
  headings: [...document.querySelectorAll('h1, h2, h3')].filter(vis).map((h) => ({ tag: h.tagName.toLowerCase(), text: clean(h.innerText).slice(0, 100), ...pos(h) })),
90
98
  anchors: [...document.querySelectorAll('[id]')].filter((el) => vis(el) && el !== scroller && el.getBoundingClientRect().height > 150 &&
91
- el.getBoundingClientRect().height < document.documentElement.scrollHeight * 0.8 && !/^(__|radix|headlessui|:)/.test(el.id))
92
- .map((el) => ({ id: el.id, ...pos(el) })).slice(0, 30),
99
+ el.getBoundingClientRect().height < document.documentElement.scrollHeight * 0.8 && !/^(__|radix|headlessui|:)/.test(el.getAttribute('id')))
100
+ .map((el) => ({ id: el.getAttribute('id'), ...pos(el) })).slice(0, 30),
93
101
  images: [...document.querySelectorAll('img[alt]')].filter((el) => vis(el) && el.alt.trim()).map((el) => ({ alt: clean(el.alt).slice(0, 80), ...pos(el) })).slice(0, 30),
94
102
  interactive: [...interactive, ...pointer],
95
103
  };
@@ -160,7 +168,7 @@ export async function describePage(page, viewport, { maxElements = 80 } = {}) {
160
168
  '',
161
169
  '## Clickable elements — verified targets (each matches exactly one visible element) for click/move/zoom',
162
170
  ' Copy targets exactly; `name` is the accessible name, which may differ from the on-screen label.',
163
- ...elements.map((e) => `${fold(e.y)} y=${e.y}\tat (${e.cx}, ${e.cy})\t${yaml(e.target)}${e.role === 'textbox' ? ` (text input ${e.w}×${e.h} — click it, then \`type: …\`)` : e.role === 'slider' ? ' (slider — `slide: { target: …, to: 0..1 }`)' : e.options ? ` (dropdown — \`select: { target: …, option: "…" }\`; options: ${e.options.map((o) => `"${o.text}"${o.on ? ' (selected)' : ''}`).join(', ')})` : shows(e)}${e.href ? ` → ${e.href}` : ''}${e.kind === 'pointer' ? ' (custom clickable)' : ''}`),
171
+ ...elements.map((e) => `${fold(e.y)} y=${e.y}\tat (${e.cx}, ${e.cy})\t${yaml(e.target)}${e.role === 'textbox' ? ` (text input ${e.w}×${e.h} — click it, then \`type: …\`)` : e.role === 'slider' ? (e.range ? ` (slider ${e.range.lo}…${e.range.hi}, now ${e.range.now} — \`slide: { target: …, value: ${e.range.now} }\`)` : ' (slider — `slide: { target: …, to: 0..1 }`)') : e.options ? ` (dropdown — \`select: { target: …, option: "…" }\`; options: ${e.options.map((o) => `"${o.text}"${o.on ? ' (selected)' : ''}`).join(', ')})` : shows(e)}${e.href ? ` → ${e.href}` : ''}${e.kind === 'pointer' ? ' (custom clickable)' : ''}`),
164
172
  '',
165
173
  '## Images with alt text (target: { alt: "..." })',
166
174
  ...(info.images.length ? info.images.map((i) => `${fold(i.y)} y=${i.y}\t"${i.alt}"`) : ['(none)']),
package/src/qwen-tts.mjs CHANGED
@@ -22,15 +22,20 @@ const REPO = 'mlx-community/Qwen3-TTS-12Hz-0.6B-CustomVoice-bf16';
22
22
  const REVISION = '6415d95f88be018ff9e46813119dc3bc12261328';
23
23
  // 0.9 (the model's default) now and then runs on into made-up words; 0.7 keeps the same voice steadier
24
24
  const TEMPERATURE = 0.7;
25
+ // Each take is checked by ear: Qwen3-ASR (Apache-2.0, ~1 GB) transcribes it, and a line it hears
26
+ // differently is sampled again (pypinyin compares Chinese by sound, so homophones don't count)
27
+ const ASR_REPO = 'mlx-community/Qwen3-ASR-0.6B-8bit';
28
+ const ASR_REVISION = '89e96d92ba34aca20b3e29fb10cc284097d1219f';
25
29
  const REQUIREMENTS = ['mlx-audio==0.6.3', 'mlx==0.32.4', 'mlx-metal==0.32.4', 'transformers==5.22.0', 'huggingface-hub==2.4.0',
26
- 'tokenizers==0.23.3', 'safetensors==0.8.0', 'numpy==2.5.4'];
30
+ 'tokenizers==0.23.3', 'safetensors==0.8.0', 'numpy==2.5.4', 'pypinyin==0.55.0'];
27
31
  // Part of every cached line's hash: change it when the model, the packages or qwen_tts.py's sampling change.
28
- export const QWEN_TTS_ID = `${REPO}@${REVISION.slice(0, 7)}|${REQUIREMENTS[0]}|t${TEMPERATURE}|guard-1`;
32
+ export const QWEN_TTS_ID = `${REPO}@${REVISION.slice(0, 7)}|${REQUIREMENTS[0]}|t${TEMPERATURE}|guard-2|${ASR_REPO}@${ASR_REVISION.slice(0, 7)}`;
29
33
  const CACHE = path.join(os.homedir(), '.cache', 'storyshot');
30
34
  export const QWEN_VENV = path.join(CACHE, 'qwen-venv');
31
35
  const HELPER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'qwen_tts.py');
32
36
  const HUB = process.env.HF_HUB_CACHE ?? path.join(process.env.HF_HOME ?? path.join(os.homedir(), '.cache', 'huggingface'), 'hub');
33
37
  const MODEL_FILE = path.join(HUB, `models--${REPO.replace('/', '--')}`, 'snapshots', REVISION, 'model.safetensors');
38
+ const ASR_FILE = path.join(HUB, `models--${ASR_REPO.replace('/', '--')}`, 'snapshots', ASR_REVISION, 'model.safetensors');
34
39
 
35
40
  // --- text (TTS only: subtitles keep the original) --------------------------------------------------
36
41
  // The model reads = + 2个 第2行 x² itself; a minus sign before a number it drops, and a number like
@@ -66,7 +71,7 @@ const NO_PYTHON = () => 'the Qwen3-TTS voice needs Python 3.10–3.12 to run (br
66
71
  // For doctor and check: can this machine use it, and what's still to come on first use?
67
72
  export function qwenStatus() {
68
73
  if (!qwenSupported()) return { supported: false };
69
- const model = fs.existsSync(MODEL_FILE);
74
+ const model = fs.existsSync(MODEL_FILE) && fs.existsSync(ASR_FILE);
70
75
  if (venvReady()) return { supported: true, ready: true, venv: QWEN_VENV, model };
71
76
  const python = findPython();
72
77
  return { supported: true, ready: false, venv: QWEN_VENV, model, python, problem: python ? null : NO_PYTHON() };
@@ -74,13 +79,22 @@ export function qwenStatus() {
74
79
 
75
80
  function ensureVenv(log) {
76
81
  if (venvReady()) return venvPython(QWEN_VENV);
77
- const py = findPython();
78
- if (!py) throw new Error(NO_PYTHON());
79
- log(`setting up the Qwen3-TTS voice (one time): a Python ${py.version} venv in ${QWEN_VENV} with mlx-audio (~400 MB, 1–2 min)…`);
80
82
  const run = (cmd, args, what) => {
81
83
  const r = spawnSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
82
84
  if (r.status !== 0) throw new Error(`${what} failed: ${(r.stderr || r.stdout || r.error?.message || '').trim().split('\n').slice(-6).join('\n')}`);
83
85
  };
86
+ if (fs.existsSync(venvPython(QWEN_VENV))) {
87
+ // a venv from an older storyshot: bring its packages up to date in place, rebuild it only if that fails
88
+ log(`updating the Qwen3-TTS venv in ${QWEN_VENV}…`);
89
+ try {
90
+ run(venvPython(QWEN_VENV), ['-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q', ...REQUIREMENTS], 'updating the venv');
91
+ fs.writeFileSync(path.join(QWEN_VENV, STAMP), REQUIREMENTS.join('\n'));
92
+ return venvPython(QWEN_VENV);
93
+ } catch (e) { log(` ${e.message.split('\n')[0]} — rebuilding it`); }
94
+ }
95
+ const py = findPython();
96
+ if (!py) throw new Error(NO_PYTHON());
97
+ log(`setting up the Qwen3-TTS voice (one time): a Python ${py.version} venv in ${QWEN_VENV} with mlx-audio (~400 MB, 1–2 min)…`);
84
98
  fs.rmSync(QWEN_VENV, { recursive: true, force: true });
85
99
  fs.mkdirSync(CACHE, { recursive: true });
86
100
  run(py.exe, ['-m', 'venv', QWEN_VENV], `creating the venv with ${py.exe} -m venv`);
@@ -96,12 +110,13 @@ export async function synthQwen(v, entries, { log = console.log } = {}) {
96
110
  if (!qwenSupported()) throw new Error('the Qwen3-TTS voice runs on Apple Silicon Macs only (it uses MLX)');
97
111
  const py = ensureVenv(log);
98
112
  if (!fs.existsSync(MODEL_FILE)) log(`downloading the Qwen3-TTS model (one time, ~2.3 GB) into ${HUB}…`);
113
+ if (!fs.existsSync(ASR_FILE)) log(`downloading the Qwen3-ASR model that listens to each take (one time, ~1 GB) into ${HUB}…`);
99
114
  const lines = entries.map((e) => ({
100
115
  text: e.text, file: `${e.file}.raw.wav`, voice: v.voice, lang: qwenLang(e.text),
101
116
  // the same line gets the same take: a seed from the text
102
117
  seed: parseInt(crypto.createHash('sha1').update(e.text).digest('hex').slice(0, 7), 16),
103
118
  }));
104
- const job = JSON.stringify({ repo: REPO, revision: REVISION, temperature: TEMPERATURE, lines });
119
+ const job = JSON.stringify({ repo: REPO, revision: REVISION, temperature: TEMPERATURE, asr: { repo: ASR_REPO, revision: ASR_REVISION }, lines });
105
120
  const { code, stdout, tail } = await new Promise((resolve, reject) => {
106
121
  const p = spawn(py, ['-I', HELPER], { env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1', TRANSFORMERS_VERBOSITY: 'error' } });
107
122
  let out = '', err = '', buf = '';
@@ -121,9 +136,15 @@ export async function synthQwen(v, entries, { log = console.log } = {}) {
121
136
  p.stdin.end(job);
122
137
  });
123
138
  if (code !== 0) throw new Error(`Qwen3-TTS failed (${py} ${HELPER}): ${tail.trim().split('\n').slice(-8).join('\n')}`);
124
- const { results } = JSON.parse(stdout);
139
+ const { results, asr } = JSON.parse(stdout);
140
+ if (asr) log(` ⚠ qwen: the speech check couldn't load (${asr}), so these takes were checked by length only`);
125
141
  for (const [i, e] of entries.entries()) {
126
- if (results[i]?.suspect) log(` ⚠ qwen: "${e.text}" still sounds off after ${results[i].tries} tries (${results[i].seconds.toFixed(1)}s) — listen to it, or rephrase the line / add a pronounce entry`);
142
+ const r = results[i];
143
+ if (r?.suspect) {
144
+ log(r.heard != null && r.mismatch > 0
145
+ ? ` ⚠ qwen: "${e.text}" still doesn't come out right after ${r.tries} tries — it sounds like "${r.heard}". Listen to it; rephrase the line or add a pronounce entry`
146
+ : ` ⚠ qwen: "${e.text}" still sounds off after ${r.tries} tries (${r.seconds.toFixed(1)}s) — listen to it, or rephrase the line / add a pronounce entry`);
147
+ }
127
148
  const raw = lines[i].file, tmp = `${e.file}.tmp.wav`;
128
149
  // the model has no speed control: ffmpeg's pitch-preserving atempo does it
129
150
  if ((v.speed ?? 1) !== 1) execFileSync('ffmpeg', ['-loglevel', 'error', '-y', '-i', raw, '-af', `atempo=${v.speed}`, tmp]);
package/src/qwen_tts.py CHANGED
@@ -1,13 +1,15 @@
1
1
  # Qwen3-TTS CustomVoice through mlx-audio (Apple Silicon), for `voice: { provider: qwen }`. Runs in
2
2
  # storyshot's own venv (qwen-tts.mjs), one process per render: the model loads once for all lines.
3
3
  #
4
- # stdin: {"repo", "revision", "temperature", "lines": [{"text", "file", "voice", "lang", "seed"}]}
4
+ # stdin: {"repo", "revision", "temperature", "asr": {"repo", "revision"} | null,
5
+ # "lines": [{"text", "file", "voice", "lang", "seed"}]}
5
6
  # stderr: "line i/n <seconds>s [<tries> tries]" per line, plus whatever the libraries log
6
- # stdout: {"results": [{"seconds", "tries", "suspect"}]}
7
+ # stdout: {"results": [{"seconds", "tries", "suspect", "heard", "mismatch"}], "asr": null | "why it didn't load"}
7
8
  #
8
- # The model samples its speech tokens, so now and then a line runs on into made-up words (a
9
- # scientific number like 2e-105 does it) or stops early. A take far longer or shorter than the text
10
- # should need is sampled again with another seed, and the closest one is kept.
9
+ # The model samples its speech tokens, so now and then a take garbles a few words, drops one, runs on
10
+ # into made-up words (a scientific number like 2e-105 does it) or stops early. Each take is checked:
11
+ # its length against the text, and what a speech recognizer (Qwen3-ASR) hears in it against the text.
12
+ # A take that fails is sampled again with another seed, and the best one is kept.
11
13
  import contextlib
12
14
  import json
13
15
  import os
@@ -18,6 +20,10 @@ os.environ.setdefault('HF_HUB_DISABLE_TELEMETRY', '1')
18
20
  os.environ.setdefault('TRANSFORMERS_VERBOSITY', 'error')
19
21
 
20
22
  TRIES = 4
23
+ # a take is sampled again when the recognizer hears this share of the line differently, or this long a run
24
+ # of mistakes in a row (a garble: 负 2 x → 负二长方荡 is 2.5); a warning if every take is still off
25
+ RETRY_AT, RETRY_RUN = 0.10, 2
26
+ WARN_AT, WARN_RUN = 0.15, 2
21
27
 
22
28
 
23
29
  def expected(text):
@@ -32,6 +38,87 @@ def plausible(seconds, est):
32
38
  return 0.35 * est - 0.5 <= seconds <= 2.0 * est + 2.0
33
39
 
34
40
 
41
+ # --- what was said vs. what was meant ---------------------------------------------------------------
42
+ # Chinese is compared by sound (toneless pinyin), so a homophone the recognizer picks (交点/焦点) is no
43
+ # mistake; numbers are read out first (0.5 → 零点五, 2 → 二), since the recognizer writes either form.
44
+ DIGITS = '零一二三四五六七八九'
45
+
46
+
47
+ def cn_int(n):
48
+ if n >= 10 ** 8:
49
+ return ''.join(DIGITS[int(c)] for c in str(n))
50
+
51
+ def four(x):
52
+ out, zero = '', False
53
+ for i, c in enumerate(f'{x:04d}'):
54
+ d = int(c)
55
+ if d == 0:
56
+ zero = bool(out)
57
+ else:
58
+ out += ('零' if zero else '') + DIGITS[d] + ['千', '百', '十', ''][i]
59
+ zero = False
60
+ return out
61
+
62
+ if n < 10:
63
+ return DIGITS[n]
64
+ hi, lo = divmod(n, 10000)
65
+ out = (four(hi) + '万' if hi else '') + (('零' if hi and lo < 1000 else '') + four(lo) if lo else '')
66
+ return re.sub('^一十', '十', out)
67
+
68
+
69
+ def cn_number(m):
70
+ whole, _, frac = m.group().partition('.')
71
+ return cn_int(int(whole)) + ('点' + ''.join(DIGITS[int(c)] for c in frac) if frac else '')
72
+
73
+
74
+ def units(text, chinese, pinyin):
75
+ """The line as a list of sounds: pinyin syllables, digits, and Latin words as '#word' (m x and mx as '#m', '#x')."""
76
+ t = text.lower().replace('−', '-')
77
+ if chinese:
78
+ t = re.sub(r'\d+(?:\.\d+)?', cn_number, t)
79
+ out = []
80
+ for tok in re.findall(r'[㐀-鿿]|[a-z]+|\d+', t):
81
+ if chinese and re.match(r'[㐀-鿿]', tok):
82
+ out += [{'liang': 'er'}.get(p, p) for p in pinyin(tok)]
83
+ elif tok.isdigit():
84
+ out += list(tok) # digit by digit: a wrong number (260 → 45) is several mistakes in a row, not one
85
+ else:
86
+ out += ['#' + c for c in tok] if len(tok) <= 2 else ['#' + tok]
87
+ return out
88
+
89
+
90
+ def mismatch(text, heard, chinese, pinyin):
91
+ """(share of the line heard differently, the longest run of mistakes in a row), both in units of the
92
+ text. A garbled take has a run of several; a homophone or a misheard letter is one on its own."""
93
+ a, b = units(text, chinese, pinyin), units(heard, chinese, pinyin)
94
+ letter = lambda u: len(u) == 2 and u[0] == '#'
95
+ d = {(0, 0): (0, None)}
96
+ for i in range(len(a) + 1):
97
+ for j in range(len(b) + 1):
98
+ if i == j == 0:
99
+ continue
100
+ ways = []
101
+ if i:
102
+ ways.append((d[i - 1, j][0] + 1, (i - 1, j, 1)))
103
+ if j:
104
+ ways.append((d[i, j - 1][0] + 1, (i, j - 1, 1)))
105
+ if i and j:
106
+ x, y = a[i - 1], b[j - 1]
107
+ # a variable (m, b, x) the recognizer wrote as a character is half a mistake
108
+ cost = 0 if x == y else 0.5 if letter(x) or letter(y) else 1
109
+ ways.append((d[i - 1, j - 1][0] + cost, (i - 1, j - 1, cost)))
110
+ # a name in Latin letters in a Chinese line (Desmos) can come back as a few syllables: one mistake
111
+ if chinese and len(x) > 3 and x[0] == '#':
112
+ ways += [(d[i - 1, j - k][0] + 1, (i - 1, j - k, 1)) for k in range(2, min(5, j) + 1)]
113
+ d[i, j] = min(ways, key=lambda w: w[0])
114
+ i, j, run, worst = len(a), len(b), 0, 0
115
+ while (i, j) != (0, 0):
116
+ i, j, cost = d[i, j][1]
117
+ run = run + cost if cost else 0
118
+ worst = max(worst, run)
119
+ return d[len(a), len(b)][0] / max(1, len(a)), worst
120
+
121
+
35
122
  def main():
36
123
  job = json.load(sys.stdin)
37
124
  out = sys.stdout
@@ -45,6 +132,28 @@ def main():
45
132
  from mlx_audio.tts.utils import load_model
46
133
 
47
134
  model = load_model(snapshot_download(job['repo'], revision=job['revision']))
135
+ asr, asr_error, pinyin = None, None, None
136
+ if job.get('asr'):
137
+ try:
138
+ from mlx_audio.stt.utils import load_model as load_stt
139
+ from pypinyin import lazy_pinyin as pinyin
140
+ asr = load_stt(snapshot_download(job['asr']['repo'], revision=job['asr']['revision']))
141
+ except Exception as e: # offline before the first download, say: check by length only
142
+ asr_error = f'{type(e).__name__}: {e}'.splitlines()[0][:300]
143
+
144
+ def hear(audio, line):
145
+ """(what the recognizer heard, mismatch), or (None, None) for a line it can't check."""
146
+ chinese = line['lang'] == 'chinese'
147
+ if asr is None or not (chinese or (line['lang'] == 'auto' and not re.search(r'[㐀-鿿]', line['text']))):
148
+ return None, None
149
+ tmp = line['file'] + '.check.wav'
150
+ write(tmp, audio, model.sample_rate)
151
+ try:
152
+ heard = asr.generate(tmp, language='Chinese' if chinese else 'English').text.strip()
153
+ finally:
154
+ os.remove(tmp)
155
+ return heard, mismatch(line['text'], heard, chinese, pinyin)
156
+
48
157
  results, n = [], len(job['lines'])
49
158
  for i, line in enumerate(job['lines']):
50
159
  est = expected(line['text'])
@@ -55,15 +164,22 @@ def main():
55
164
  temperature=job['temperature'])
56
165
  audio = np.concatenate([np.array(r.audio, dtype=np.float32) for r in parts])
57
166
  seconds = len(audio) / model.sample_rate
58
- score = abs(math.log(max(seconds, 0.1) / est))
59
- if best is None or score < best[0]:
60
- best = (score, audio, seconds)
61
- if plausible(seconds, est):
167
+ fits = plausible(seconds, est)
168
+ heard, m = hear(audio, line) if fits else (None, None)
169
+ # how far off the recognizer heard it: up to 1 is fine (0 when it couldn't check)
170
+ worse = max(m[0] / RETRY_AT, m[1] / RETRY_RUN) if m else 0
171
+ # a take of the right length that says the line beats one that doesn't; then the closest length
172
+ rank = (not fits, worse if worse > 1 else 0, abs(math.log(max(seconds, 0.1) / est)))
173
+ if best is None or rank < best['rank']:
174
+ best = {'rank': rank, 'audio': audio, 'seconds': seconds, 'fits': fits, 'heard': heard, 'm': m}
175
+ if fits and worse <= 1:
62
176
  break
63
- write(line['file'], best[1], model.sample_rate)
64
- results.append({'seconds': best[2], 'tries': k + 1, 'suspect': not plausible(best[2], est)})
65
- print(f'line {i + 1}/{n} {best[2]:.1f}s' + (f' {k + 1} tries' if k else ''), file=sys.stderr, flush=True)
66
- out.write(json.dumps({'results': results}))
177
+ write(line['file'], best['audio'], model.sample_rate)
178
+ m = best['m']
179
+ results.append({'seconds': best['seconds'], 'tries': k + 1, 'heard': best['heard'], 'mismatch': m and m[0],
180
+ 'suspect': not best['fits'] or bool(m and (m[0] > WARN_AT or m[1] > WARN_RUN))})
181
+ print(f'line {i + 1}/{n} {best["seconds"]:.1f}s' + (f' {k + 1} tries' if k else ''), file=sys.stderr, flush=True)
182
+ out.write(json.dumps({'results': results, 'asr': asr_error}))
67
183
 
68
184
 
69
185
  if __name__ == '__main__':
package/src/record.mjs CHANGED
@@ -55,19 +55,60 @@ function pageScript({ scrollOffset }) {
55
55
  for (let p = el.parentElement; p && p !== document.body; p = p.parentElement) if (scrollable(p)) return p;
56
56
  return docEl();
57
57
  };
58
+ // `behavior: 'instant'` on every frame: with `scroll-behavior: smooth` in the page's CSS (GitHub has it),
59
+ // plain scrollTop writes each start a smooth scroll of their own, so the page lags behind and is still
60
+ // moving after the step that scrolled it is over
58
61
  window.__demoScrollBy = (dy, ms = 1200, sc = mainScroller()) => new Promise((done) => {
59
62
  const start = sc.scrollTop, t0 = performance.now();
60
63
  const ease = (t) => (t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2);
61
64
  const step = (now) => {
62
- const k = Math.min(1, (now - t0) / ms);
63
- sc.scrollTop = start + dy * ease(k);
65
+ const k = ms > 0 ? Math.min(1, Math.max(0, (now - t0) / ms)) : 1;
66
+ sc.scrollTo({ top: start + dy * ease(k), behavior: 'instant' });
64
67
  if (k < 1) requestAnimationFrame(step); else done();
65
68
  };
66
69
  requestAnimationFrame(step);
67
70
  });
71
+ // How far down a header will cover the window once `el` is scrolled to the top: bars fixed at the top,
72
+ // and sticky ones that stick inside a box holding `el` (GitHub's README bar sticks within the README).
73
+ const headerCover = (el) => {
74
+ let cover = 0;
75
+ for (const h of document.querySelectorAll('body *')) {
76
+ if (h.getAttribute('id')?.startsWith('__demo') || h.contains(el)) continue; // not h.id: a form with an <input name=id> has that input as its .id
77
+ const cs = getComputedStyle(h);
78
+ if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;
79
+ const r = h.getBoundingClientRect();
80
+ if (!r.height || r.height > innerHeight * 0.3 || r.width < innerWidth * 0.3 || cs.visibility === 'hidden' || cs.opacity === '0') continue;
81
+ if (cs.position === 'fixed' && r.top <= 1) cover = Math.max(cover, r.bottom);
82
+ const stick = parseFloat(cs.top);
83
+ if (cs.position === 'sticky' && !Number.isNaN(stick) && stick < innerHeight * 0.3 && h.parentElement?.contains(el)) cover = Math.max(cover, stick + r.height);
84
+ }
85
+ return cover;
86
+ };
87
+ // the element's top lands `scrollOffset` px down, or just below a header that would cover it there
68
88
  window.__demoScrollTo = (el, ms) => {
69
- const sc = scrollerFor(el), top = sc === docEl() ? 0 : sc.getBoundingClientRect().top;
70
- return window.__demoScrollBy(el.getBoundingClientRect().top - top - scrollOffset, ms, sc);
89
+ const sc = scrollerFor(el), docs = sc === docEl(), top = docs ? 0 : sc.getBoundingClientRect().top;
90
+ const offset = docs ? Math.max(scrollOffset, headerCover(el) + 12) : scrollOffset;
91
+ return window.__demoScrollBy(el.getBoundingClientRect().top - top - offset, ms, sc);
92
+ };
93
+ // resolves once nothing has scrolled for a few frames (the page's own smooth scrolling included)
94
+ window.__demoScrollIdle = (max = 2000) => new Promise((done) => {
95
+ const sc = mainScroller(), pos = () => `${scrollX},${scrollY},${sc.scrollTop}`;
96
+ const end = performance.now() + max;
97
+ let last = pos(), still = 0;
98
+ const check = () => {
99
+ const now = pos();
100
+ still = now === last ? still + 1 : 0; last = now;
101
+ if (still >= 3 || performance.now() > end) done(); else requestAnimationFrame(check);
102
+ };
103
+ requestAnimationFrame(check);
104
+ });
105
+ // a slider's value and range: an <input type=range>, or an ARIA slider (Desmos: [role=slider] on its knob)
106
+ window.__demoSliderValue = (el) => {
107
+ const r = el.type === 'range' ? el : el.querySelector?.('input[type=range]');
108
+ if (r) return { now: Number(r.value), lo: Number(r.min || 0), hi: Number(r.max || 100), native: r };
109
+ const aria = el.hasAttribute('aria-valuenow') ? el : el.querySelector?.('[aria-valuenow]');
110
+ const now = Number(aria?.getAttribute('aria-valuenow'));
111
+ return aria && Number.isFinite(now) ? { now, lo: Number(aria.getAttribute('aria-valuemin') ?? 0), hi: Number(aria.getAttribute('aria-valuemax') ?? 100) } : null;
71
112
  };
72
113
  // Tutorial callouts: spotlight a target, show a pressed key combo.
73
114
  window.__demoSpot = (r) => {
@@ -86,9 +127,10 @@ function pageScript({ scrollOffset }) {
86
127
  Object.assign(el.style, { left: `${r.x - pad}px`, top: `${r.y - pad}px`, width: `${r.width + 2 * pad}px`, height: `${r.height + 2 * pad}px` });
87
128
  requestAnimationFrame(() => { el.style.opacity = '1'; });
88
129
  };
89
- window.__demoSpotOff = () => {
130
+ window.__demoSpotOff = (instant) => {
90
131
  const el = document.getElementById('__demo_spot');
91
- if (el) { el.style.opacity = '0'; setTimeout(() => el.remove(), 300); }
132
+ if (el && instant) el.remove();
133
+ else if (el) { el.style.opacity = '0'; setTimeout(() => el.remove(), 300); }
92
134
  };
93
135
  window.__demoKeys = (label) => {
94
136
  const el = document.createElement('div');
@@ -424,7 +466,7 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
424
466
  await sleep(280);
425
467
  return true;
426
468
  }
427
- const spotOff = () => page.evaluate(() => window.__demoSpotOff()).catch(() => {});
469
+ const spotOff = (instant) => page.evaluate((i) => window.__demoSpotOff(i), !!instant).catch(() => {});
428
470
  const keyLabel = (k) => k.split('+').map((p) => ({ Meta: '⌘', ControlOrMeta: '⌘', Control: 'Ctrl', Shift: '⇧', Alt: '⌥', Enter: '↵ Enter', Escape: 'Esc', Backspace: '⌫', Tab: '⇥ Tab' })[p] ?? (p.length === 1 ? p.toUpperCase() : p)).join(' ');
429
471
 
430
472
  async function runAction(action) {
@@ -473,6 +515,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
473
515
  case 'scrollTo': {
474
516
  const h = await locate(page, spec).elementHandle({ timeout: opt.timeout ?? 15000 });
475
517
  await page.evaluate(([el, ms]) => window.__demoScrollTo(el, ms), [h, opt.ms ?? 1200]);
518
+ // where it landed: a zoom on the same target frames it there, not where it was before the scroll
519
+ const b = await h.boundingBox().catch(() => null);
520
+ if (b) boxes.set(keyOf(spec), b);
476
521
  break;
477
522
  }
478
523
  case 'scroll': await page.evaluate(([dy, ms]) => window.__demoScrollBy(dy, ms), [value.y ?? value, value.ms ?? 1000]); break;
@@ -570,25 +615,28 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
570
615
  }
571
616
  case 'slide': {
572
617
  // slide: { target: <slider>, to: 0.2 } — drag a slider to a position along its track (0 = start,
573
- // 1 = end), found from the element instead of pixel positions that break when the layout moves
574
- if (!value?.target || typeof value?.to !== 'number') throw new Error('slide needs { target: <slider>, to: 0..1 }');
618
+ // 1 = end), or { target, value: 3 } — to a value, for sliders that report their range; found from
619
+ // the element instead of pixel positions that break when the layout moves
620
+ if (!value?.target || (typeof value?.to !== 'number' && typeof value?.value !== 'number')) throw new Error('slide needs { target: <slider>, to: 0..1 } or { target: <slider>, value: <number> }');
575
621
  const p = await pointOf(value.target, value);
576
- const b = p.box;
577
- const from = await locate(page, value.target).evaluate((el) => {
578
- const r = el.type === 'range' ? el : el.querySelector?.('input[type=range]');
579
- if (r) return (Number(r.value) - Number(r.min || 0)) / ((Number(r.max || 100) - Number(r.min || 0)) || 1);
580
- const aria = el.hasAttribute('aria-valuenow') ? el : el.querySelector?.('[aria-valuenow]');
581
- if (aria) {
582
- const now = Number(aria.getAttribute('aria-valuenow')), lo = Number(aria.getAttribute('aria-valuemin') ?? 0), hi = Number(aria.getAttribute('aria-valuemax') ?? 100);
583
- if (Number.isFinite(now)) return (now - lo) / ((hi - lo) || 1);
622
+ const s = await locate(page, value.target).evaluate((el) => {
623
+ // a knob (Desmos's [role=slider] is its 38 px thumb): slide along its track, the nearest wide parent
624
+ let track = el;
625
+ const w = el.getBoundingClientRect().width;
626
+ for (let a = el.parentElement, i = 0; w < 60 && a && i < 4; a = a.parentElement, i++) if (a.getBoundingClientRect().width >= 3 * w) { track = a; break; }
627
+ const t = track.getBoundingClientRect(), v = window.__demoSliderValue(el);
628
+ let from = v ? (v.now - v.lo) / ((v.hi - v.lo) || 1) : null;
629
+ if (from == null) {
630
+ // custom sliders: start where the knob is drawn, so the value doesn't jump first
631
+ const knob = [...track.querySelectorAll('[role=slider], [class*=thumb], [class*=handle], [class*=knob]')]
632
+ .map((k) => k.getBoundingClientRect()).find((k) => k.width > 0 && k.width < t.width * 0.5);
633
+ from = knob ? (knob.x + knob.width / 2 - t.x) / (t.width || 1) : 0.5;
584
634
  }
585
- // custom sliders (e.g. Desmos): start where the knob is drawn, so the value doesn't jump first
586
- const t = el.getBoundingClientRect();
587
- const knob = [...el.querySelectorAll('[role=slider], [class*=thumb], [class*=handle], [class*=knob]')]
588
- .map((k) => k.getBoundingClientRect()).find((k) => k.width > 0 && k.width < t.width * 0.5);
589
- return knob ? (knob.x + knob.width / 2 - t.x) / (t.width || 1) : 0.5;
590
- }).catch(() => 0.5);
591
- const x0 = b.x + b.width * from, x1 = b.x + b.width * Math.max(0, Math.min(1, value.to)), y = b.y + b.height / 2;
635
+ return { box: { x: t.x, y: t.y, width: t.width, height: t.height }, from, range: v && { lo: v.lo, hi: v.hi } };
636
+ }).catch(() => ({ box: p.box, from: 0.5, range: null }));
637
+ if (typeof value.value === 'number' && !s.range) throw new Error('slide value: this slider doesn\'t report its range (no aria-valuemin/max, not an <input type=range>) — use to: 0..1');
638
+ const to = typeof value.value === 'number' ? (value.value - s.range.lo) / ((s.range.hi - s.range.lo) || 1) : value.to;
639
+ const b = s.box, x0 = b.x + b.width * s.from, x1 = b.x + b.width * Math.max(0, Math.min(1, to)), y = b.y + b.height / 2;
592
640
  await moveTo(x0, y, value.ms ?? cfg.pace.cursor);
593
641
  await sleep(120);
594
642
  clicks.push(now());
@@ -596,6 +644,19 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
596
644
  await moveTo(x1, y, value.dragMs ?? 900);
597
645
  await sleep(120);
598
646
  await page.mouse.up();
647
+ if (typeof value.value === 'number') {
648
+ // a native range's thumb doesn't reach the ends of its box, so the drag can land a step off: set it
649
+ const got = await locate(page, value.target).evaluate((el, want) => {
650
+ const v = window.__demoSliderValue(el), r = v?.native;
651
+ if (r && v.now !== want) {
652
+ Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set.call(r, String(want));
653
+ r.dispatchEvent(new Event('input', { bubbles: true })); r.dispatchEvent(new Event('change', { bubbles: true }));
654
+ return Number(r.value);
655
+ }
656
+ return v?.now;
657
+ }, value.value).catch(() => null);
658
+ if (got != null && Math.abs(got - value.value) > 1e-9) log(` ⚠ ${acting}: slide stopped at ${got}, not ${value.value} — the slider snaps to its step`);
659
+ }
599
660
  await settle();
600
661
  await beat();
601
662
  break;
@@ -608,11 +669,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
608
669
  let box = null;
609
670
  const phraseOnly = v.text && Object.keys(v).every((k) => ['text', 'exact'].includes(k));
610
671
  if (phraseOnly) {
672
+ await page.evaluate(() => window.__demoScrollIdle()).catch(() => {}); // judge "in view" where the page ends up
611
673
  const found = await page.evaluate((t) => window.__demoFind(t), v.text);
612
- if (found && !found.inView) {
613
- await page.evaluate((ms) => window.__demoScrollTo(window.__demoFound.startContainer.parentElement, ms), fast ? 0 : 900);
614
- await sleep(fast ? 50 : 950);
615
- }
674
+ if (found && !found.inView) await page.evaluate((ms) => window.__demoScrollTo(window.__demoFound.startContainer.parentElement, ms), fast ? 0 : 900);
616
675
  if (found) box = await page.evaluate(([ms, color, b]) => window.__demoMark('found', { ms, color, box: b }), [ms, color, opt.box ?? false]);
617
676
  }
618
677
  if (!box) {
@@ -659,7 +718,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
659
718
 
660
719
  // --- setup (not recorded) -----------------------------------------------------------
661
720
  // Run a list of actions; on failure, say exactly which one and save a screenshot.
721
+ let acting = '';
662
722
  async function runAll(actions, where) {
723
+ acting = where;
663
724
  for (const [j, a] of actions.entries()) {
664
725
  try {
665
726
  // watchdog: an action that hangs (a frozen page, a stalled browser) fails with a clear message
@@ -759,9 +820,10 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
759
820
  const n = Math.min(inspectAfter, cfg.steps.length);
760
821
  for (const [i, s] of cfg.steps.slice(0, n).entries()) {
761
822
  stepHighlight = false;
762
- // like the recording: marks and notes end with their step (the last step's stay in the screenshot)
763
- if (i > 0) await page.evaluate(() => { window.__demoUnmark(); window.__demoUnnote(); }).catch(() => {});
823
+ // like the recording: marks, notes and spotlights end with their step (the last step's stay in the screenshot)
824
+ if (i > 0) await page.evaluate(() => { window.__demoUnmark(true); window.__demoUnnote(true); window.__demoSpotOff(true); }).catch(() => {});
764
825
  await runAll(s.actions, `step ${i + 1}`);
826
+ await page.evaluate(() => window.__demoScrollIdle()).catch(() => {});
765
827
  }
766
828
  await sleep(800);
767
829
  const shot = path.join(cfg.work, `inspect-after-${n}.jpg`);
@@ -804,10 +866,14 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
804
866
  const zoomTarget = s.zoom?.target && !isPoint(s.zoom.target);
805
867
  const startBox = zoomTarget ? await boxNow() : null;
806
868
  await runAll(s.actions, `step ${i + 1}`);
869
+ // a scroll still running (the page's own smooth scrolling) ends in this step, not in the next one's first shot
870
+ await page.evaluate(() => window.__demoScrollIdle()).catch(() => {});
807
871
  const end = now(); // before clearing: a step that is held for its narration keeps its marks
808
- await spotOff();
809
- // docs clips start where the step starts: clear at once, so no fading callout leaks into the next clip
810
- await page.evaluate((i) => { window.__demoUnmark(i); window.__demoUnnote(i); }, cfg.clips.length > 0).catch(() => {});
872
+ // docs clips start where the step starts, and a hidden step is cut: clear at once, so no fading
873
+ // callout leaks into the next shot
874
+ const instant = cfg.clips.length > 0 || !!s.hidden;
875
+ await spotOff(instant);
876
+ await page.evaluate((i) => { window.__demoUnmark(i); window.__demoUnnote(i); }, instant).catch(() => {});
811
877
  let zoomBox = null;
812
878
  if (s.zoom?.target) {
813
879
  if (isPoint(s.zoom.target)) {
package/src/voice.mjs CHANGED
@@ -118,8 +118,29 @@ function cached(cfg, line) {
118
118
  return { text, file: path.join(dir, `${hash}.${ext}`), zh, qwen };
119
119
  }
120
120
 
121
- // Returns stereo Float32 PCM at 48 kHz with leading/trailing silence trimmed.
122
- export async function speak(cfg, line, { log = console.log } = {}) {
121
+ // A line's sentences, so each is spoken on its own with a pause after it (TTS engines run the
122
+ // sentences of one line together with barely a breath). Latin text breaks after . ! ? when a capital,
123
+ // digit or quote follows, but not after e.g., Dr., a lone initial (N. P. X.), 2.5 or myapp.com;
124
+ // Chinese and Japanese after 。!?.
125
+ const ABBREV = /^(e\.g|i\.e|etc|vs|approx|incl|cf|Mr|Mrs|Ms|Dr|Prof|St|No|Fig)\.$/i;
126
+ export function sentences(text) {
127
+ const s = String(text).replace(/\s+/g, ' ').trim(), out = [];
128
+ const re = /([.!?…]+["'”’)\]]*) +(?=["'“‘(\[]?[\p{Lu}\p{N}\p{Lo}])|([。!?]+[”’」』))]*) */gu;
129
+ let from = 0;
130
+ for (let m; (m = re.exec(s));) {
131
+ if (m[1]) {
132
+ const word = s.slice(from, m.index + 1).match(/(\S+)$/)?.[1] ?? '';
133
+ if (ABBREV.test(word) || /^\p{Lu}\.$/u.test(word)) continue;
134
+ }
135
+ const piece = s.slice(from, m.index + (m[1] ?? m[2]).length).trim();
136
+ if (piece) out.push(piece);
137
+ from = re.lastIndex;
138
+ }
139
+ if (s.slice(from).trim()) out.push(s.slice(from).trim());
140
+ return out;
141
+ }
142
+ // One line as stereo Float32 PCM at 48 kHz, leading and trailing silence trimmed.
143
+ async function speakLine(cfg, line, log) {
123
144
  const v = cfg.voice;
124
145
  const { text, file, zh, qwen } = cached(cfg, line);
125
146
  if (!fs.existsSync(file)) {
@@ -135,6 +156,80 @@ export async function speak(cfg, line, { log = console.log } = {}) {
135
156
  return new Float32Array(pcm.buffer, pcm.byteOffset, pcm.length / 4);
136
157
  }
137
158
 
159
+ // Where a line's sentences meet in its audio. Every break in the text (sentence ends, and commas,
160
+ // which voices pause at too) gets an expected time from the letters spoken before it; each quiet
161
+ // stretch belongs to the break it is nearest, and a sentence end takes the longest stretch that is
162
+ // its own, so a long pause at a comma is never mistaken for it. [start, end] in samples per channel,
163
+ // or null when the voice runs straight on (no quiet there to widen).
164
+ const WIN = SR / 100; // 10 ms
165
+ const letters = (s) => (s.match(/[\p{L}\p{N}]/gu) ?? []).length;
166
+ export function sentenceGaps(pcm, sents) {
167
+ const n = Math.floor(pcm.length / 2 / WIN), env = new Float32Array(n);
168
+ let peak = 0;
169
+ for (let w = 0; w < n; w++) {
170
+ let e = 0;
171
+ for (let i = w * WIN; i < (w + 1) * WIN; i++) e = Math.max(e, Math.abs(pcm[2 * i]), Math.abs(pcm[2 * i + 1]));
172
+ env[w] = e; peak = Math.max(peak, e);
173
+ }
174
+ const quiet = [], thr = peak * 0.02;
175
+ for (let w = 0; w < n;) {
176
+ if (env[w] >= thr) { w++; continue; }
177
+ const a = w;
178
+ while (w < n && env[w] < thr) w++;
179
+ if (w - a >= 4 && a > 0 && w < n) quiet.push([a, w]); // ≥ 40 ms, not the trimmed ends
180
+ }
181
+ // breaks in the text, at the share of letters spoken before them: { at, end: k } for the end of sentence k
182
+ const total = letters(sents.join(' ')) || 1, breaks = [];
183
+ let before = 0;
184
+ for (const [k, s] of sents.entries()) {
185
+ for (const m of s.matchAll(/[,;:,、;:]/g)) breaks.push({ at: (before + letters(s.slice(0, m.index))) / total });
186
+ before += letters(s);
187
+ if (k < sents.length - 1) breaks.push({ at: before / total, end: k });
188
+ }
189
+ const mine = sents.slice(0, -1).map(() => []);
190
+ for (const q of quiet) {
191
+ const mid = (q[0] + q[1]) / 2 / n;
192
+ const b = breaks.reduce((x, y) => (Math.abs(y.at - mid) < Math.abs(x.at - mid) ? y : x));
193
+ if (b.end != null && Math.abs(b.at - mid) < 0.25) mine[b.end].push(q);
194
+ }
195
+ let after = 0;
196
+ return mine.map((qs) => {
197
+ const best = qs.filter((q) => q[0] >= after).reduce((x, q) => (!x || q[1] - q[0] > x[1] - x[0] ? q : x), null);
198
+ if (!best) return null;
199
+ after = best[1];
200
+ return [best[0] * WIN, best[1] * WIN];
201
+ });
202
+ }
203
+
204
+ // A line as stereo Float32 PCM at 48 kHz, spoken in one go (so the voice keeps its flow from sentence
205
+ // to sentence), with each pause between sentences widened to at least `voice.pause` seconds: voices
206
+ // often run sentences together with barely a breath. parts: where each sentence is, in seconds (for
207
+ // subtitles).
208
+ export async function speak(cfg, line, { log = console.log } = {}) {
209
+ const raw = await speakLine(cfg, line, log), pause = cfg.voice.pause ?? 0;
210
+ const sents = pause > 0 ? sentences(line) : [];
211
+ if (sents.length < 2) return { pcm: raw, parts: [{ text: line, start: 0, end: raw.length / 2 / SR }] };
212
+ const gaps = sentenceGaps(raw, sents), want = Math.round(pause * SR);
213
+ const add = gaps.map((g) => (g ? Math.max(0, want - (g[1] - g[0])) : 0));
214
+ const pcm = new Float32Array(raw.length + 2 * add.reduce((a, b) => a + b, 0));
215
+ const parts = [];
216
+ let from = 0, to = 0, startAt = 0;
217
+ for (const [k, g] of gaps.entries()) {
218
+ if (!g) continue; // no quiet stretch: this sentence runs on into the next
219
+ const mid = Math.round((g[0] + g[1]) / 2);
220
+ pcm.set(raw.subarray(2 * from, 2 * mid), 2 * to);
221
+ to += mid - from; from = mid;
222
+ const end = to - (mid - g[0]);
223
+ to += add[k]; // silence: the array is zero-filled
224
+ parts.push({ text: sents.slice(parts.length ? parts.at(-1).next : 0, k + 1).join(' '), start: startAt / SR, end: end / SR, next: k + 1 });
225
+ startAt = to + (g[1] - mid);
226
+ }
227
+ pcm.set(raw.subarray(2 * from), 2 * to);
228
+ const last = parts.length ? parts.at(-1).next : 0;
229
+ parts.push({ text: sents.slice(last).join(' '), start: startAt / SR, end: pcm.length / 2 / SR, next: sents.length });
230
+ return { pcm, parts: parts.map(({ next, ...p }) => p) };
231
+ }
232
+
138
233
  // Speed a clip up (factor > 1) with ffmpeg's pitch-preserving atempo.
139
234
  export function tempo(pcm, factor) {
140
235
  if (!(factor >= 0.5 && factor <= 100)) throw new Error(`internal: speed factor ${factor} is outside what ffmpeg's atempo accepts (0.5–100)`);
@@ -146,6 +241,7 @@ export function tempo(pcm, factor) {
146
241
  // Synthesize every narration line up front and return their lengths (seconds),
147
242
  // so `duration: auto` steps can be timed to the voice.
148
243
  export async function measureVoice(cfg, { log = console.log } = {}) {
244
+ const seconds = ({ pcm }) => pcm.length / 2 / SR;
149
245
  const lens = { steps: {}, endCard: null, intro: null, cards: {} };
150
246
  if (!cfg.voice) return lens;
151
247
  if (cfg.voice.fallback) log(`note: ${cfg.voice.fallback}`);
@@ -157,10 +253,10 @@ export async function measureVoice(cfg, { log = console.log } = {}) {
157
253
  await (zh ? synthZh : synthQwen)(cfg.voice, [...todo.values()], { log });
158
254
  }
159
255
  for (const [i, s] of cfg.steps.entries()) {
160
- if (s.say && !s.hidden) lens.steps[i] = (await speak(cfg, s.say, { log })).length / 2 / SR;
161
- if (s.card?.say && !s.hidden) lens.cards[i] = (await speak(cfg, s.card.say, { log })).length / 2 / SR;
256
+ if (s.say && !s.hidden) lens.steps[i] = seconds(await speak(cfg, s.say, { log }));
257
+ if (s.card?.say && !s.hidden) lens.cards[i] = seconds(await speak(cfg, s.card.say, { log }));
162
258
  }
163
- if (cfg.intro?.say) lens.intro = (await speak(cfg, cfg.intro.say, { log })).length / 2 / SR;
164
- if (cfg.endCard?.say) lens.endCard = (await speak(cfg, cfg.endCard.say, { log })).length / 2 / SR;
259
+ if (cfg.intro?.say) lens.intro = seconds(await speak(cfg, cfg.intro.say, { log }));
260
+ if (cfg.endCard?.say) lens.endCard = seconds(await speak(cfg, cfg.endCard.say, { log }));
165
261
  return lens;
166
262
  }