storyshot 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -17
- package/package.json +1 -1
- package/skills/storyshot/SKILL.md +76 -12
- package/src/audio.mjs +9 -4
- package/src/check.mjs +40 -2
- package/src/compose.mjs +5 -0
- package/src/config.mjs +15 -12
- package/src/doctor.mjs +2 -2
- package/src/init.mjs +10 -2
- package/src/inspect.mjs +13 -5
- package/src/qwen-tts.mjs +30 -9
- package/src/qwen_tts.py +129 -13
- package/src/record.mjs +99 -33
- package/src/voice.mjs +102 -6
package/README.md
CHANGED
|
@@ -153,7 +153,7 @@ Rendering is free and runs locally. Only the voiceover can cost money:
|
|
|
153
153
|
|---|---|---|
|
|
154
154
|
| none | free | captions only |
|
|
155
155
|
| `kokoro` | free | local, English and Mandarin Chinese (`zf_001`); downloads a ~90 MB model on first use (Chinese: ~130 MB, and needs Python 3.10–3.12) |
|
|
156
|
-
| `qwen` | free | local, Apple Silicon Macs only; the most natural free Chinese voice (`dylan`, `serena`);
|
|
156
|
+
| `qwen` | free | local, Apple Silicon Macs only; the most natural free Chinese voice (`dylan`, `serena`); every take is checked by a speech recognizer and made again if it comes out wrong; downloads ~3.3 GB of models and needs Python 3.10–3.12 |
|
|
157
157
|
| ElevenLabs | your credits | the most natural voice; many languages including Chinese. Put `ELEVENLABS_API_KEY` in `.env`. |
|
|
158
158
|
| OpenAI | your credits | `OPENAI_API_KEY` |
|
|
159
159
|
|
|
@@ -191,7 +191,7 @@ from `storyshot guide`.
|
|
|
191
191
|
| `inspect_page` | Verified click targets and a screenshot, for a URL or for the state after step N of a storyboard. Reports login walls. |
|
|
192
192
|
| `login` | Opens a browser window where the **user** logs in, and saves the session. |
|
|
193
193
|
| `init_storyboard` | Writes a starter storyboard with verified targets. |
|
|
194
|
-
| `check_storyboard` | Validates in about a second: typos with "did you mean", action and target shapes, `TODO`s, keys, the session file. |
|
|
194
|
+
| `check_storyboard` | Validates in about a second: typos with "did you mean", action and target shapes, `TODO`s, keys, the session file, and narration that sounds machine-written ("not just X, but Y", "seamless", 赋能…) or runs on without a breath. |
|
|
195
195
|
| `render_video` / `render_status` | Renders in the background and returns the MP4 path, warnings, a per-step speed log and the preview image. If the render isn't done within 45 s, it returns a `job_id`, so client tool timeouts never kill a render. |
|
|
196
196
|
|
|
197
197
|
Any other MCP client: run `npx -y storyshot mcp` (stdio), and point the agent at the playbook (`storyshot guide`). When run from a git
|
|
@@ -248,7 +248,7 @@ viewport: { width: 1600, height: 900 } # CSS px the site is laid out at
|
|
|
248
248
|
output: { width: 1920, height: 1080, fps: 30 } # same aspect ratio; use 1080x1920 with a 900x1600 viewport for vertical
|
|
249
249
|
theme: { accent: "#d9f75b", ink: "#0f172a", font: Inter }
|
|
250
250
|
music: { bpm: 120, volume: 1 } # or { file: track.mp3 }, or false
|
|
251
|
-
sfx: true # click / whoosh / riser sounds; clicks = no whooshes on cards and page changes (tutorial and course default); false = none
|
|
251
|
+
sfx: true # click / whoosh / riser sounds; clicks = no whooshes on cards and page changes, and no rising sound under a riser step, only its chime (tutorial and course default); false = none
|
|
252
252
|
voice: { provider: elevenlabs, voice: hpp4J3VqNfWAUOO0d1Us } # optional voiceover, see below
|
|
253
253
|
storageState: auth.json # optional Playwright login state
|
|
254
254
|
ignoreHTTPSErrors: true # internal site with a self-signed HTTPS certificate (plain http:// needs nothing)
|
|
@@ -287,7 +287,7 @@ steps:
|
|
|
287
287
|
- click: { role: button, name: Try it }
|
|
288
288
|
- caption: AI does the work
|
|
289
289
|
duration: 1.3
|
|
290
|
-
sfx: riser # rising sound + chime (or `chime`)
|
|
290
|
+
sfx: riser # rising sound + chime (or `chime`); a tutorial or lesson plays only the chime
|
|
291
291
|
actions:
|
|
292
292
|
- waitFor: { text: Done, timeout: 120000 } # waiting is compressed automatically
|
|
293
293
|
- card: { kicker: Part 2, title: Read the results } # a section card before this step (tutorials); becomes a chapter
|
|
@@ -332,15 +332,15 @@ You can add `exact: true`, `last: true` or `nth: 2` to any of them.
|
|
|
332
332
|
| `waitForUrl` | `waitForUrl: "**/dashboard"` |
|
|
333
333
|
| `networkIdle` | `networkIdle: true` |
|
|
334
334
|
| `goto` | `goto: /pricing` |
|
|
335
|
-
| `scrollTo` | `scrollTo: { css: "#pricing", ms: 1200 }` |
|
|
335
|
+
| `scrollTo` | `scrollTo: { css: "#pricing", ms: 1200 }` scrolls the target to the top of the window, below a sticky header |
|
|
336
336
|
| `scroll` | `scroll: 600` |
|
|
337
337
|
| `dismiss` | `dismiss: { role: button, name: Close }` (clicks only if visible) |
|
|
338
338
|
| `hide` | `hide: '[id*="QSI"], .chat-widget'` hides pop-ups, survey prompts, feedback tabs and chat widgets for the whole video, including ones that appear late (use it in `setup`) |
|
|
339
|
-
| `mark` | `mark: "automatic zoom, captions"` sweeps a highlighter over that phrase, even inside a paragraph (scrolling to it if
|
|
339
|
+
| `mark` | `mark: "automatic zoom, captions"` sweeps a highlighter over that phrase, even inside a paragraph (scrolling to it only if it's off screen, and below a sticky header). `mark: { css: "table tr:nth-child(3)" }` marks any target with a box. Options: `ms` (sweep time, default 700), `color` (default: theme accent), `box: true/false`. Marks stay until the step ends. |
|
|
340
340
|
| `unmark` | `unmark: true` clears the marks (and notes) earlier |
|
|
341
341
|
| `note` | `note: { target: { label: Database }, text: Curated proteins only }` puts a short label with an arrow next to the element, on the side with the most room (or `side: left/right/top/bottom`). It stays until the step ends and gets reading time (about 3 words a second, or `ms`). |
|
|
342
342
|
| `paste` | `paste: { target: { label: JSON }, text: '{"a": 1}' }` pastes text in one go through the real clipboard (with a ⌘V badge), so editors react as to a real paste; `paste: "..."` pastes into whatever has focus |
|
|
343
|
-
| `slide` | `slide: { target: { label: Quality }, to: 0.2 }` drags a slider to a position along its track (0 = start, 1 = end), found from the element, not pixels |
|
|
343
|
+
| `slide` | `slide: { target: { label: Quality }, to: 0.2 }` drags a slider to a position along its track (0 = start, 1 = end), found from the element, not pixels. `value: 3` drags it to that value instead, when the slider reports its range (an `<input type=range>`, or an ARIA slider such as Desmos's; `inspect` shows the range). The target can be the slider or its knob |
|
|
344
344
|
| `select` | `select: { target: { label: Database }, option: "UniProtKB/Swiss-Prot" }` picks an option in a dropdown (by its visible text, or part of it). `inspect` lists each dropdown's options. |
|
|
345
345
|
|
|
346
346
|
`ms` sets how long the cursor takes to travel to the target (`dragMs` for each
|
|
@@ -375,7 +375,7 @@ In-page anchor links (`href="#..."`) scroll smoothly instead of jumping.
|
|
|
375
375
|
|---|---|
|
|
376
376
|
| `target` | What to center on: a target, or a point `{ x: 0.5, y: 0.3 }`. If a step action used the same target, its position at that moment is used (so you can zoom on a button that disappears after the click). Otherwise it is measured at the start of the step when the zoom begins in the first half, or at the end of the step otherwise. |
|
|
377
377
|
| `scale` | Zoom factor (default 1.3). 1.2–1.5 works well. |
|
|
378
|
-
| `start`, `end` | Seconds from the start of the step. The camera eases in from 1× at `start` and is back at 1× at `end` (default: the step's end). `end`
|
|
378
|
+
| `start`, `end` | Seconds from the start of the step. The camera eases in from 1× at `start` and is back at 1× at `end` (default: the step's end). A zoom belongs to its step: an `end` past the step's end stops there, with a warning. |
|
|
379
379
|
| `ramp` | Ease-in / ease-out seconds: `0.5` or `[in, out]`. |
|
|
380
380
|
| `offset` | Moves the camera center from the target's center, in viewport CSS px. `{ y: -120 }` centers 120 px above the target. |
|
|
381
381
|
|
|
@@ -419,14 +419,20 @@ for Chinese tutorials, lessons and docs), `serena` (female, warm; the default fo
|
|
|
419
419
|
`vivian` (female, bright), `uncle_fu` (male, mellow), `eric` (male, Sichuan accent), and `ryan`, `aiden` (English), `ono_anna` (Japanese), `sohee` (Korean). On
|
|
420
420
|
first use Storyshot creates a venv in `~/.cache/storyshot/qwen-venv` (Python 3.10–3.12, about 400 MB)
|
|
421
421
|
and downloads the ~2.3 GB model into the Hugging Face cache. It speaks at about 2x real time on an M4 Pro
|
|
422
|
-
and peaks at about 8 GB of memory. The model samples its speech, so a
|
|
423
|
-
invented words or stops short. Storyshot
|
|
424
|
-
|
|
422
|
+
and peaks at about 8 GB of memory. The model samples its speech, so a take now and then garbles a few
|
|
423
|
+
words, drops one, runs on into invented words or stops short. Storyshot listens to every take with
|
|
424
|
+
[Qwen3-ASR 0.6B](https://huggingface.co/Qwen/Qwen3-ASR-0.6B) (Apache-2.0, a ~1 GB model downloaded on
|
|
425
|
+
first use) and compares what it hears with the line: Chinese by sound, so a homophone isn't a mistake, and
|
|
426
|
+
numbers digit by digit. A take that comes out wrong, or far too long or short, is made again with another
|
|
427
|
+
seed (up to 4 tries), and Storyshot warns, quoting what it heard, if a line still doesn't come out right.
|
|
428
|
+
This adds about 0.2 s per line. A scientific number like `2e-105` is spelled out in Chinese, because it sets the
|
|
425
429
|
model babbling. On other machines the same storyboard renders with `kokoro` (`zf_001`, or `draftVoice`),
|
|
426
430
|
with a note.
|
|
427
431
|
|
|
428
|
-
Other fields: `speed` (default 1), `volume` (default 1),
|
|
429
|
-
|
|
432
|
+
Other fields: `speed` (default 1), `volume` (default 1), `pause` (the shortest pause between the
|
|
433
|
+
sentences of a line, in seconds: 0.3 for promos, 0.6 for tutorials and lessons; a voice's own longer pause
|
|
434
|
+
stays, and `0` leaves the voice as it is),
|
|
435
|
+
and `pronounce`, a map of word → spoken form used only for TTS, e.g.
|
|
430
436
|
`pronounce: { pxGenius: P X Genius, PPTX: P P T X }`. Subtitles keep the real
|
|
431
437
|
spelling. Synthesized lines
|
|
432
438
|
are cached in `.storyshot/tts-cache/`, so `compose` only pays for lines you
|
|
@@ -459,11 +465,12 @@ usually shouldn't.
|
|
|
459
465
|
| `say` | Voiceover line played over the end card. |
|
|
460
466
|
| `emblem` | With `hyperframes`: one to three short monospace lines over the title (an alignment, a formula, a line of code); `false` for none. |
|
|
461
467
|
|
|
462
|
-
### HyperFrames cards
|
|
468
|
+
### HyperFrames cards
|
|
463
469
|
|
|
464
|
-
|
|
470
|
+
Tutorials and lessons render their intro, section and end cards with
|
|
465
471
|
[HyperFrames](https://hyperframes.heygen.com) (HeyGen's open-source HTML-to-video
|
|
466
|
-
renderer, Apache-2.0) for more motion
|
|
472
|
+
renderer, Apache-2.0) for more motion; `hyperframes: false` keeps the built-in
|
|
473
|
+
cards, and `hyperframes: true` turns them on for a promo. Two templates, picked by `purpose`:
|
|
467
474
|
|
|
468
475
|
- **lesson** (tutorials, courses, docs): the title lands word by word over a
|
|
469
476
|
slowly drifting, blurred view of the page the section leads into; section
|
|
@@ -485,7 +492,7 @@ hyperframes:
|
|
|
485
492
|
backdrop: false # lesson: plain background instead of the blurred page
|
|
486
493
|
```
|
|
487
494
|
|
|
488
|
-
It needs Node 22 or newer. On first use storyshot installs the pinned
|
|
495
|
+
It needs Node 22 or newer; on older Node, tutorials and lessons use the built-in cards. On first use storyshot installs the pinned
|
|
489
496
|
HyperFrames version into `~/.cache/storyshot/hyperframes/` (a local install
|
|
490
497
|
of the `hyperframes` package wins); its telemetry is switched off. Cards are
|
|
491
498
|
cached, so `compose` only re-renders the ones that changed (about 10–25 s
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "storyshot",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Turn a short YAML storyboard into a polished product demo video of your web app — smooth cursor, auto zoom, captions, music.",
|
|
5
5
|
"author": "boheling <boheling@gmail.com>",
|
|
6
6
|
"repository": {
|
|
@@ -92,8 +92,11 @@ frames or stock footage presented as a recording of their app.
|
|
|
92
92
|
best free Chinese voice is `voice: { provider: qwen }` (Qwen3-TTS: `dylan`,
|
|
93
93
|
a male teacher's voice, for tutorials and lessons; `serena`, female, for
|
|
94
94
|
promos; those are the defaults): clearer than
|
|
95
|
-
zf_001 with English words and letters, about 2x real time,
|
|
96
|
-
on first use.
|
|
95
|
+
zf_001 with English words and letters, about 2x real time, ~3.3 GB of
|
|
96
|
+
models on first use. Storyshot listens to each take and makes it again if
|
|
97
|
+
it comes out garbled; a `⚠ qwen: … sounds like "…"` warning means a line
|
|
98
|
+
still came out wrong after 4 tries: rephrase it (spell out a number, say
|
|
99
|
+
`一格` for `1 格`). Elsewhere a `qwen` storyboard renders with kokoro. For other
|
|
97
100
|
languages, use ElevenLabs.
|
|
98
101
|
3. **Login.** If the flow needs a logged-in page:
|
|
99
102
|
- Tell the user a browser window will open and that they should log in
|
|
@@ -184,14 +187,18 @@ small controls are readable. Your job is the structure:
|
|
|
184
187
|
`note` the field the reader must not miss. Set the article's heading and
|
|
185
188
|
first paragraph with `article: { title: …, intro: … }`. Downloads show a
|
|
186
189
|
"Downloaded" bubble; `paste` and copy-to-clipboard work.
|
|
187
|
-
- **
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
190
|
+
- **Animated cards that help explain.** Tutorials and lessons render their
|
|
191
|
+
cards with HyperFrames by default (Node 22+; it installs itself on first
|
|
192
|
+
use): titles land word by word, and each bullet lights up when the `say`
|
|
193
|
+
line names it, so write the bullets in the narration's own words. Use them:
|
|
194
|
+
a concept card for every idea that isn't on screen (the idea as the title,
|
|
195
|
+
its 2–4 steps as bullets, a formula or a line of code as `emblem`), a
|
|
196
|
+
question card for each check, and a recap end card. For a subject with its
|
|
197
|
+
own material, give every title a backdrop from it:
|
|
191
198
|
`hyperframes: { band: <a sequence or formula>, emblem: [<1–3 short lines>] }`.
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
199
|
+
For a promo, `hyperframes: true` gives the launch template with hand-drawn
|
|
200
|
+
accents (`hand: true` for hand lettering). `hyperframes: false` keeps the
|
|
201
|
+
built-in cards.
|
|
195
202
|
- **End with a recap**: `end_card` with `bullets` listing the steps or rules
|
|
196
203
|
to remember, and a `say` that repeats them.
|
|
197
204
|
- **Narrating real results.** You can't know the numbers (scores, counts,
|
|
@@ -206,6 +213,59 @@ small controls are readable. Your job is the structure:
|
|
|
206
213
|
don't pad: if a part is long, split it into more parts rather than
|
|
207
214
|
speeding it up.
|
|
208
215
|
|
|
216
|
+
### Writing the narration
|
|
217
|
+
|
|
218
|
+
Viewers notice three things first: lines that run together, phrasing nobody
|
|
219
|
+
says out loud, and an explanation that jumps around. Write the `say:` lines as
|
|
220
|
+
a script before the steps around them, then read them in order on their own.
|
|
221
|
+
They should sound like one person talking a friend through it.
|
|
222
|
+
|
|
223
|
+
**Say it like a person.**
|
|
224
|
+
- Short sentences, one idea each: about 15 words at most in English, 25
|
|
225
|
+
characters in Chinese. If you would run out of breath, split it.
|
|
226
|
+
- Spoken words: contractions ("you'll", "it's"), "you", present tense, active
|
|
227
|
+
voice. In Chinese, 口语 rather than 书面语: "点一下这里", not "进行点击操作".
|
|
228
|
+
- Use the exact labels on screen, and the same term every time (no synonyms
|
|
229
|
+
for variety).
|
|
230
|
+
- Each sentence gets a breath after it: Storyshot widens the pause between
|
|
231
|
+
sentences to at least `voice.pause` (0.3 s for `pitch`, 0.6 s for tutorials
|
|
232
|
+
and lessons). Commas get no extra pause, so split long lines into sentences
|
|
233
|
+
rather than chaining clauses with commas.
|
|
234
|
+
- For fixed durations, budget about 2.5 words a second in English and 4
|
|
235
|
+
characters a second in Chinese, and leave silence while the screen is busy.
|
|
236
|
+
|
|
237
|
+
**Cut the phrases that mark text as machine-written** (`check` flags the
|
|
238
|
+
common ones):
|
|
239
|
+
- Contrasts that only add weight: "not just X, but Y", "it's not X, it's Y".
|
|
240
|
+
- Inflated words: seamless, effortless, powerful, robust, unlock, elevate,
|
|
241
|
+
supercharge, revolutionize, leverage, cutting-edge, game-changer, delve;
|
|
242
|
+
"in today's fast-paced world". In Chinese: 赋能, 助力, 一站式, 全方位, 极致,
|
|
243
|
+
无缝, 轻松搞定, 不仅……更……, 让我们一起.
|
|
244
|
+
- Openers and closers that say nothing: "Let's dive in", "Ever wondered…?",
|
|
245
|
+
"And that's it!", "Simple as that."
|
|
246
|
+
- Three adjectives in a row, line after line ("fast, simple and powerful").
|
|
247
|
+
- Filler that talks down to a stuck viewer: basically, simply, just, easily.
|
|
248
|
+
- Dashes (—) in `say:` lines. Voices read them as a run-on; use a period.
|
|
249
|
+
|
|
250
|
+
**Explain in an order the viewer can follow** (tutorials and lessons):
|
|
251
|
+
1. The question or the goal ("How does b move the line?").
|
|
252
|
+
2. A concrete case on screen first: change the value, show the result.
|
|
253
|
+
3. Then name the idea, in plain words, the first time it appears.
|
|
254
|
+
4. Why it works, in one or two sentences, with a concept card if it isn't on
|
|
255
|
+
screen (its bullets light up as you name them; a formula goes in `emblem`).
|
|
256
|
+
5. A check: a question card, then the step that reveals the answer.
|
|
257
|
+
6. The recap, in the same words as before.
|
|
258
|
+
|
|
259
|
+
Every step answers the question the step before it raised. Link them out loud:
|
|
260
|
+
"Now that b has moved the line up, let's see what m does." When the camera
|
|
261
|
+
has to go somewhere else on the page, say why first ("The results are further
|
|
262
|
+
down, in the table."). If a line only makes sense with the video, or the
|
|
263
|
+
script reads as a list of disconnected facts, reorder the steps before you
|
|
264
|
+
render.
|
|
265
|
+
|
|
266
|
+
(These rules borrow from [humanizer](https://github.com/blader/humanizer) and
|
|
267
|
+
[stop-slop](https://github.com/hardikpandya/stop-slop), both MIT.)
|
|
268
|
+
|
|
209
269
|
Rules of thumb from real renders (promos and tutorials):
|
|
210
270
|
|
|
211
271
|
- **One idea per step.**
|
|
@@ -225,11 +285,15 @@ Rules of thumb from real renders (promos and tutorials):
|
|
|
225
285
|
`voice.pronounce` (e.g. `pxGenius: P X Genius`, `AI: A I`) so subtitles stay
|
|
226
286
|
correct. For fixed durations, budget about 2.5 words per second.
|
|
227
287
|
- **Isolate long waits** (AI processing, uploads) in their own step: a
|
|
228
|
-
`waitFor` the finished state
|
|
229
|
-
`duration: 1.0–1.5`.
|
|
288
|
+
`waitFor` the finished state. In `pitch` mode add `sfx: riser` and
|
|
289
|
+
`duration: 1.0–1.5`. In tutorials and lessons, say what is happening
|
|
290
|
+
instead; at most `sfx: chime` when the result arrives (a riser there plays
|
|
291
|
+
only its chime). Waiting is compressed automatically.
|
|
230
292
|
- **Zoom sparingly**, on 2–4 payoff moments (headline, clicked button, result,
|
|
231
293
|
price), with `scale: 1.2–1.5`.
|
|
232
|
-
- **Show the signature interaction.** Use `
|
|
294
|
+
- **Show the signature interaction.** Use `slide` for sliders:
|
|
295
|
+
`slide: { target: <the slider inspect lists>, value: 3 }` lands on that value
|
|
296
|
+
(`to: 0..1` for sliders that don't report a range). Use `drag` for before/after
|
|
233
297
|
views (`drag: { from: { x: 0.57, y: 0.5 }, by: [{ x: -300 }, { x: 400 }] }`;
|
|
234
298
|
`from` can be a target or a point). Use `type` for inputs: `type: y=x^2`
|
|
235
299
|
types into whatever has focus (for editors and canvases with hidden inputs),
|
package/src/audio.mjs
CHANGED
|
@@ -108,7 +108,8 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
|
|
|
108
108
|
return filt(rand(), 400 + 5000 * Math.sin(Math.PI * p), 1.6).bp * Math.pow(Math.sin(Math.PI * p), 2);
|
|
109
109
|
}, 0.5);
|
|
110
110
|
}
|
|
111
|
-
|
|
111
|
+
// the rising sound under a wait is for promos; tutorials and lessons (sfx: clicks) keep only its chime
|
|
112
|
+
for (const [a, b] of cfg.sfx.transitions ? events.risers : []) {
|
|
112
113
|
const len = b - a, filt = svf();
|
|
113
114
|
let ph = 0;
|
|
114
115
|
add(sfx, a, len, (t) => {
|
|
@@ -133,12 +134,15 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
|
|
|
133
134
|
if (cfg.voice && events.voice?.length) {
|
|
134
135
|
let prevEnd = 0;
|
|
135
136
|
for (const line of events.voice) {
|
|
136
|
-
let pcm = await speak(cfg, line.text, { log });
|
|
137
|
+
let { pcm, parts } = await speak(cfg, line.text, { log });
|
|
137
138
|
let dur = pcm.length / 2 / SR;
|
|
138
139
|
const start = Math.max(line.start, prevEnd + 0.12), room = line.until - start - 0.1;
|
|
139
140
|
let sped = 1;
|
|
140
141
|
// a line pushed past its window by the one before has no room left (room <= 0): it plays late, at normal speed
|
|
141
|
-
if (dur > room && room > 0 && dur / room <= 1.2) {
|
|
142
|
+
if (dur > room && room > 0 && dur / room <= 1.2) {
|
|
143
|
+
sped = dur / room; pcm = tempo(pcm, sped); dur = pcm.length / 2 / SR;
|
|
144
|
+
parts = parts.map((p) => ({ ...p, start: p.start / sped, end: p.end / sped }));
|
|
145
|
+
}
|
|
142
146
|
else if (dur > room) warn(room > 0
|
|
143
147
|
? `${line.where}: narration is ${dur.toFixed(1)}s but only ${room.toFixed(1)}s until ${line === events.voice.at(-1) ? 'the video ends' : 'the next line'} — shorten "${line.text}" or give the step more duration`
|
|
144
148
|
: `${line.where}: the narration before it runs ${(start - line.start).toFixed(1)}s past where this line should start, so "${line.text}" plays late — shorten the earlier lines or give their steps more duration`);
|
|
@@ -154,7 +158,8 @@ export async function soundtrack(cfg, events, silentVideo, { log = console.log }
|
|
|
154
158
|
voiceOn[s0 + i] = 1;
|
|
155
159
|
}
|
|
156
160
|
prevEnd = start + dur;
|
|
157
|
-
|
|
161
|
+
// subtitles per sentence, so the pauses between them don't throw off the word timing
|
|
162
|
+
for (const p of parts) narration.push({ start: start + p.start, end: Math.min(start + p.end, start + dur), text: p.text });
|
|
158
163
|
}
|
|
159
164
|
}
|
|
160
165
|
|
package/src/check.mjs
CHANGED
|
@@ -5,6 +5,34 @@ import { loadConfig } from './config.mjs';
|
|
|
5
5
|
import { gitIgnored } from './auth.mjs';
|
|
6
6
|
import { isZhVoice, zhVoiceStatus } from './kokoro-zh.mjs';
|
|
7
7
|
import { qwenStatus } from './qwen-tts.mjs';
|
|
8
|
+
import { sentences } from './voice.mjs';
|
|
9
|
+
|
|
10
|
+
// Phrases that mark narration as machine-written, and sentences too long to say in one breath.
|
|
11
|
+
const TELLS = [
|
|
12
|
+
/\bnot (?:just|only|merely) [^.,;!?]{1,40}?,? but(?: also)?\b/i, // "not just X, but Y"
|
|
13
|
+
/\bit'?s not (?:about )?[^.,;!?]{1,30}[,;—–-]+ it'?s\b/i, // "it's not X, it's Y"
|
|
14
|
+
/\b(?:seamless(?:ly)?|effortless(?:ly)?|supercharg\w*|revolutioni[sz]\w*|game[- ]chang\w*|cutting[- ]edge|elevat(?:e|es|ed|ing)|leverag(?:e|es|ed|ing)|delv(?:e|es|ing))\b/i,
|
|
15
|
+
/\bin today'?s (?:fast-paced|digital|modern)\b/i,
|
|
16
|
+
/\blet'?s dive in\b|\bever wondered\b|\bsimple as that\b|\band that'?s it\b/i,
|
|
17
|
+
/\b(?:basically|simply)\b/i, // ("easily" is too often literal: "binds more easily")
|
|
18
|
+
/[—–]/,
|
|
19
|
+
/赋能|助力|一站式|全方位|极致|无缝|轻松搞定|让我们一起|不仅[^。!?]{1,20}更/,
|
|
20
|
+
];
|
|
21
|
+
export function narrationLint(text) {
|
|
22
|
+
const t = String(text).replace(/\*\*/g, ''), tells = [];
|
|
23
|
+
for (const re of TELLS) {
|
|
24
|
+
const g = new RegExp(re.source, re.flags.includes('g') ? re.flags : `${re.flags}g`);
|
|
25
|
+
for (const m of t.matchAll(g)) if (!tells.includes(m[0].trim())) tells.push(m[0].trim());
|
|
26
|
+
}
|
|
27
|
+
let long = null;
|
|
28
|
+
for (const s of sentences(t)) {
|
|
29
|
+
const cjk = (s.match(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g) ?? []).length;
|
|
30
|
+
const words = s.replace(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g, ' ').split(/\s+/).filter((w) => /\w/.test(w)).length;
|
|
31
|
+
if (cjk > 40 && !long) long = `${cjk}-character`;
|
|
32
|
+
else if (!cjk && words > 25 && !long) long = `${words}-word`;
|
|
33
|
+
}
|
|
34
|
+
return { tells, long };
|
|
35
|
+
}
|
|
8
36
|
|
|
9
37
|
const TOP = ['name', 'url', 'out', 'purpose', 'viewport', 'output', 'theme', 'music', 'voice', 'sfx', 'storageState', 'ignoreHTTPSErrors', 'intro', 'frame', 'pace', 'follow', 'article', 'cards', 'lang', 'colorScheme',
|
|
10
38
|
'scrollOffset', 'captionPosition', 'captions', 'highlight', 'stepNumbers', 'subtitles', 'chapters', 'clips', 'maxWait', 'setup', 'steps', 'end_card', 'hyperframes'];
|
|
@@ -107,7 +135,7 @@ export function check(file, { draft = false } = {}) {
|
|
|
107
135
|
if (type === 'target') target(v, `${where} (${kind})`);
|
|
108
136
|
if (type === 'mark' && typeof v !== 'string') target(v, `${where} (mark)`); // a string is a phrase to highlight
|
|
109
137
|
if (type === 'slide') {
|
|
110
|
-
if (!v?.target || typeof v?.to !== 'number' || v.to < 0 || v.to > 1) errors.push(`${where}: slide needs { target: <slider>, to: 0..1 } (0 = start of the track, 1 = end)`);
|
|
138
|
+
if (!v?.target || (typeof v?.value !== 'number' && (typeof v?.to !== 'number' || v.to < 0 || v.to > 1))) errors.push(`${where}: slide needs { target: <slider>, to: 0..1 } (0 = start of the track, 1 = end) or { target: <slider>, value: <number> }`);
|
|
111
139
|
else target(v.target, `${where} (slide.target)`);
|
|
112
140
|
}
|
|
113
141
|
if (type === 'select') {
|
|
@@ -146,7 +174,7 @@ export function check(file, { draft = false } = {}) {
|
|
|
146
174
|
// estimate from its length; the render measures the real voice)
|
|
147
175
|
if (cfg.voice) {
|
|
148
176
|
const words = (t) => { const x = String(t).replace(/\*\*/g, ''); const c = (x.match(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g) ?? []).length; return { c, w: x.replace(/[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af]/g, ' ').split(/\s+/).filter(Boolean).length }; };
|
|
149
|
-
const secs = (t) => { const { c, w } = words(t); return (c / 4.5 + w / 2.8) / (cfg.voice.speed ?? 1); };
|
|
177
|
+
const secs = (t) => { const { c, w } = words(t); return (c / 4.5 + w / 2.8) / (cfg.voice.speed ?? 1) + (sentences(t).length - 1) * (cfg.voice.pause ?? 0); };
|
|
150
178
|
const vis = visible.map((s) => ({ s, i: cfg.steps.indexOf(s) }));
|
|
151
179
|
for (const [k, { s, i }] of vis.entries()) {
|
|
152
180
|
if (!s.say) continue;
|
|
@@ -158,6 +186,16 @@ export function check(file, { draft = false } = {}) {
|
|
|
158
186
|
if (need > room * 1.4 && need - room > 1) warnings.push(`step ${i + 1}: the say: line takes about ${need.toFixed(1)}s to speak but has ${room.toFixed(1)}s before ${nextK < 0 ? 'the video ends' : 'the next line'} — shorten it or raise duration (the lines after it would play late)`);
|
|
159
187
|
}
|
|
160
188
|
}
|
|
189
|
+
// spoken lines that read as machine-written, or run on without a breath (the skill's "Writing the narration")
|
|
190
|
+
if (cfg.voice) {
|
|
191
|
+
const lines = [...visible.flatMap((s) => [[s.say, `step ${cfg.steps.indexOf(s) + 1}`], [s.card?.say, `step ${cfg.steps.indexOf(s) + 1} card`]]),
|
|
192
|
+
[cfg.intro?.say, 'intro'], [cfg.endCard?.say, 'end card']].filter(([t]) => t);
|
|
193
|
+
for (const [text, where] of lines) {
|
|
194
|
+
const { tells, long } = narrationLint(text);
|
|
195
|
+
if (tells.length) warnings.push(`${where}: the narration sounds machine-written (${tells.map((x) => `"${x}"`).join(', ')}) — say it plainly, as you would out loud`);
|
|
196
|
+
if (long) warnings.push(`${where}: a ${long} sentence — split it, so the voice gets a breath between the parts`);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
161
199
|
// the Chinese Kokoro voice needs a Python for its G2P: say so now, not after the recording
|
|
162
200
|
const v = draft && cfg.voice ? loadConfig(file, { draft }).voice : cfg.voice;
|
|
163
201
|
if (v?.provider === 'kokoro' && isZhVoice(v.voice)) {
|
package/src/compose.mjs
CHANGED
|
@@ -235,6 +235,11 @@ export async function compose(cfg, { log = console.log, voiceLens = { steps: {},
|
|
|
235
235
|
g.zoomBox = null;
|
|
236
236
|
} else if (z && !g.zoomBox) warn(`step ${g.i + 1}: zoom target not found on the page, zoom skipped`);
|
|
237
237
|
if (!z || !g.zoomBox) continue;
|
|
238
|
+
{
|
|
239
|
+
// measured off screen (e.g. before the step scrolled to it): the camera can only frame what's in the window
|
|
240
|
+
const b = g.zoomBox, cx = b.x + b.width / 2, cy = b.y + b.height / 2, { width: VW, height: VH } = cfg.viewport;
|
|
241
|
+
if (cx < 0 || cx > VW || cy < 0 || cy > VH) warn(`step ${g.i + 1}: the zoom target was off screen when it was measured (${cy > VH ? `${Math.round(cy - VH)}px below` : cy < 0 ? `${Math.round(-cy)}px above` : 'beside'} the window), so the zoom frames the edge of the page instead — zoom on something an action in this step moves to, or use a point target`);
|
|
242
|
+
}
|
|
238
243
|
g.zoomPeak = g.o0 + (z.start ?? 0) + (Array.isArray(z.ramp) ? z.ramp[0] : z.ramp ?? 0.5) + 0.1;
|
|
239
244
|
const b = g.zoomBox, off = z.offset ?? {};
|
|
240
245
|
const c = [(b.x + b.width / 2 + (off.x ?? 0)) * K, (b.y + b.height / 2 + (off.y ?? 0)) * K];
|
package/src/config.mjs
CHANGED
|
@@ -31,10 +31,10 @@ export function loadEnv(dir) {
|
|
|
31
31
|
|
|
32
32
|
// What the video is for sets sensible defaults; anything in the YAML overrides them.
|
|
33
33
|
export const PURPOSES = {
|
|
34
|
-
pitch: { duration: null, music: { bpm: 120, volume: 1 }, sfx: true, highlight: false, stepNumbers: false, subtitles: false, chapters: false, clips: null, frame: 'aurora', pace: 'fast', follow: false },
|
|
35
|
-
tutorial: { duration: 'auto', music: { bpm: 96, volume: 0.35 }, sfx: 'clicks', highlight: true, stepNumbers: true, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true },
|
|
36
|
-
course: { duration: 'auto', music: false, sfx: 'clicks', highlight: true, stepNumbers: false, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true },
|
|
37
|
-
docs: { duration: 'auto', music: false, sfx: false, highlight: true, stepNumbers: true, subtitles: false, chapters: false, clips: ['gif', 'png'], frame: false, pace: 'normal', follow: false },
|
|
34
|
+
pitch: { duration: null, music: { bpm: 120, volume: 1 }, sfx: true, highlight: false, stepNumbers: false, subtitles: false, chapters: false, clips: null, frame: 'aurora', pace: 'fast', follow: false, hyperframes: false },
|
|
35
|
+
tutorial: { duration: 'auto', music: { bpm: 96, volume: 0.35 }, sfx: 'clicks', highlight: true, stepNumbers: true, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true, hyperframes: true },
|
|
36
|
+
course: { duration: 'auto', music: false, sfx: 'clicks', highlight: true, stepNumbers: false, subtitles: true, chapters: true, clips: null, frame: 'studio', pace: 'slow', follow: true, hyperframes: true },
|
|
37
|
+
docs: { duration: 'auto', music: false, sfx: false, highlight: true, stepNumbers: true, subtitles: false, chapters: false, clips: ['gif', 'png'], frame: false, pace: 'normal', follow: false, hyperframes: false },
|
|
38
38
|
};
|
|
39
39
|
|
|
40
40
|
// Where storyboards and their videos go, and how they're named: <site>-<kind>[-<topic>].yaml →
|
|
@@ -43,9 +43,10 @@ export const PURPOSES = {
|
|
|
43
43
|
// follow each action and to read its result before the next one starts.
|
|
44
44
|
export const PACES = {
|
|
45
45
|
// cursor ms typing ms/char beat after a click (ms) hold after narration (s) waits shown up to (s) voice speed
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
// shortest pause between the sentences of one line (s)
|
|
47
|
+
fast: { cursor: 700, type: 55, beat: 0, tail: 0.4, maxWait: 1.0, voiceSpeed: 1, sentencePause: 0.3 },
|
|
48
|
+
normal: { cursor: 900, type: 70, beat: 350, tail: 0.8, maxWait: 1.5, voiceSpeed: 1, sentencePause: 0.45 },
|
|
49
|
+
slow: { cursor: 1200, type: 95, beat: 800, tail: 1.6, maxWait: 3.0, voiceSpeed: 0.92, sentencePause: 0.6 },
|
|
49
50
|
};
|
|
50
51
|
|
|
51
52
|
export const FRAME_STYLES = ['aurora', 'studio', 'dark', 'pastel', 'blur'];
|
|
@@ -138,8 +139,9 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
138
139
|
if (frame?.icon) frame.iconFile = path.resolve(dir, frame.icon);
|
|
139
140
|
|
|
140
141
|
// hyperframes: true | { template: lesson | launch, band, bandColors, emblem, hand, backdrop } — cards
|
|
141
|
-
// rendered by HyperFrames (see hyperframes.mjs); promos get the launch template, the rest the lesson one
|
|
142
|
-
|
|
142
|
+
// rendered by HyperFrames (see hyperframes.mjs); promos get the launch template, the rest the lesson one.
|
|
143
|
+
// On by default for tutorials and lessons, where Node can run it (22+); else the built-in cards
|
|
144
|
+
const hf0 = raw.hyperframes !== undefined ? raw.hyperframes : P.hyperframes && Number(process.versions.node.split('.')[0]) >= 22;
|
|
143
145
|
const hyperframes = hf0 ? { template: purpose === 'pitch' ? 'launch' : 'lesson', backdrop: true, ...(typeof hf0 === 'object' ? hf0 : {}) } : null;
|
|
144
146
|
if (hyperframes && !['lesson', 'launch'].includes(hyperframes.template)) throw new Error('hyperframes.template must be lesson or launch');
|
|
145
147
|
if (hyperframes?.bandColors && !['aminoacid', 'nucleotide', 'accent'].includes(hyperframes.bandColors)) throw new Error('hyperframes.bandColors must be aminoacid, nucleotide or accent');
|
|
@@ -152,7 +154,8 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
152
154
|
// kokoro's and qwen's default voices follow the narration's language (zf_001 / serena for Chinese)
|
|
153
155
|
const kokoroVoice = kokoroVoiceFor(raw);
|
|
154
156
|
const byLang = { kokoro: kokoroVoice, qwen: qwenVoiceFor(raw) }[provider];
|
|
155
|
-
voice = { speed: pace0.voiceSpeed, volume: 1, ...VOICE_DEFAULTS[provider], ...(byLang ? { voice: byLang } : {}), ...raw.voice, provider };
|
|
157
|
+
voice = { speed: pace0.voiceSpeed, volume: 1, pause: pace0.sentencePause, ...VOICE_DEFAULTS[provider], ...(byLang ? { voice: byLang } : {}), ...raw.voice, provider };
|
|
158
|
+
if (!(typeof voice.pause === 'number' && voice.pause >= 0 && voice.pause <= 3)) throw new Error('voice.pause is the shortest pause between the sentences of a line, in seconds, 0–3 (0 leaves the voice as it is)');
|
|
156
159
|
if (provider === 'qwen') {
|
|
157
160
|
const v = qwenVoice(voice.voice);
|
|
158
161
|
if (!v) throw new Error(`voice: qwen has no voice "${voice.voice}" (one of ${QWEN_VOICES.join(', ')})`);
|
|
@@ -214,8 +217,8 @@ export function loadConfig(file, { draft = false } = {}) {
|
|
|
214
217
|
follow,
|
|
215
218
|
music,
|
|
216
219
|
voice,
|
|
217
|
-
// sfx: true (clicks
|
|
218
|
-
// sound at every card soon grates), or false
|
|
220
|
+
// sfx: true (clicks, transition whooshes and risers), 'clicks' (no whooshes or risers: tutorials and
|
|
221
|
+
// lessons, where a sound at every card soon grates), or false
|
|
219
222
|
sfx: pick('sfx') === false ? false : { clicks: true, transitions: pick('sfx') !== 'clicks' },
|
|
220
223
|
draft,
|
|
221
224
|
out: path.resolve(dir, raw.out ?? `${name}.mp4`).replace(/(\.[^./]+)?$/, (ext) => (draft ? `.draft${ext || '.mp4'}` : ext)),
|
package/src/doctor.mjs
CHANGED
|
@@ -81,8 +81,8 @@ export async function doctor({ log = console.log } = {}) {
|
|
|
81
81
|
: '· Chinese voice (kokoro zf_001) needs Python 3.10–3.12 — optional: brew install python@3.12, or uv python install 3.12 (or set STORYSHOT_PYTHON)');
|
|
82
82
|
const q = qwenStatus();
|
|
83
83
|
lines.push(!q.supported ? '· Qwen3-TTS voice (provider: qwen) needs an Apple Silicon Mac; storyboards that use it fall back to kokoro here'
|
|
84
|
-
: q.ready ? `✔ Qwen3-TTS voice (provider: qwen): venv in ${q.venv}${q.model ? ',
|
|
85
|
-
: q.python ? `· Qwen3-TTS voice (provider: qwen): Python ${q.python.version} found; storyshot sets up its venv in ${q.venv} and downloads the
|
|
84
|
+
: q.ready ? `✔ Qwen3-TTS voice (provider: qwen): venv in ${q.venv}${q.model ? ', models downloaded' : '; the models (~3.3 GB: the voice, and a speech recognizer that checks each take) download on first use'}`
|
|
85
|
+
: q.python ? `· Qwen3-TTS voice (provider: qwen): Python ${q.python.version} found; storyshot sets up its venv in ${q.venv} and downloads the models (~3.3 GB) on first use`
|
|
86
86
|
: '· Qwen3-TTS voice (provider: qwen) needs Python 3.10–3.12 — optional: brew install python@3.12, or uv python install 3.12 (or set STORYSHOT_PYTHON)');
|
|
87
87
|
const hf = hyperframesCli();
|
|
88
88
|
lines.push(hf.error ? `· HyperFrames cards (hyperframes: true) unavailable — optional: ${hf.error}`
|
package/src/init.mjs
CHANGED
|
@@ -7,6 +7,14 @@ import { PURPOSES, defaultStoryboard } from './config.mjs';
|
|
|
7
7
|
import { loginAdvice } from './auth.mjs';
|
|
8
8
|
|
|
9
9
|
const q = (s) => JSON.stringify(String(s));
|
|
10
|
+
// The product's name from the page title: its first part ("Squoosh | Image compressor" → "Squoosh"),
|
|
11
|
+
// or the repo on a code host ("GitHub - boheling/storyshot: Ask…" → "storyshot"). Only spaced
|
|
12
|
+
// separators split, so a hyphenated name (Tic-Tac-Toe) stays whole.
|
|
13
|
+
export function productName(title, host) {
|
|
14
|
+
const segs = String(title ?? '').split(/\s+[|—–·-]\s+|:\s+/).map((s) => s.trim()).filter(Boolean);
|
|
15
|
+
const repo = segs.find((s) => /^[\w.-]+ ?\/ ?[\w.-]+$/.test(s));
|
|
16
|
+
return repo ? repo.split('/')[1].trim() : segs[0] ?? host;
|
|
17
|
+
}
|
|
10
18
|
const tgt = (t) => (typeof t === 'string' ? q(t) : `{ ${Object.entries(t).map(([k, v]) => `${k}: ${typeof v === 'string' ? q(v) : v}`).join(', ')} }`);
|
|
11
19
|
|
|
12
20
|
export async function initStoryboard(url, { purpose = 'pitch', topic, file, storageState, ignoreHTTPSErrors = false, viewport = { width: 1600, height: 900 } } = {}) {
|
|
@@ -41,7 +49,7 @@ out: ${name}.mp4
|
|
|
41
49
|
${storageState ? `storageState: ${path.relative(path.dirname(path.resolve(file)), path.resolve(storageState)).split(path.sep).join('/')} # saved login (never commit it)\n` : ''}${ignoreHTTPSErrors ? 'ignoreHTTPSErrors: true # internal site with a self-signed certificate\n' : ''}${narrated ? 'voice: { provider: kokoro } # free, local, English. For the final: { provider: elevenlabs, voice: <voice id> } + ELEVENLABS_API_KEY' : '# voice: { provider: kokoro } # optional voiceover; add `say:` lines to steps'}
|
|
42
50
|
|
|
43
51
|
${purpose === 'pitch' ? `intro: # title card before the recording (promos; optional for tutorials/lessons)
|
|
44
|
-
title: ${q(page.title
|
|
52
|
+
title: ${q(productName(page.title, host))}
|
|
45
53
|
subtitle: TODO the one-line promise, with the **key words** in bold
|
|
46
54
|
${PURPOSES[purpose]?.frame ? '# frame: false # the recording sits in a browser window on a backdrop; false = full frame\n' : ''}
|
|
47
55
|
` : purpose === 'tutorial' || purpose === 'course' ? `# intro: { title: TODO, subtitle: TODO } # optional title card (e.g. "How to export a report")
|
|
@@ -65,7 +73,7 @@ steps:
|
|
|
65
73
|
# run \`storyshot inspect ${file} --after <step>\` to list verified targets at that point.
|
|
66
74
|
|
|
67
75
|
end_card:
|
|
68
|
-
title: ${q(page.title
|
|
76
|
+
title: ${q(productName(page.title, host))}
|
|
69
77
|
cta: ${q(`Try it at ${host}`)}${narrated ? `\n say: ${q(`Try it at ${host}.`)}` : ''}
|
|
70
78
|
|
|
71
79
|
# ── Verified targets on ${page.url} (each matches exactly one visible element) ──
|
package/src/inspect.mjs
CHANGED
|
@@ -44,7 +44,8 @@ function collect() {
|
|
|
44
44
|
const clean = (s) => (s || '').replace(/\s+/g, ' ').trim();
|
|
45
45
|
// a short CSS selector (tag + classes, or #id) — the last-resort target when names don't resolve
|
|
46
46
|
const cssOf = (el) => {
|
|
47
|
-
|
|
47
|
+
const id = el.getAttribute('id'); // not el.id: a form with an <input name=id> has that input as its .id
|
|
48
|
+
if (id && !/^[0-9:]|:/.test(id)) return `#${CSS.escape(id)}`;
|
|
48
49
|
const cls = [...el.classList].filter((c) => !/^(css|sc|jsx|svelte|tw)-|\d{3,}|__/.test(c)).slice(0, 2).map((c) => `.${CSS.escape(c)}`).join('');
|
|
49
50
|
return cls ? `${el.tagName.toLowerCase()}${cls}` : null;
|
|
50
51
|
};
|
|
@@ -68,10 +69,17 @@ function collect() {
|
|
|
68
69
|
// a label's own text, without the links/buttons inside it (e.g. a "Clear" link)
|
|
69
70
|
const labelText = (lab) => { if (!lab) return null; const c = lab.cloneNode(true); c.querySelectorAll('a,button,select,input,textarea').forEach((x) => x.remove()); return clean(c.innerText) || null; };
|
|
70
71
|
// styled checkboxes and radios hide the real input and draw their label instead: list them where the label is
|
|
72
|
+
// a slider's range and value, when it says (an <input type=range>, or aria-valuemin/max/now)
|
|
73
|
+
const sliderRange = (el) => {
|
|
74
|
+
const n = (a) => (el.getAttribute(a) == null ? NaN : Number(el.getAttribute(a)));
|
|
75
|
+
const [lo, hi, now] = el.type === 'range' ? [Number(el.min || 0), Number(el.max || 100), Number(el.value)] : [n('aria-valuemin'), n('aria-valuemax'), n('aria-valuenow')];
|
|
76
|
+
return [lo, hi, now].every(Number.isFinite) ? { lo, hi, now } : null;
|
|
77
|
+
};
|
|
71
78
|
const shownBy = (el) => (el.tagName === 'INPUT' && ['checkbox', 'radio'].includes(el.type) && !vis(el) ? [...(el.labels ?? [])].find(vis) : null);
|
|
72
79
|
const interactive = [...document.querySelectorAll(NATIVE)]
|
|
73
80
|
.filter((el) => (vis(el) || shownBy(el)) && el.tagName !== 'LABEL' && !isAd(el) && !(el.isContentEditable && el.parentElement?.isContentEditable))
|
|
74
81
|
.map((el) => ({ kind: 'role', role: roleOf(el), name: (el.isContentEditable || el.tagName === 'TEXTAREA' ? editorName(el) : nameOf(el)).slice(0, 80), shows: clean(el.innerText).slice(0, 60), alt: clean(el.querySelector('img[alt]')?.alt) || null, href: el.getAttribute('href'), label: labelText(el.labels?.[0]),
|
|
82
|
+
range: roleOf(el) === 'slider' ? sliderRange(el) : null,
|
|
75
83
|
options: el.tagName === 'SELECT' ? [...el.options].map((o) => ({ text: clean(o.text), on: o.selected })).filter((o) => o.text).slice(0, 12) : null, ...pos(shownBy(el) ?? el) }));
|
|
76
84
|
// custom clickables: outermost cursor:pointer elements that aren't native controls
|
|
77
85
|
const pointer = [];
|
|
@@ -85,11 +93,11 @@ function collect() {
|
|
|
85
93
|
return {
|
|
86
94
|
title: document.title,
|
|
87
95
|
height: scroller ? scroller.scrollHeight : document.documentElement.scrollHeight,
|
|
88
|
-
scroller: scroller ? (scroller.id ? `#${scroller.id}` : scroller.tagName.toLowerCase() + (scroller.className && typeof scroller.className === 'string' ? '.' + scroller.className.trim().split(/\s+/)[0] : '')) : null,
|
|
96
|
+
scroller: scroller ? (scroller.getAttribute('id') ? `#${scroller.getAttribute('id')}` : scroller.tagName.toLowerCase() + (scroller.className && typeof scroller.className === 'string' ? '.' + scroller.className.trim().split(/\s+/)[0] : '')) : null,
|
|
89
97
|
headings: [...document.querySelectorAll('h1, h2, h3')].filter(vis).map((h) => ({ tag: h.tagName.toLowerCase(), text: clean(h.innerText).slice(0, 100), ...pos(h) })),
|
|
90
98
|
anchors: [...document.querySelectorAll('[id]')].filter((el) => vis(el) && el !== scroller && el.getBoundingClientRect().height > 150 &&
|
|
91
|
-
el.getBoundingClientRect().height < document.documentElement.scrollHeight * 0.8 && !/^(__|radix|headlessui|:)/.test(el.id))
|
|
92
|
-
.map((el) => ({ id: el.id, ...pos(el) })).slice(0, 30),
|
|
99
|
+
el.getBoundingClientRect().height < document.documentElement.scrollHeight * 0.8 && !/^(__|radix|headlessui|:)/.test(el.getAttribute('id')))
|
|
100
|
+
.map((el) => ({ id: el.getAttribute('id'), ...pos(el) })).slice(0, 30),
|
|
93
101
|
images: [...document.querySelectorAll('img[alt]')].filter((el) => vis(el) && el.alt.trim()).map((el) => ({ alt: clean(el.alt).slice(0, 80), ...pos(el) })).slice(0, 30),
|
|
94
102
|
interactive: [...interactive, ...pointer],
|
|
95
103
|
};
|
|
@@ -160,7 +168,7 @@ export async function describePage(page, viewport, { maxElements = 80 } = {}) {
|
|
|
160
168
|
'',
|
|
161
169
|
'## Clickable elements — verified targets (each matches exactly one visible element) for click/move/zoom',
|
|
162
170
|
' Copy targets exactly; `name` is the accessible name, which may differ from the on-screen label.',
|
|
163
|
-
...elements.map((e) => `${fold(e.y)} y=${e.y}\tat (${e.cx}, ${e.cy})\t${yaml(e.target)}${e.role === 'textbox' ? ` (text input ${e.w}×${e.h} — click it, then \`type: …\`)` : e.role === 'slider' ? ' (slider — `slide: { target: …, to: 0..1 }`)' : e.options ? ` (dropdown — \`select: { target: …, option: "…" }\`; options: ${e.options.map((o) => `"${o.text}"${o.on ? ' (selected)' : ''}`).join(', ')})` : shows(e)}${e.href ? ` → ${e.href}` : ''}${e.kind === 'pointer' ? ' (custom clickable)' : ''}`),
|
|
171
|
+
...elements.map((e) => `${fold(e.y)} y=${e.y}\tat (${e.cx}, ${e.cy})\t${yaml(e.target)}${e.role === 'textbox' ? ` (text input ${e.w}×${e.h} — click it, then \`type: …\`)` : e.role === 'slider' ? (e.range ? ` (slider ${e.range.lo}…${e.range.hi}, now ${e.range.now} — \`slide: { target: …, value: ${e.range.now} }\`)` : ' (slider — `slide: { target: …, to: 0..1 }`)') : e.options ? ` (dropdown — \`select: { target: …, option: "…" }\`; options: ${e.options.map((o) => `"${o.text}"${o.on ? ' (selected)' : ''}`).join(', ')})` : shows(e)}${e.href ? ` → ${e.href}` : ''}${e.kind === 'pointer' ? ' (custom clickable)' : ''}`),
|
|
164
172
|
'',
|
|
165
173
|
'## Images with alt text (target: { alt: "..." })',
|
|
166
174
|
...(info.images.length ? info.images.map((i) => `${fold(i.y)} y=${i.y}\t"${i.alt}"`) : ['(none)']),
|
package/src/qwen-tts.mjs
CHANGED
|
@@ -22,15 +22,20 @@ const REPO = 'mlx-community/Qwen3-TTS-12Hz-0.6B-CustomVoice-bf16';
|
|
|
22
22
|
const REVISION = '6415d95f88be018ff9e46813119dc3bc12261328';
|
|
23
23
|
// 0.9 (the model's default) now and then runs on into made-up words; 0.7 keeps the same voice steadier
|
|
24
24
|
const TEMPERATURE = 0.7;
|
|
25
|
+
// Each take is checked by ear: Qwen3-ASR (Apache-2.0, ~1 GB) transcribes it, and a line it hears
|
|
26
|
+
// differently is sampled again (pypinyin compares Chinese by sound, so homophones don't count)
|
|
27
|
+
const ASR_REPO = 'mlx-community/Qwen3-ASR-0.6B-8bit';
|
|
28
|
+
const ASR_REVISION = '89e96d92ba34aca20b3e29fb10cc284097d1219f';
|
|
25
29
|
const REQUIREMENTS = ['mlx-audio==0.6.3', 'mlx==0.32.4', 'mlx-metal==0.32.4', 'transformers==5.22.0', 'huggingface-hub==2.4.0',
|
|
26
|
-
'tokenizers==0.23.3', 'safetensors==0.8.0', 'numpy==2.5.4'];
|
|
30
|
+
'tokenizers==0.23.3', 'safetensors==0.8.0', 'numpy==2.5.4', 'pypinyin==0.55.0'];
|
|
27
31
|
// Part of every cached line's hash: change it when the model, the packages or qwen_tts.py's sampling change.
|
|
28
|
-
export const QWEN_TTS_ID = `${REPO}@${REVISION.slice(0, 7)}|${REQUIREMENTS[0]}|t${TEMPERATURE}|guard-
|
|
32
|
+
export const QWEN_TTS_ID = `${REPO}@${REVISION.slice(0, 7)}|${REQUIREMENTS[0]}|t${TEMPERATURE}|guard-2|${ASR_REPO}@${ASR_REVISION.slice(0, 7)}`;
|
|
29
33
|
const CACHE = path.join(os.homedir(), '.cache', 'storyshot');
|
|
30
34
|
export const QWEN_VENV = path.join(CACHE, 'qwen-venv');
|
|
31
35
|
const HELPER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'qwen_tts.py');
|
|
32
36
|
const HUB = process.env.HF_HUB_CACHE ?? path.join(process.env.HF_HOME ?? path.join(os.homedir(), '.cache', 'huggingface'), 'hub');
|
|
33
37
|
const MODEL_FILE = path.join(HUB, `models--${REPO.replace('/', '--')}`, 'snapshots', REVISION, 'model.safetensors');
|
|
38
|
+
const ASR_FILE = path.join(HUB, `models--${ASR_REPO.replace('/', '--')}`, 'snapshots', ASR_REVISION, 'model.safetensors');
|
|
34
39
|
|
|
35
40
|
// --- text (TTS only: subtitles keep the original) --------------------------------------------------
|
|
36
41
|
// The model reads = + 2个 第2行 x² itself; a minus sign before a number it drops, and a number like
|
|
@@ -66,7 +71,7 @@ const NO_PYTHON = () => 'the Qwen3-TTS voice needs Python 3.10–3.12 to run (br
|
|
|
66
71
|
// For doctor and check: can this machine use it, and what's still to come on first use?
|
|
67
72
|
export function qwenStatus() {
|
|
68
73
|
if (!qwenSupported()) return { supported: false };
|
|
69
|
-
const model = fs.existsSync(MODEL_FILE);
|
|
74
|
+
const model = fs.existsSync(MODEL_FILE) && fs.existsSync(ASR_FILE);
|
|
70
75
|
if (venvReady()) return { supported: true, ready: true, venv: QWEN_VENV, model };
|
|
71
76
|
const python = findPython();
|
|
72
77
|
return { supported: true, ready: false, venv: QWEN_VENV, model, python, problem: python ? null : NO_PYTHON() };
|
|
@@ -74,13 +79,22 @@ export function qwenStatus() {
|
|
|
74
79
|
|
|
75
80
|
function ensureVenv(log) {
|
|
76
81
|
if (venvReady()) return venvPython(QWEN_VENV);
|
|
77
|
-
const py = findPython();
|
|
78
|
-
if (!py) throw new Error(NO_PYTHON());
|
|
79
|
-
log(`setting up the Qwen3-TTS voice (one time): a Python ${py.version} venv in ${QWEN_VENV} with mlx-audio (~400 MB, 1–2 min)…`);
|
|
80
82
|
const run = (cmd, args, what) => {
|
|
81
83
|
const r = spawnSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
|
|
82
84
|
if (r.status !== 0) throw new Error(`${what} failed: ${(r.stderr || r.stdout || r.error?.message || '').trim().split('\n').slice(-6).join('\n')}`);
|
|
83
85
|
};
|
|
86
|
+
if (fs.existsSync(venvPython(QWEN_VENV))) {
|
|
87
|
+
// a venv from an older storyshot: bring its packages up to date in place, rebuild it only if that fails
|
|
88
|
+
log(`updating the Qwen3-TTS venv in ${QWEN_VENV}…`);
|
|
89
|
+
try {
|
|
90
|
+
run(venvPython(QWEN_VENV), ['-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q', ...REQUIREMENTS], 'updating the venv');
|
|
91
|
+
fs.writeFileSync(path.join(QWEN_VENV, STAMP), REQUIREMENTS.join('\n'));
|
|
92
|
+
return venvPython(QWEN_VENV);
|
|
93
|
+
} catch (e) { log(` ${e.message.split('\n')[0]} — rebuilding it`); }
|
|
94
|
+
}
|
|
95
|
+
const py = findPython();
|
|
96
|
+
if (!py) throw new Error(NO_PYTHON());
|
|
97
|
+
log(`setting up the Qwen3-TTS voice (one time): a Python ${py.version} venv in ${QWEN_VENV} with mlx-audio (~400 MB, 1–2 min)…`);
|
|
84
98
|
fs.rmSync(QWEN_VENV, { recursive: true, force: true });
|
|
85
99
|
fs.mkdirSync(CACHE, { recursive: true });
|
|
86
100
|
run(py.exe, ['-m', 'venv', QWEN_VENV], `creating the venv with ${py.exe} -m venv`);
|
|
@@ -96,12 +110,13 @@ export async function synthQwen(v, entries, { log = console.log } = {}) {
|
|
|
96
110
|
if (!qwenSupported()) throw new Error('the Qwen3-TTS voice runs on Apple Silicon Macs only (it uses MLX)');
|
|
97
111
|
const py = ensureVenv(log);
|
|
98
112
|
if (!fs.existsSync(MODEL_FILE)) log(`downloading the Qwen3-TTS model (one time, ~2.3 GB) into ${HUB}…`);
|
|
113
|
+
if (!fs.existsSync(ASR_FILE)) log(`downloading the Qwen3-ASR model that listens to each take (one time, ~1 GB) into ${HUB}…`);
|
|
99
114
|
const lines = entries.map((e) => ({
|
|
100
115
|
text: e.text, file: `${e.file}.raw.wav`, voice: v.voice, lang: qwenLang(e.text),
|
|
101
116
|
// the same line gets the same take: a seed from the text
|
|
102
117
|
seed: parseInt(crypto.createHash('sha1').update(e.text).digest('hex').slice(0, 7), 16),
|
|
103
118
|
}));
|
|
104
|
-
const job = JSON.stringify({ repo: REPO, revision: REVISION, temperature: TEMPERATURE, lines });
|
|
119
|
+
const job = JSON.stringify({ repo: REPO, revision: REVISION, temperature: TEMPERATURE, asr: { repo: ASR_REPO, revision: ASR_REVISION }, lines });
|
|
105
120
|
const { code, stdout, tail } = await new Promise((resolve, reject) => {
|
|
106
121
|
const p = spawn(py, ['-I', HELPER], { env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1', TRANSFORMERS_VERBOSITY: 'error' } });
|
|
107
122
|
let out = '', err = '', buf = '';
|
|
@@ -121,9 +136,15 @@ export async function synthQwen(v, entries, { log = console.log } = {}) {
|
|
|
121
136
|
p.stdin.end(job);
|
|
122
137
|
});
|
|
123
138
|
if (code !== 0) throw new Error(`Qwen3-TTS failed (${py} ${HELPER}): ${tail.trim().split('\n').slice(-8).join('\n')}`);
|
|
124
|
-
const { results } = JSON.parse(stdout);
|
|
139
|
+
const { results, asr } = JSON.parse(stdout);
|
|
140
|
+
if (asr) log(` ⚠ qwen: the speech check couldn't load (${asr}), so these takes were checked by length only`);
|
|
125
141
|
for (const [i, e] of entries.entries()) {
|
|
126
|
-
|
|
142
|
+
const r = results[i];
|
|
143
|
+
if (r?.suspect) {
|
|
144
|
+
log(r.heard != null && r.mismatch > 0
|
|
145
|
+
? ` ⚠ qwen: "${e.text}" still doesn't come out right after ${r.tries} tries — it sounds like "${r.heard}". Listen to it; rephrase the line or add a pronounce entry`
|
|
146
|
+
: ` ⚠ qwen: "${e.text}" still sounds off after ${r.tries} tries (${r.seconds.toFixed(1)}s) — listen to it, or rephrase the line / add a pronounce entry`);
|
|
147
|
+
}
|
|
127
148
|
const raw = lines[i].file, tmp = `${e.file}.tmp.wav`;
|
|
128
149
|
// the model has no speed control: ffmpeg's pitch-preserving atempo does it
|
|
129
150
|
if ((v.speed ?? 1) !== 1) execFileSync('ffmpeg', ['-loglevel', 'error', '-y', '-i', raw, '-af', `atempo=${v.speed}`, tmp]);
|
package/src/qwen_tts.py
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
# Qwen3-TTS CustomVoice through mlx-audio (Apple Silicon), for `voice: { provider: qwen }`. Runs in
|
|
2
2
|
# storyshot's own venv (qwen-tts.mjs), one process per render: the model loads once for all lines.
|
|
3
3
|
#
|
|
4
|
-
# stdin: {"repo", "revision", "temperature", "
|
|
4
|
+
# stdin: {"repo", "revision", "temperature", "asr": {"repo", "revision"} | null,
|
|
5
|
+
# "lines": [{"text", "file", "voice", "lang", "seed"}]}
|
|
5
6
|
# stderr: "line i/n <seconds>s [<tries> tries]" per line, plus whatever the libraries log
|
|
6
|
-
# stdout: {"results": [{"seconds", "tries", "suspect"}]}
|
|
7
|
+
# stdout: {"results": [{"seconds", "tries", "suspect", "heard", "mismatch"}], "asr": null | "why it didn't load"}
|
|
7
8
|
#
|
|
8
|
-
# The model samples its speech tokens, so now and then a
|
|
9
|
-
# scientific number like 2e-105 does it) or stops early.
|
|
10
|
-
#
|
|
9
|
+
# The model samples its speech tokens, so now and then a take garbles a few words, drops one, runs on
|
|
10
|
+
# into made-up words (a scientific number like 2e-105 does it) or stops early. Each take is checked:
|
|
11
|
+
# its length against the text, and what a speech recognizer (Qwen3-ASR) hears in it against the text.
|
|
12
|
+
# A take that fails is sampled again with another seed, and the best one is kept.
|
|
11
13
|
import contextlib
|
|
12
14
|
import json
|
|
13
15
|
import os
|
|
@@ -18,6 +20,10 @@ os.environ.setdefault('HF_HUB_DISABLE_TELEMETRY', '1')
|
|
|
18
20
|
os.environ.setdefault('TRANSFORMERS_VERBOSITY', 'error')
|
|
19
21
|
|
|
20
22
|
TRIES = 4
|
|
23
|
+
# a take is sampled again when the recognizer hears this share of the line differently, or this long a run
|
|
24
|
+
# of mistakes in a row (a garble: 负 2 x → 负二长方荡 is 2.5); a warning if every take is still off
|
|
25
|
+
RETRY_AT, RETRY_RUN = 0.10, 2
|
|
26
|
+
WARN_AT, WARN_RUN = 0.15, 2
|
|
21
27
|
|
|
22
28
|
|
|
23
29
|
def expected(text):
|
|
@@ -32,6 +38,87 @@ def plausible(seconds, est):
|
|
|
32
38
|
return 0.35 * est - 0.5 <= seconds <= 2.0 * est + 2.0
|
|
33
39
|
|
|
34
40
|
|
|
41
|
+
# --- what was said vs. what was meant ---------------------------------------------------------------
|
|
42
|
+
# Chinese is compared by sound (toneless pinyin), so a homophone the recognizer picks (交点/焦点) is no
|
|
43
|
+
# mistake; numbers are read out first (0.5 → 零点五, 2 → 二), since the recognizer writes either form.
|
|
44
|
+
DIGITS = '零一二三四五六七八九'
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def cn_int(n):
|
|
48
|
+
if n >= 10 ** 8:
|
|
49
|
+
return ''.join(DIGITS[int(c)] for c in str(n))
|
|
50
|
+
|
|
51
|
+
def four(x):
|
|
52
|
+
out, zero = '', False
|
|
53
|
+
for i, c in enumerate(f'{x:04d}'):
|
|
54
|
+
d = int(c)
|
|
55
|
+
if d == 0:
|
|
56
|
+
zero = bool(out)
|
|
57
|
+
else:
|
|
58
|
+
out += ('零' if zero else '') + DIGITS[d] + ['千', '百', '十', ''][i]
|
|
59
|
+
zero = False
|
|
60
|
+
return out
|
|
61
|
+
|
|
62
|
+
if n < 10:
|
|
63
|
+
return DIGITS[n]
|
|
64
|
+
hi, lo = divmod(n, 10000)
|
|
65
|
+
out = (four(hi) + '万' if hi else '') + (('零' if hi and lo < 1000 else '') + four(lo) if lo else '')
|
|
66
|
+
return re.sub('^一十', '十', out)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def cn_number(m):
|
|
70
|
+
whole, _, frac = m.group().partition('.')
|
|
71
|
+
return cn_int(int(whole)) + ('点' + ''.join(DIGITS[int(c)] for c in frac) if frac else '')
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def units(text, chinese, pinyin):
|
|
75
|
+
"""The line as a list of sounds: pinyin syllables, digits, and Latin words as '#word' (m x and mx as '#m', '#x')."""
|
|
76
|
+
t = text.lower().replace('−', '-')
|
|
77
|
+
if chinese:
|
|
78
|
+
t = re.sub(r'\d+(?:\.\d+)?', cn_number, t)
|
|
79
|
+
out = []
|
|
80
|
+
for tok in re.findall(r'[㐀-鿿]|[a-z]+|\d+', t):
|
|
81
|
+
if chinese and re.match(r'[㐀-鿿]', tok):
|
|
82
|
+
out += [{'liang': 'er'}.get(p, p) for p in pinyin(tok)]
|
|
83
|
+
elif tok.isdigit():
|
|
84
|
+
out += list(tok) # digit by digit: a wrong number (260 → 45) is several mistakes in a row, not one
|
|
85
|
+
else:
|
|
86
|
+
out += ['#' + c for c in tok] if len(tok) <= 2 else ['#' + tok]
|
|
87
|
+
return out
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def mismatch(text, heard, chinese, pinyin):
|
|
91
|
+
"""(share of the line heard differently, the longest run of mistakes in a row), both in units of the
|
|
92
|
+
text. A garbled take has a run of several; a homophone or a misheard letter is one on its own."""
|
|
93
|
+
a, b = units(text, chinese, pinyin), units(heard, chinese, pinyin)
|
|
94
|
+
letter = lambda u: len(u) == 2 and u[0] == '#'
|
|
95
|
+
d = {(0, 0): (0, None)}
|
|
96
|
+
for i in range(len(a) + 1):
|
|
97
|
+
for j in range(len(b) + 1):
|
|
98
|
+
if i == j == 0:
|
|
99
|
+
continue
|
|
100
|
+
ways = []
|
|
101
|
+
if i:
|
|
102
|
+
ways.append((d[i - 1, j][0] + 1, (i - 1, j, 1)))
|
|
103
|
+
if j:
|
|
104
|
+
ways.append((d[i, j - 1][0] + 1, (i, j - 1, 1)))
|
|
105
|
+
if i and j:
|
|
106
|
+
x, y = a[i - 1], b[j - 1]
|
|
107
|
+
# a variable (m, b, x) the recognizer wrote as a character is half a mistake
|
|
108
|
+
cost = 0 if x == y else 0.5 if letter(x) or letter(y) else 1
|
|
109
|
+
ways.append((d[i - 1, j - 1][0] + cost, (i - 1, j - 1, cost)))
|
|
110
|
+
# a name in Latin letters in a Chinese line (Desmos) can come back as a few syllables: one mistake
|
|
111
|
+
if chinese and len(x) > 3 and x[0] == '#':
|
|
112
|
+
ways += [(d[i - 1, j - k][0] + 1, (i - 1, j - k, 1)) for k in range(2, min(5, j) + 1)]
|
|
113
|
+
d[i, j] = min(ways, key=lambda w: w[0])
|
|
114
|
+
i, j, run, worst = len(a), len(b), 0, 0
|
|
115
|
+
while (i, j) != (0, 0):
|
|
116
|
+
i, j, cost = d[i, j][1]
|
|
117
|
+
run = run + cost if cost else 0
|
|
118
|
+
worst = max(worst, run)
|
|
119
|
+
return d[len(a), len(b)][0] / max(1, len(a)), worst
|
|
120
|
+
|
|
121
|
+
|
|
35
122
|
def main():
|
|
36
123
|
job = json.load(sys.stdin)
|
|
37
124
|
out = sys.stdout
|
|
@@ -45,6 +132,28 @@ def main():
|
|
|
45
132
|
from mlx_audio.tts.utils import load_model
|
|
46
133
|
|
|
47
134
|
model = load_model(snapshot_download(job['repo'], revision=job['revision']))
|
|
135
|
+
asr, asr_error, pinyin = None, None, None
|
|
136
|
+
if job.get('asr'):
|
|
137
|
+
try:
|
|
138
|
+
from mlx_audio.stt.utils import load_model as load_stt
|
|
139
|
+
from pypinyin import lazy_pinyin as pinyin
|
|
140
|
+
asr = load_stt(snapshot_download(job['asr']['repo'], revision=job['asr']['revision']))
|
|
141
|
+
except Exception as e: # offline before the first download, say: check by length only
|
|
142
|
+
asr_error = f'{type(e).__name__}: {e}'.splitlines()[0][:300]
|
|
143
|
+
|
|
144
|
+
def hear(audio, line):
|
|
145
|
+
"""(what the recognizer heard, mismatch), or (None, None) for a line it can't check."""
|
|
146
|
+
chinese = line['lang'] == 'chinese'
|
|
147
|
+
if asr is None or not (chinese or (line['lang'] == 'auto' and not re.search(r'[㐀-鿿]', line['text']))):
|
|
148
|
+
return None, None
|
|
149
|
+
tmp = line['file'] + '.check.wav'
|
|
150
|
+
write(tmp, audio, model.sample_rate)
|
|
151
|
+
try:
|
|
152
|
+
heard = asr.generate(tmp, language='Chinese' if chinese else 'English').text.strip()
|
|
153
|
+
finally:
|
|
154
|
+
os.remove(tmp)
|
|
155
|
+
return heard, mismatch(line['text'], heard, chinese, pinyin)
|
|
156
|
+
|
|
48
157
|
results, n = [], len(job['lines'])
|
|
49
158
|
for i, line in enumerate(job['lines']):
|
|
50
159
|
est = expected(line['text'])
|
|
@@ -55,15 +164,22 @@ def main():
|
|
|
55
164
|
temperature=job['temperature'])
|
|
56
165
|
audio = np.concatenate([np.array(r.audio, dtype=np.float32) for r in parts])
|
|
57
166
|
seconds = len(audio) / model.sample_rate
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
167
|
+
fits = plausible(seconds, est)
|
|
168
|
+
heard, m = hear(audio, line) if fits else (None, None)
|
|
169
|
+
# how far off the recognizer heard it: up to 1 is fine (0 when it couldn't check)
|
|
170
|
+
worse = max(m[0] / RETRY_AT, m[1] / RETRY_RUN) if m else 0
|
|
171
|
+
# a take of the right length that says the line beats one that doesn't; then the closest length
|
|
172
|
+
rank = (not fits, worse if worse > 1 else 0, abs(math.log(max(seconds, 0.1) / est)))
|
|
173
|
+
if best is None or rank < best['rank']:
|
|
174
|
+
best = {'rank': rank, 'audio': audio, 'seconds': seconds, 'fits': fits, 'heard': heard, 'm': m}
|
|
175
|
+
if fits and worse <= 1:
|
|
62
176
|
break
|
|
63
|
-
write(line['file'], best[
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
177
|
+
write(line['file'], best['audio'], model.sample_rate)
|
|
178
|
+
m = best['m']
|
|
179
|
+
results.append({'seconds': best['seconds'], 'tries': k + 1, 'heard': best['heard'], 'mismatch': m and m[0],
|
|
180
|
+
'suspect': not best['fits'] or bool(m and (m[0] > WARN_AT or m[1] > WARN_RUN))})
|
|
181
|
+
print(f'line {i + 1}/{n} {best["seconds"]:.1f}s' + (f' {k + 1} tries' if k else ''), file=sys.stderr, flush=True)
|
|
182
|
+
out.write(json.dumps({'results': results, 'asr': asr_error}))
|
|
67
183
|
|
|
68
184
|
|
|
69
185
|
if __name__ == '__main__':
|
package/src/record.mjs
CHANGED
|
@@ -55,19 +55,60 @@ function pageScript({ scrollOffset }) {
|
|
|
55
55
|
for (let p = el.parentElement; p && p !== document.body; p = p.parentElement) if (scrollable(p)) return p;
|
|
56
56
|
return docEl();
|
|
57
57
|
};
|
|
58
|
+
// `behavior: 'instant'` on every frame: with `scroll-behavior: smooth` in the page's CSS (GitHub has it),
|
|
59
|
+
// plain scrollTop writes each start a smooth scroll of their own, so the page lags behind and is still
|
|
60
|
+
// moving after the step that scrolled it is over
|
|
58
61
|
window.__demoScrollBy = (dy, ms = 1200, sc = mainScroller()) => new Promise((done) => {
|
|
59
62
|
const start = sc.scrollTop, t0 = performance.now();
|
|
60
63
|
const ease = (t) => (t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2);
|
|
61
64
|
const step = (now) => {
|
|
62
|
-
const k = Math.min(1, (now - t0) / ms);
|
|
63
|
-
sc.
|
|
65
|
+
const k = ms > 0 ? Math.min(1, Math.max(0, (now - t0) / ms)) : 1;
|
|
66
|
+
sc.scrollTo({ top: start + dy * ease(k), behavior: 'instant' });
|
|
64
67
|
if (k < 1) requestAnimationFrame(step); else done();
|
|
65
68
|
};
|
|
66
69
|
requestAnimationFrame(step);
|
|
67
70
|
});
|
|
71
|
+
// How far down a header will cover the window once `el` is scrolled to the top: bars fixed at the top,
|
|
72
|
+
// and sticky ones that stick inside a box holding `el` (GitHub's README bar sticks within the README).
|
|
73
|
+
const headerCover = (el) => {
|
|
74
|
+
let cover = 0;
|
|
75
|
+
for (const h of document.querySelectorAll('body *')) {
|
|
76
|
+
if (h.getAttribute('id')?.startsWith('__demo') || h.contains(el)) continue; // not h.id: a form with an <input name=id> has that input as its .id
|
|
77
|
+
const cs = getComputedStyle(h);
|
|
78
|
+
if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;
|
|
79
|
+
const r = h.getBoundingClientRect();
|
|
80
|
+
if (!r.height || r.height > innerHeight * 0.3 || r.width < innerWidth * 0.3 || cs.visibility === 'hidden' || cs.opacity === '0') continue;
|
|
81
|
+
if (cs.position === 'fixed' && r.top <= 1) cover = Math.max(cover, r.bottom);
|
|
82
|
+
const stick = parseFloat(cs.top);
|
|
83
|
+
if (cs.position === 'sticky' && !Number.isNaN(stick) && stick < innerHeight * 0.3 && h.parentElement?.contains(el)) cover = Math.max(cover, stick + r.height);
|
|
84
|
+
}
|
|
85
|
+
return cover;
|
|
86
|
+
};
|
|
87
|
+
// the element's top lands `scrollOffset` px down, or just below a header that would cover it there
|
|
68
88
|
window.__demoScrollTo = (el, ms) => {
|
|
69
|
-
const sc = scrollerFor(el),
|
|
70
|
-
|
|
89
|
+
const sc = scrollerFor(el), docs = sc === docEl(), top = docs ? 0 : sc.getBoundingClientRect().top;
|
|
90
|
+
const offset = docs ? Math.max(scrollOffset, headerCover(el) + 12) : scrollOffset;
|
|
91
|
+
return window.__demoScrollBy(el.getBoundingClientRect().top - top - offset, ms, sc);
|
|
92
|
+
};
|
|
93
|
+
// resolves once nothing has scrolled for a few frames (the page's own smooth scrolling included)
|
|
94
|
+
window.__demoScrollIdle = (max = 2000) => new Promise((done) => {
|
|
95
|
+
const sc = mainScroller(), pos = () => `${scrollX},${scrollY},${sc.scrollTop}`;
|
|
96
|
+
const end = performance.now() + max;
|
|
97
|
+
let last = pos(), still = 0;
|
|
98
|
+
const check = () => {
|
|
99
|
+
const now = pos();
|
|
100
|
+
still = now === last ? still + 1 : 0; last = now;
|
|
101
|
+
if (still >= 3 || performance.now() > end) done(); else requestAnimationFrame(check);
|
|
102
|
+
};
|
|
103
|
+
requestAnimationFrame(check);
|
|
104
|
+
});
|
|
105
|
+
// a slider's value and range: an <input type=range>, or an ARIA slider (Desmos: [role=slider] on its knob)
|
|
106
|
+
window.__demoSliderValue = (el) => {
|
|
107
|
+
const r = el.type === 'range' ? el : el.querySelector?.('input[type=range]');
|
|
108
|
+
if (r) return { now: Number(r.value), lo: Number(r.min || 0), hi: Number(r.max || 100), native: r };
|
|
109
|
+
const aria = el.hasAttribute('aria-valuenow') ? el : el.querySelector?.('[aria-valuenow]');
|
|
110
|
+
const now = Number(aria?.getAttribute('aria-valuenow'));
|
|
111
|
+
return aria && Number.isFinite(now) ? { now, lo: Number(aria.getAttribute('aria-valuemin') ?? 0), hi: Number(aria.getAttribute('aria-valuemax') ?? 100) } : null;
|
|
71
112
|
};
|
|
72
113
|
// Tutorial callouts: spotlight a target, show a pressed key combo.
|
|
73
114
|
window.__demoSpot = (r) => {
|
|
@@ -86,9 +127,10 @@ function pageScript({ scrollOffset }) {
|
|
|
86
127
|
Object.assign(el.style, { left: `${r.x - pad}px`, top: `${r.y - pad}px`, width: `${r.width + 2 * pad}px`, height: `${r.height + 2 * pad}px` });
|
|
87
128
|
requestAnimationFrame(() => { el.style.opacity = '1'; });
|
|
88
129
|
};
|
|
89
|
-
window.__demoSpotOff = () => {
|
|
130
|
+
window.__demoSpotOff = (instant) => {
|
|
90
131
|
const el = document.getElementById('__demo_spot');
|
|
91
|
-
if (el
|
|
132
|
+
if (el && instant) el.remove();
|
|
133
|
+
else if (el) { el.style.opacity = '0'; setTimeout(() => el.remove(), 300); }
|
|
92
134
|
};
|
|
93
135
|
window.__demoKeys = (label) => {
|
|
94
136
|
const el = document.createElement('div');
|
|
@@ -424,7 +466,7 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
424
466
|
await sleep(280);
|
|
425
467
|
return true;
|
|
426
468
|
}
|
|
427
|
-
const spotOff = () => page.evaluate(() => window.__demoSpotOff()).catch(() => {});
|
|
469
|
+
const spotOff = (instant) => page.evaluate((i) => window.__demoSpotOff(i), !!instant).catch(() => {});
|
|
428
470
|
const keyLabel = (k) => k.split('+').map((p) => ({ Meta: '⌘', ControlOrMeta: '⌘', Control: 'Ctrl', Shift: '⇧', Alt: '⌥', Enter: '↵ Enter', Escape: 'Esc', Backspace: '⌫', Tab: '⇥ Tab' })[p] ?? (p.length === 1 ? p.toUpperCase() : p)).join(' ');
|
|
429
471
|
|
|
430
472
|
async function runAction(action) {
|
|
@@ -473,6 +515,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
473
515
|
case 'scrollTo': {
|
|
474
516
|
const h = await locate(page, spec).elementHandle({ timeout: opt.timeout ?? 15000 });
|
|
475
517
|
await page.evaluate(([el, ms]) => window.__demoScrollTo(el, ms), [h, opt.ms ?? 1200]);
|
|
518
|
+
// where it landed: a zoom on the same target frames it there, not where it was before the scroll
|
|
519
|
+
const b = await h.boundingBox().catch(() => null);
|
|
520
|
+
if (b) boxes.set(keyOf(spec), b);
|
|
476
521
|
break;
|
|
477
522
|
}
|
|
478
523
|
case 'scroll': await page.evaluate(([dy, ms]) => window.__demoScrollBy(dy, ms), [value.y ?? value, value.ms ?? 1000]); break;
|
|
@@ -570,25 +615,28 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
570
615
|
}
|
|
571
616
|
case 'slide': {
|
|
572
617
|
// slide: { target: <slider>, to: 0.2 } — drag a slider to a position along its track (0 = start,
|
|
573
|
-
// 1 = end),
|
|
574
|
-
|
|
618
|
+
// 1 = end), or { target, value: 3 } — to a value, for sliders that report their range; found from
|
|
619
|
+
// the element instead of pixel positions that break when the layout moves
|
|
620
|
+
if (!value?.target || (typeof value?.to !== 'number' && typeof value?.value !== 'number')) throw new Error('slide needs { target: <slider>, to: 0..1 } or { target: <slider>, value: <number> }');
|
|
575
621
|
const p = await pointOf(value.target, value);
|
|
576
|
-
const
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
622
|
+
const s = await locate(page, value.target).evaluate((el) => {
|
|
623
|
+
// a knob (Desmos's [role=slider] is its 38 px thumb): slide along its track, the nearest wide parent
|
|
624
|
+
let track = el;
|
|
625
|
+
const w = el.getBoundingClientRect().width;
|
|
626
|
+
for (let a = el.parentElement, i = 0; w < 60 && a && i < 4; a = a.parentElement, i++) if (a.getBoundingClientRect().width >= 3 * w) { track = a; break; }
|
|
627
|
+
const t = track.getBoundingClientRect(), v = window.__demoSliderValue(el);
|
|
628
|
+
let from = v ? (v.now - v.lo) / ((v.hi - v.lo) || 1) : null;
|
|
629
|
+
if (from == null) {
|
|
630
|
+
// custom sliders: start where the knob is drawn, so the value doesn't jump first
|
|
631
|
+
const knob = [...track.querySelectorAll('[role=slider], [class*=thumb], [class*=handle], [class*=knob]')]
|
|
632
|
+
.map((k) => k.getBoundingClientRect()).find((k) => k.width > 0 && k.width < t.width * 0.5);
|
|
633
|
+
from = knob ? (knob.x + knob.width / 2 - t.x) / (t.width || 1) : 0.5;
|
|
584
634
|
}
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
}).catch(() => 0.5);
|
|
591
|
-
const x0 = b.x + b.width * from, x1 = b.x + b.width * Math.max(0, Math.min(1, value.to)), y = b.y + b.height / 2;
|
|
635
|
+
return { box: { x: t.x, y: t.y, width: t.width, height: t.height }, from, range: v && { lo: v.lo, hi: v.hi } };
|
|
636
|
+
}).catch(() => ({ box: p.box, from: 0.5, range: null }));
|
|
637
|
+
if (typeof value.value === 'number' && !s.range) throw new Error('slide value: this slider doesn\'t report its range (no aria-valuemin/max, not an <input type=range>) — use to: 0..1');
|
|
638
|
+
const to = typeof value.value === 'number' ? (value.value - s.range.lo) / ((s.range.hi - s.range.lo) || 1) : value.to;
|
|
639
|
+
const b = s.box, x0 = b.x + b.width * s.from, x1 = b.x + b.width * Math.max(0, Math.min(1, to)), y = b.y + b.height / 2;
|
|
592
640
|
await moveTo(x0, y, value.ms ?? cfg.pace.cursor);
|
|
593
641
|
await sleep(120);
|
|
594
642
|
clicks.push(now());
|
|
@@ -596,6 +644,19 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
596
644
|
await moveTo(x1, y, value.dragMs ?? 900);
|
|
597
645
|
await sleep(120);
|
|
598
646
|
await page.mouse.up();
|
|
647
|
+
if (typeof value.value === 'number') {
|
|
648
|
+
// a native range's thumb doesn't reach the ends of its box, so the drag can land a step off: set it
|
|
649
|
+
const got = await locate(page, value.target).evaluate((el, want) => {
|
|
650
|
+
const v = window.__demoSliderValue(el), r = v?.native;
|
|
651
|
+
if (r && v.now !== want) {
|
|
652
|
+
Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set.call(r, String(want));
|
|
653
|
+
r.dispatchEvent(new Event('input', { bubbles: true })); r.dispatchEvent(new Event('change', { bubbles: true }));
|
|
654
|
+
return Number(r.value);
|
|
655
|
+
}
|
|
656
|
+
return v?.now;
|
|
657
|
+
}, value.value).catch(() => null);
|
|
658
|
+
if (got != null && Math.abs(got - value.value) > 1e-9) log(` ⚠ ${acting}: slide stopped at ${got}, not ${value.value} — the slider snaps to its step`);
|
|
659
|
+
}
|
|
599
660
|
await settle();
|
|
600
661
|
await beat();
|
|
601
662
|
break;
|
|
@@ -608,11 +669,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
608
669
|
let box = null;
|
|
609
670
|
const phraseOnly = v.text && Object.keys(v).every((k) => ['text', 'exact'].includes(k));
|
|
610
671
|
if (phraseOnly) {
|
|
672
|
+
await page.evaluate(() => window.__demoScrollIdle()).catch(() => {}); // judge "in view" where the page ends up
|
|
611
673
|
const found = await page.evaluate((t) => window.__demoFind(t), v.text);
|
|
612
|
-
if (found && !found.inView)
|
|
613
|
-
await page.evaluate((ms) => window.__demoScrollTo(window.__demoFound.startContainer.parentElement, ms), fast ? 0 : 900);
|
|
614
|
-
await sleep(fast ? 50 : 950);
|
|
615
|
-
}
|
|
674
|
+
if (found && !found.inView) await page.evaluate((ms) => window.__demoScrollTo(window.__demoFound.startContainer.parentElement, ms), fast ? 0 : 900);
|
|
616
675
|
if (found) box = await page.evaluate(([ms, color, b]) => window.__demoMark('found', { ms, color, box: b }), [ms, color, opt.box ?? false]);
|
|
617
676
|
}
|
|
618
677
|
if (!box) {
|
|
@@ -659,7 +718,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
659
718
|
|
|
660
719
|
// --- setup (not recorded) -----------------------------------------------------------
|
|
661
720
|
// Run a list of actions; on failure, say exactly which one and save a screenshot.
|
|
721
|
+
let acting = '';
|
|
662
722
|
async function runAll(actions, where) {
|
|
723
|
+
acting = where;
|
|
663
724
|
for (const [j, a] of actions.entries()) {
|
|
664
725
|
try {
|
|
665
726
|
// watchdog: an action that hangs (a frozen page, a stalled browser) fails with a clear message
|
|
@@ -759,9 +820,10 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
759
820
|
const n = Math.min(inspectAfter, cfg.steps.length);
|
|
760
821
|
for (const [i, s] of cfg.steps.slice(0, n).entries()) {
|
|
761
822
|
stepHighlight = false;
|
|
762
|
-
// like the recording: marks and
|
|
763
|
-
if (i > 0) await page.evaluate(() => { window.__demoUnmark(); window.__demoUnnote(); }).catch(() => {});
|
|
823
|
+
// like the recording: marks, notes and spotlights end with their step (the last step's stay in the screenshot)
|
|
824
|
+
if (i > 0) await page.evaluate(() => { window.__demoUnmark(true); window.__demoUnnote(true); window.__demoSpotOff(true); }).catch(() => {});
|
|
764
825
|
await runAll(s.actions, `step ${i + 1}`);
|
|
826
|
+
await page.evaluate(() => window.__demoScrollIdle()).catch(() => {});
|
|
765
827
|
}
|
|
766
828
|
await sleep(800);
|
|
767
829
|
const shot = path.join(cfg.work, `inspect-after-${n}.jpg`);
|
|
@@ -804,10 +866,14 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
804
866
|
const zoomTarget = s.zoom?.target && !isPoint(s.zoom.target);
|
|
805
867
|
const startBox = zoomTarget ? await boxNow() : null;
|
|
806
868
|
await runAll(s.actions, `step ${i + 1}`);
|
|
869
|
+
// a scroll still running (the page's own smooth scrolling) ends in this step, not in the next one's first shot
|
|
870
|
+
await page.evaluate(() => window.__demoScrollIdle()).catch(() => {});
|
|
807
871
|
const end = now(); // before clearing: a step that is held for its narration keeps its marks
|
|
808
|
-
|
|
809
|
-
//
|
|
810
|
-
|
|
872
|
+
// docs clips start where the step starts, and a hidden step is cut: clear at once, so no fading
|
|
873
|
+
// callout leaks into the next shot
|
|
874
|
+
const instant = cfg.clips.length > 0 || !!s.hidden;
|
|
875
|
+
await spotOff(instant);
|
|
876
|
+
await page.evaluate((i) => { window.__demoUnmark(i); window.__demoUnnote(i); }, instant).catch(() => {});
|
|
811
877
|
let zoomBox = null;
|
|
812
878
|
if (s.zoom?.target) {
|
|
813
879
|
if (isPoint(s.zoom.target)) {
|
package/src/voice.mjs
CHANGED
|
@@ -118,8 +118,29 @@ function cached(cfg, line) {
|
|
|
118
118
|
return { text, file: path.join(dir, `${hash}.${ext}`), zh, qwen };
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
-
//
|
|
122
|
-
|
|
121
|
+
// A line's sentences, so each is spoken on its own with a pause after it (TTS engines run the
|
|
122
|
+
// sentences of one line together with barely a breath). Latin text breaks after . ! ? when a capital,
|
|
123
|
+
// digit or quote follows, but not after e.g., Dr., a lone initial (N. P. X.), 2.5 or myapp.com;
|
|
124
|
+
// Chinese and Japanese after 。!?.
|
|
125
|
+
const ABBREV = /^(e\.g|i\.e|etc|vs|approx|incl|cf|Mr|Mrs|Ms|Dr|Prof|St|No|Fig)\.$/i;
|
|
126
|
+
export function sentences(text) {
|
|
127
|
+
const s = String(text).replace(/\s+/g, ' ').trim(), out = [];
|
|
128
|
+
const re = /([.!?…]+["'”’)\]]*) +(?=["'“‘(\[]?[\p{Lu}\p{N}\p{Lo}])|([。!?]+[”’」』))]*) */gu;
|
|
129
|
+
let from = 0;
|
|
130
|
+
for (let m; (m = re.exec(s));) {
|
|
131
|
+
if (m[1]) {
|
|
132
|
+
const word = s.slice(from, m.index + 1).match(/(\S+)$/)?.[1] ?? '';
|
|
133
|
+
if (ABBREV.test(word) || /^\p{Lu}\.$/u.test(word)) continue;
|
|
134
|
+
}
|
|
135
|
+
const piece = s.slice(from, m.index + (m[1] ?? m[2]).length).trim();
|
|
136
|
+
if (piece) out.push(piece);
|
|
137
|
+
from = re.lastIndex;
|
|
138
|
+
}
|
|
139
|
+
if (s.slice(from).trim()) out.push(s.slice(from).trim());
|
|
140
|
+
return out;
|
|
141
|
+
}
|
|
142
|
+
// One line as stereo Float32 PCM at 48 kHz, leading and trailing silence trimmed.
|
|
143
|
+
async function speakLine(cfg, line, log) {
|
|
123
144
|
const v = cfg.voice;
|
|
124
145
|
const { text, file, zh, qwen } = cached(cfg, line);
|
|
125
146
|
if (!fs.existsSync(file)) {
|
|
@@ -135,6 +156,80 @@ export async function speak(cfg, line, { log = console.log } = {}) {
|
|
|
135
156
|
return new Float32Array(pcm.buffer, pcm.byteOffset, pcm.length / 4);
|
|
136
157
|
}
|
|
137
158
|
|
|
159
|
+
// Where a line's sentences meet in its audio. Every break in the text (sentence ends, and commas,
|
|
160
|
+
// which voices pause at too) gets an expected time from the letters spoken before it; each quiet
|
|
161
|
+
// stretch belongs to the break it is nearest, and a sentence end takes the longest stretch that is
|
|
162
|
+
// its own, so a long pause at a comma is never mistaken for it. [start, end] in samples per channel,
|
|
163
|
+
// or null when the voice runs straight on (no quiet there to widen).
|
|
164
|
+
const WIN = SR / 100; // 10 ms
|
|
165
|
+
const letters = (s) => (s.match(/[\p{L}\p{N}]/gu) ?? []).length;
|
|
166
|
+
export function sentenceGaps(pcm, sents) {
|
|
167
|
+
const n = Math.floor(pcm.length / 2 / WIN), env = new Float32Array(n);
|
|
168
|
+
let peak = 0;
|
|
169
|
+
for (let w = 0; w < n; w++) {
|
|
170
|
+
let e = 0;
|
|
171
|
+
for (let i = w * WIN; i < (w + 1) * WIN; i++) e = Math.max(e, Math.abs(pcm[2 * i]), Math.abs(pcm[2 * i + 1]));
|
|
172
|
+
env[w] = e; peak = Math.max(peak, e);
|
|
173
|
+
}
|
|
174
|
+
const quiet = [], thr = peak * 0.02;
|
|
175
|
+
for (let w = 0; w < n;) {
|
|
176
|
+
if (env[w] >= thr) { w++; continue; }
|
|
177
|
+
const a = w;
|
|
178
|
+
while (w < n && env[w] < thr) w++;
|
|
179
|
+
if (w - a >= 4 && a > 0 && w < n) quiet.push([a, w]); // ≥ 40 ms, not the trimmed ends
|
|
180
|
+
}
|
|
181
|
+
// breaks in the text, at the share of letters spoken before them: { at, end: k } for the end of sentence k
|
|
182
|
+
const total = letters(sents.join(' ')) || 1, breaks = [];
|
|
183
|
+
let before = 0;
|
|
184
|
+
for (const [k, s] of sents.entries()) {
|
|
185
|
+
for (const m of s.matchAll(/[,;:,、;:]/g)) breaks.push({ at: (before + letters(s.slice(0, m.index))) / total });
|
|
186
|
+
before += letters(s);
|
|
187
|
+
if (k < sents.length - 1) breaks.push({ at: before / total, end: k });
|
|
188
|
+
}
|
|
189
|
+
const mine = sents.slice(0, -1).map(() => []);
|
|
190
|
+
for (const q of quiet) {
|
|
191
|
+
const mid = (q[0] + q[1]) / 2 / n;
|
|
192
|
+
const b = breaks.reduce((x, y) => (Math.abs(y.at - mid) < Math.abs(x.at - mid) ? y : x));
|
|
193
|
+
if (b.end != null && Math.abs(b.at - mid) < 0.25) mine[b.end].push(q);
|
|
194
|
+
}
|
|
195
|
+
let after = 0;
|
|
196
|
+
return mine.map((qs) => {
|
|
197
|
+
const best = qs.filter((q) => q[0] >= after).reduce((x, q) => (!x || q[1] - q[0] > x[1] - x[0] ? q : x), null);
|
|
198
|
+
if (!best) return null;
|
|
199
|
+
after = best[1];
|
|
200
|
+
return [best[0] * WIN, best[1] * WIN];
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// A line as stereo Float32 PCM at 48 kHz, spoken in one go (so the voice keeps its flow from sentence
|
|
205
|
+
// to sentence), with each pause between sentences widened to at least `voice.pause` seconds: voices
|
|
206
|
+
// often run sentences together with barely a breath. parts: where each sentence is, in seconds (for
|
|
207
|
+
// subtitles).
|
|
208
|
+
export async function speak(cfg, line, { log = console.log } = {}) {
|
|
209
|
+
const raw = await speakLine(cfg, line, log), pause = cfg.voice.pause ?? 0;
|
|
210
|
+
const sents = pause > 0 ? sentences(line) : [];
|
|
211
|
+
if (sents.length < 2) return { pcm: raw, parts: [{ text: line, start: 0, end: raw.length / 2 / SR }] };
|
|
212
|
+
const gaps = sentenceGaps(raw, sents), want = Math.round(pause * SR);
|
|
213
|
+
const add = gaps.map((g) => (g ? Math.max(0, want - (g[1] - g[0])) : 0));
|
|
214
|
+
const pcm = new Float32Array(raw.length + 2 * add.reduce((a, b) => a + b, 0));
|
|
215
|
+
const parts = [];
|
|
216
|
+
let from = 0, to = 0, startAt = 0;
|
|
217
|
+
for (const [k, g] of gaps.entries()) {
|
|
218
|
+
if (!g) continue; // no quiet stretch: this sentence runs on into the next
|
|
219
|
+
const mid = Math.round((g[0] + g[1]) / 2);
|
|
220
|
+
pcm.set(raw.subarray(2 * from, 2 * mid), 2 * to);
|
|
221
|
+
to += mid - from; from = mid;
|
|
222
|
+
const end = to - (mid - g[0]);
|
|
223
|
+
to += add[k]; // silence: the array is zero-filled
|
|
224
|
+
parts.push({ text: sents.slice(parts.length ? parts.at(-1).next : 0, k + 1).join(' '), start: startAt / SR, end: end / SR, next: k + 1 });
|
|
225
|
+
startAt = to + (g[1] - mid);
|
|
226
|
+
}
|
|
227
|
+
pcm.set(raw.subarray(2 * from), 2 * to);
|
|
228
|
+
const last = parts.length ? parts.at(-1).next : 0;
|
|
229
|
+
parts.push({ text: sents.slice(last).join(' '), start: startAt / SR, end: pcm.length / 2 / SR, next: sents.length });
|
|
230
|
+
return { pcm, parts: parts.map(({ next, ...p }) => p) };
|
|
231
|
+
}
|
|
232
|
+
|
|
138
233
|
// Speed a clip up (factor > 1) with ffmpeg's pitch-preserving atempo.
|
|
139
234
|
export function tempo(pcm, factor) {
|
|
140
235
|
if (!(factor >= 0.5 && factor <= 100)) throw new Error(`internal: speed factor ${factor} is outside what ffmpeg's atempo accepts (0.5–100)`);
|
|
@@ -146,6 +241,7 @@ export function tempo(pcm, factor) {
|
|
|
146
241
|
// Synthesize every narration line up front and return their lengths (seconds),
|
|
147
242
|
// so `duration: auto` steps can be timed to the voice.
|
|
148
243
|
export async function measureVoice(cfg, { log = console.log } = {}) {
|
|
244
|
+
const seconds = ({ pcm }) => pcm.length / 2 / SR;
|
|
149
245
|
const lens = { steps: {}, endCard: null, intro: null, cards: {} };
|
|
150
246
|
if (!cfg.voice) return lens;
|
|
151
247
|
if (cfg.voice.fallback) log(`note: ${cfg.voice.fallback}`);
|
|
@@ -157,10 +253,10 @@ export async function measureVoice(cfg, { log = console.log } = {}) {
|
|
|
157
253
|
await (zh ? synthZh : synthQwen)(cfg.voice, [...todo.values()], { log });
|
|
158
254
|
}
|
|
159
255
|
for (const [i, s] of cfg.steps.entries()) {
|
|
160
|
-
if (s.say && !s.hidden) lens.steps[i] = (await speak(cfg, s.say, { log }))
|
|
161
|
-
if (s.card?.say && !s.hidden) lens.cards[i] = (await speak(cfg, s.card.say, { log }))
|
|
256
|
+
if (s.say && !s.hidden) lens.steps[i] = seconds(await speak(cfg, s.say, { log }));
|
|
257
|
+
if (s.card?.say && !s.hidden) lens.cards[i] = seconds(await speak(cfg, s.card.say, { log }));
|
|
162
258
|
}
|
|
163
|
-
if (cfg.intro?.say) lens.intro = (await speak(cfg, cfg.intro.say, { log }))
|
|
164
|
-
if (cfg.endCard?.say) lens.endCard = (await speak(cfg, cfg.endCard.say, { log }))
|
|
259
|
+
if (cfg.intro?.say) lens.intro = seconds(await speak(cfg, cfg.intro.say, { log }));
|
|
260
|
+
if (cfg.endCard?.say) lens.endCard = seconds(await speak(cfg, cfg.endCard.say, { log }));
|
|
165
261
|
return lens;
|
|
166
262
|
}
|