storyshot 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -14
- package/package.json +1 -1
- package/skills/storyshot/SKILL.md +81 -12
- package/src/audio.mjs +9 -4
- package/src/check.mjs +46 -2
- package/src/compose.mjs +5 -0
- package/src/config.mjs +31 -15
- package/src/doctor.mjs +6 -0
- package/src/init.mjs +10 -2
- package/src/inspect.mjs +13 -5
- package/src/qwen-tts.mjs +155 -0
- package/src/qwen_tts.py +186 -0
- package/src/record.mjs +99 -33
- package/src/voice.mjs +139 -22
package/src/qwen-tts.mjs
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// Narration with Qwen3-TTS 0.6B CustomVoice (Apache-2.0): nine preset voices, five of them Chinese
|
|
2
|
+
// (serena, vivian, uncle_fu, dylan with a Beijing accent, eric with a Sichuan one), free and local.
|
|
3
|
+
// It runs through mlx-audio, so on Apple Silicon only; elsewhere config.mjs falls back to kokoro.
|
|
4
|
+
// Python (qwen_tts.py) runs in storyshot's own venv, once per render for all uncached lines.
|
|
5
|
+
import { execFileSync, spawn, spawnSync } from 'node:child_process';
|
|
6
|
+
import crypto from 'node:crypto';
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import os from 'node:os';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
import { fileURLToPath } from 'node:url';
|
|
11
|
+
import { findPython } from './kokoro-zh.mjs';
|
|
12
|
+
|
|
13
|
+
export const QWEN_VOICES = ['serena', 'vivian', 'uncle_fu', 'dylan', 'eric', 'ryan', 'aiden', 'ono_anna', 'sohee'];
|
|
14
|
+
export const qwenSupported = () => process.platform === 'darwin' && process.arch === 'arm64';
|
|
15
|
+
// `Uncle Fu`, `Serena` → uncle_fu, serena (null when it isn't one of the nine)
|
|
16
|
+
export const qwenVoice = (name) => {
|
|
17
|
+
const v = String(name ?? '').trim().toLowerCase().replace(/[\s-]+/g, '_');
|
|
18
|
+
return QWEN_VOICES.includes(v) ? v : null;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const REPO = 'mlx-community/Qwen3-TTS-12Hz-0.6B-CustomVoice-bf16';
|
|
22
|
+
const REVISION = '6415d95f88be018ff9e46813119dc3bc12261328';
|
|
23
|
+
// 0.9 (the model's default) now and then runs on into made-up words; 0.7 keeps the same voice steadier
|
|
24
|
+
const TEMPERATURE = 0.7;
|
|
25
|
+
// Each take is checked by ear: Qwen3-ASR (Apache-2.0, ~1 GB) transcribes it, and a line it hears
|
|
26
|
+
// differently is sampled again (pypinyin compares Chinese by sound, so homophones don't count)
|
|
27
|
+
const ASR_REPO = 'mlx-community/Qwen3-ASR-0.6B-8bit';
|
|
28
|
+
const ASR_REVISION = '89e96d92ba34aca20b3e29fb10cc284097d1219f';
|
|
29
|
+
const REQUIREMENTS = ['mlx-audio==0.6.3', 'mlx==0.32.4', 'mlx-metal==0.32.4', 'transformers==5.22.0', 'huggingface-hub==2.4.0',
|
|
30
|
+
'tokenizers==0.23.3', 'safetensors==0.8.0', 'numpy==2.5.4', 'pypinyin==0.55.0'];
|
|
31
|
+
// Part of every cached line's hash: change it when the model, the packages or qwen_tts.py's sampling change.
|
|
32
|
+
export const QWEN_TTS_ID = `${REPO}@${REVISION.slice(0, 7)}|${REQUIREMENTS[0]}|t${TEMPERATURE}|guard-2|${ASR_REPO}@${ASR_REVISION.slice(0, 7)}`;
|
|
33
|
+
const CACHE = path.join(os.homedir(), '.cache', 'storyshot');
|
|
34
|
+
export const QWEN_VENV = path.join(CACHE, 'qwen-venv');
|
|
35
|
+
const HELPER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'qwen_tts.py');
|
|
36
|
+
const HUB = process.env.HF_HUB_CACHE ?? path.join(process.env.HF_HOME ?? path.join(os.homedir(), '.cache', 'huggingface'), 'hub');
|
|
37
|
+
const MODEL_FILE = path.join(HUB, `models--${REPO.replace('/', '--')}`, 'snapshots', REVISION, 'model.safetensors');
|
|
38
|
+
const ASR_FILE = path.join(HUB, `models--${ASR_REPO.replace('/', '--')}`, 'snapshots', ASR_REVISION, 'model.safetensors');
|
|
39
|
+
|
|
40
|
+
// --- text (TTS only: subtitles keep the original) --------------------------------------------------
|
|
41
|
+
// The model reads = + 2个 第2行 x² itself; a minus sign before a number it drops, and a number like
|
|
42
|
+
// 2e-105 sets it babbling, so those are spelled out in Chinese lines.
|
|
43
|
+
export function qwenSpeakText(text, lang = qwenLang(text)) {
|
|
44
|
+
let t = String(text).replace(/\s*\n+\s*/g, ' ');
|
|
45
|
+
if (lang === 'chinese') {
|
|
46
|
+
t = t.replace(/(\d(?:\.\d+)?)[eE]\s*([-−+]?)\s*(\d+)/g, (_, m, s, e) => `${m}乘10的${s && s !== '+' ? '负' : ''}${e}次方`)
|
|
47
|
+
.replace(/(?<![\w)²³]\s*)−\s*(?=\d)/g, '负');
|
|
48
|
+
}
|
|
49
|
+
return t;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// The language the model is told: the script decides; Latin text is left to the model.
|
|
53
|
+
export function qwenLang(text) {
|
|
54
|
+
const s = String(text).replace(/\d+(?:\.\d+)?(?:[eE][-−+]?\d+)?/g, ' '); // the e in 2e-105 is no English word
|
|
55
|
+
if (/[-ヿ]/.test(s)) return 'japanese';
|
|
56
|
+
if (/[가-]/.test(s)) return 'korean';
|
|
57
|
+
const han = (s.match(/[㐀-鿿]/g) ?? []).length, latin = (s.match(/[A-Za-z]+/g) ?? []).length;
|
|
58
|
+
return han > latin ? 'chinese' : 'auto';
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// --- the venv ---------------------------------------------------------------------------------------
|
|
62
|
+
const venvPython = (dir) => path.join(dir, 'bin', 'python');
|
|
63
|
+
const STAMP = 'storyshot-requirements.txt';
|
|
64
|
+
const venvReady = () => {
|
|
65
|
+
try { return fs.readFileSync(path.join(QWEN_VENV, STAMP), 'utf8') === REQUIREMENTS.join('\n') && fs.existsSync(venvPython(QWEN_VENV)); } catch { return false; }
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const NO_PYTHON = () => 'the Qwen3-TTS voice needs Python 3.10–3.12 to run (brew install python@3.12, or uv python install 3.12, ' +
|
|
69
|
+
`or set STORYSHOT_PYTHON); storyshot then keeps its packages in its own venv (${QWEN_VENV}), nothing is installed globally`;
|
|
70
|
+
|
|
71
|
+
// For doctor and check: can this machine use it, and what's still to come on first use?
|
|
72
|
+
export function qwenStatus() {
|
|
73
|
+
if (!qwenSupported()) return { supported: false };
|
|
74
|
+
const model = fs.existsSync(MODEL_FILE) && fs.existsSync(ASR_FILE);
|
|
75
|
+
if (venvReady()) return { supported: true, ready: true, venv: QWEN_VENV, model };
|
|
76
|
+
const python = findPython();
|
|
77
|
+
return { supported: true, ready: false, venv: QWEN_VENV, model, python, problem: python ? null : NO_PYTHON() };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function ensureVenv(log) {
|
|
81
|
+
if (venvReady()) return venvPython(QWEN_VENV);
|
|
82
|
+
const run = (cmd, args, what) => {
|
|
83
|
+
const r = spawnSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
|
|
84
|
+
if (r.status !== 0) throw new Error(`${what} failed: ${(r.stderr || r.stdout || r.error?.message || '').trim().split('\n').slice(-6).join('\n')}`);
|
|
85
|
+
};
|
|
86
|
+
if (fs.existsSync(venvPython(QWEN_VENV))) {
|
|
87
|
+
// a venv from an older storyshot: bring its packages up to date in place, rebuild it only if that fails
|
|
88
|
+
log(`updating the Qwen3-TTS venv in ${QWEN_VENV}…`);
|
|
89
|
+
try {
|
|
90
|
+
run(venvPython(QWEN_VENV), ['-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q', ...REQUIREMENTS], 'updating the venv');
|
|
91
|
+
fs.writeFileSync(path.join(QWEN_VENV, STAMP), REQUIREMENTS.join('\n'));
|
|
92
|
+
return venvPython(QWEN_VENV);
|
|
93
|
+
} catch (e) { log(` ${e.message.split('\n')[0]} — rebuilding it`); }
|
|
94
|
+
}
|
|
95
|
+
const py = findPython();
|
|
96
|
+
if (!py) throw new Error(NO_PYTHON());
|
|
97
|
+
log(`setting up the Qwen3-TTS voice (one time): a Python ${py.version} venv in ${QWEN_VENV} with mlx-audio (~400 MB, 1–2 min)…`);
|
|
98
|
+
fs.rmSync(QWEN_VENV, { recursive: true, force: true });
|
|
99
|
+
fs.mkdirSync(CACHE, { recursive: true });
|
|
100
|
+
run(py.exe, ['-m', 'venv', QWEN_VENV], `creating the venv with ${py.exe} -m venv`);
|
|
101
|
+
run(venvPython(QWEN_VENV), ['-m', 'pip', 'install', '--disable-pip-version-check', '--no-input', '-q', ...REQUIREMENTS], 'installing mlx-audio into the venv');
|
|
102
|
+
fs.writeFileSync(path.join(QWEN_VENV, STAMP), REQUIREMENTS.join('\n')); // written last: a half-built venv is rebuilt next time
|
|
103
|
+
return venvPython(QWEN_VENV);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// --- synthesis --------------------------------------------------------------------------------------
|
|
107
|
+
// entries: [{ text, file }] (text already through qwenSpeakText) → a WAV at each `file`.
|
|
108
|
+
export async function synthQwen(v, entries, { log = console.log } = {}) {
|
|
109
|
+
if (!entries.length) return;
|
|
110
|
+
if (!qwenSupported()) throw new Error('the Qwen3-TTS voice runs on Apple Silicon Macs only (it uses MLX)');
|
|
111
|
+
const py = ensureVenv(log);
|
|
112
|
+
if (!fs.existsSync(MODEL_FILE)) log(`downloading the Qwen3-TTS model (one time, ~2.3 GB) into ${HUB}…`);
|
|
113
|
+
if (!fs.existsSync(ASR_FILE)) log(`downloading the Qwen3-ASR model that listens to each take (one time, ~1 GB) into ${HUB}…`);
|
|
114
|
+
const lines = entries.map((e) => ({
|
|
115
|
+
text: e.text, file: `${e.file}.raw.wav`, voice: v.voice, lang: qwenLang(e.text),
|
|
116
|
+
// the same line gets the same take: a seed from the text
|
|
117
|
+
seed: parseInt(crypto.createHash('sha1').update(e.text).digest('hex').slice(0, 7), 16),
|
|
118
|
+
}));
|
|
119
|
+
const job = JSON.stringify({ repo: REPO, revision: REVISION, temperature: TEMPERATURE, asr: { repo: ASR_REPO, revision: ASR_REVISION }, lines });
|
|
120
|
+
const { code, stdout, tail } = await new Promise((resolve, reject) => {
|
|
121
|
+
const p = spawn(py, ['-I', HELPER], { env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1', TRANSFORMERS_VERBOSITY: 'error' } });
|
|
122
|
+
let out = '', err = '', buf = '';
|
|
123
|
+
p.stdout.on('data', (d) => { out += d; });
|
|
124
|
+
p.stderr.on('data', (d) => {
|
|
125
|
+
err = (err + d).slice(-4000);
|
|
126
|
+
buf += d;
|
|
127
|
+
let nl;
|
|
128
|
+
while ((nl = buf.search(/[\r\n]/)) >= 0) {
|
|
129
|
+
const m = buf.slice(0, nl).match(/^line (\d+)\/\d+ (\S+)( \d+ tries)?/);
|
|
130
|
+
buf = buf.slice(nl + 1);
|
|
131
|
+
if (m) log(` tts (qwen/${v.voice}, ${m[2]}${m[3] ? `,${m[3]}` : ''}): "${entries[m[1] - 1].text}"`);
|
|
132
|
+
}
|
|
133
|
+
});
|
|
134
|
+
p.on('error', reject);
|
|
135
|
+
p.on('close', (c) => resolve({ code: c, stdout: out, tail: err }));
|
|
136
|
+
p.stdin.end(job);
|
|
137
|
+
});
|
|
138
|
+
if (code !== 0) throw new Error(`Qwen3-TTS failed (${py} ${HELPER}): ${tail.trim().split('\n').slice(-8).join('\n')}`);
|
|
139
|
+
const { results, asr } = JSON.parse(stdout);
|
|
140
|
+
if (asr) log(` ⚠ qwen: the speech check couldn't load (${asr}), so these takes were checked by length only`);
|
|
141
|
+
for (const [i, e] of entries.entries()) {
|
|
142
|
+
const r = results[i];
|
|
143
|
+
if (r?.suspect) {
|
|
144
|
+
log(r.heard != null && r.mismatch > 0
|
|
145
|
+
? ` ⚠ qwen: "${e.text}" still doesn't come out right after ${r.tries} tries — it sounds like "${r.heard}". Listen to it; rephrase the line or add a pronounce entry`
|
|
146
|
+
: ` ⚠ qwen: "${e.text}" still sounds off after ${r.tries} tries (${r.seconds.toFixed(1)}s) — listen to it, or rephrase the line / add a pronounce entry`);
|
|
147
|
+
}
|
|
148
|
+
const raw = lines[i].file, tmp = `${e.file}.tmp.wav`;
|
|
149
|
+
// the model has no speed control: ffmpeg's pitch-preserving atempo does it
|
|
150
|
+
if ((v.speed ?? 1) !== 1) execFileSync('ffmpeg', ['-loglevel', 'error', '-y', '-i', raw, '-af', `atempo=${v.speed}`, tmp]);
|
|
151
|
+
else fs.renameSync(raw, tmp);
|
|
152
|
+
fs.rmSync(raw, { force: true });
|
|
153
|
+
fs.renameSync(tmp, e.file);
|
|
154
|
+
}
|
|
155
|
+
}
|
package/src/qwen_tts.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# Qwen3-TTS CustomVoice through mlx-audio (Apple Silicon), for `voice: { provider: qwen }`. Runs in
|
|
2
|
+
# storyshot's own venv (qwen-tts.mjs), one process per render: the model loads once for all lines.
|
|
3
|
+
#
|
|
4
|
+
# stdin: {"repo", "revision", "temperature", "asr": {"repo", "revision"} | null,
|
|
5
|
+
# "lines": [{"text", "file", "voice", "lang", "seed"}]}
|
|
6
|
+
# stderr: "line i/n <seconds>s [<tries> tries]" per line, plus whatever the libraries log
|
|
7
|
+
# stdout: {"results": [{"seconds", "tries", "suspect", "heard", "mismatch"}], "asr": null | "why it didn't load"}
|
|
8
|
+
#
|
|
9
|
+
# The model samples its speech tokens, so now and then a take garbles a few words, drops one, runs on
|
|
10
|
+
# into made-up words (a scientific number like 2e-105 does it) or stops early. Each take is checked:
|
|
11
|
+
# its length against the text, and what a speech recognizer (Qwen3-ASR) hears in it against the text.
|
|
12
|
+
# A take that fails is sampled again with another seed, and the best one is kept.
|
|
13
|
+
import contextlib
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
|
|
19
|
+
os.environ.setdefault('HF_HUB_DISABLE_TELEMETRY', '1')
|
|
20
|
+
os.environ.setdefault('TRANSFORMERS_VERBOSITY', 'error')
|
|
21
|
+
|
|
22
|
+
TRIES = 4
|
|
23
|
+
# a take is sampled again when the recognizer hears this share of the line differently, or this long a run
|
|
24
|
+
# of mistakes in a row (a garble: 负 2 x → 负二长方荡 is 2.5); a warning if every take is still off
|
|
25
|
+
RETRY_AT, RETRY_RUN = 0.10, 2
|
|
26
|
+
WARN_AT, WARN_RUN = 0.15, 2
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def expected(text):
|
|
30
|
+
"""Rough speaking time in seconds: ~3.6 Chinese characters a second, Latin words and numbers."""
|
|
31
|
+
han = len(re.findall(r'[㐀-鿿]', text))
|
|
32
|
+
words = len(re.findall(r'[A-Za-z]+', text))
|
|
33
|
+
nums = len(re.findall(r'\d+(?:\.\d+)?', text))
|
|
34
|
+
return han / 3.6 + words * 0.4 + nums * 0.5 + 0.3
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def plausible(seconds, est):
|
|
38
|
+
return 0.35 * est - 0.5 <= seconds <= 2.0 * est + 2.0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# --- what was said vs. what was meant ---------------------------------------------------------------
|
|
42
|
+
# Chinese is compared by sound (toneless pinyin), so a homophone the recognizer picks (交点/焦点) is no
|
|
43
|
+
# mistake; numbers are read out first (0.5 → 零点五, 2 → 二), since the recognizer writes either form.
|
|
44
|
+
DIGITS = '零一二三四五六七八九'
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def cn_int(n):
|
|
48
|
+
if n >= 10 ** 8:
|
|
49
|
+
return ''.join(DIGITS[int(c)] for c in str(n))
|
|
50
|
+
|
|
51
|
+
def four(x):
|
|
52
|
+
out, zero = '', False
|
|
53
|
+
for i, c in enumerate(f'{x:04d}'):
|
|
54
|
+
d = int(c)
|
|
55
|
+
if d == 0:
|
|
56
|
+
zero = bool(out)
|
|
57
|
+
else:
|
|
58
|
+
out += ('零' if zero else '') + DIGITS[d] + ['千', '百', '十', ''][i]
|
|
59
|
+
zero = False
|
|
60
|
+
return out
|
|
61
|
+
|
|
62
|
+
if n < 10:
|
|
63
|
+
return DIGITS[n]
|
|
64
|
+
hi, lo = divmod(n, 10000)
|
|
65
|
+
out = (four(hi) + '万' if hi else '') + (('零' if hi and lo < 1000 else '') + four(lo) if lo else '')
|
|
66
|
+
return re.sub('^一十', '十', out)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def cn_number(m):
|
|
70
|
+
whole, _, frac = m.group().partition('.')
|
|
71
|
+
return cn_int(int(whole)) + ('点' + ''.join(DIGITS[int(c)] for c in frac) if frac else '')
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def units(text, chinese, pinyin):
|
|
75
|
+
"""The line as a list of sounds: pinyin syllables, digits, and Latin words as '#word' (m x and mx as '#m', '#x')."""
|
|
76
|
+
t = text.lower().replace('−', '-')
|
|
77
|
+
if chinese:
|
|
78
|
+
t = re.sub(r'\d+(?:\.\d+)?', cn_number, t)
|
|
79
|
+
out = []
|
|
80
|
+
for tok in re.findall(r'[㐀-鿿]|[a-z]+|\d+', t):
|
|
81
|
+
if chinese and re.match(r'[㐀-鿿]', tok):
|
|
82
|
+
out += [{'liang': 'er'}.get(p, p) for p in pinyin(tok)]
|
|
83
|
+
elif tok.isdigit():
|
|
84
|
+
out += list(tok) # digit by digit: a wrong number (260 → 45) is several mistakes in a row, not one
|
|
85
|
+
else:
|
|
86
|
+
out += ['#' + c for c in tok] if len(tok) <= 2 else ['#' + tok]
|
|
87
|
+
return out
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def mismatch(text, heard, chinese, pinyin):
|
|
91
|
+
"""(share of the line heard differently, the longest run of mistakes in a row), both in units of the
|
|
92
|
+
text. A garbled take has a run of several; a homophone or a misheard letter is one on its own."""
|
|
93
|
+
a, b = units(text, chinese, pinyin), units(heard, chinese, pinyin)
|
|
94
|
+
letter = lambda u: len(u) == 2 and u[0] == '#'
|
|
95
|
+
d = {(0, 0): (0, None)}
|
|
96
|
+
for i in range(len(a) + 1):
|
|
97
|
+
for j in range(len(b) + 1):
|
|
98
|
+
if i == j == 0:
|
|
99
|
+
continue
|
|
100
|
+
ways = []
|
|
101
|
+
if i:
|
|
102
|
+
ways.append((d[i - 1, j][0] + 1, (i - 1, j, 1)))
|
|
103
|
+
if j:
|
|
104
|
+
ways.append((d[i, j - 1][0] + 1, (i, j - 1, 1)))
|
|
105
|
+
if i and j:
|
|
106
|
+
x, y = a[i - 1], b[j - 1]
|
|
107
|
+
# a variable (m, b, x) the recognizer wrote as a character is half a mistake
|
|
108
|
+
cost = 0 if x == y else 0.5 if letter(x) or letter(y) else 1
|
|
109
|
+
ways.append((d[i - 1, j - 1][0] + cost, (i - 1, j - 1, cost)))
|
|
110
|
+
# a name in Latin letters in a Chinese line (Desmos) can come back as a few syllables: one mistake
|
|
111
|
+
if chinese and len(x) > 3 and x[0] == '#':
|
|
112
|
+
ways += [(d[i - 1, j - k][0] + 1, (i - 1, j - k, 1)) for k in range(2, min(5, j) + 1)]
|
|
113
|
+
d[i, j] = min(ways, key=lambda w: w[0])
|
|
114
|
+
i, j, run, worst = len(a), len(b), 0, 0
|
|
115
|
+
while (i, j) != (0, 0):
|
|
116
|
+
i, j, cost = d[i, j][1]
|
|
117
|
+
run = run + cost if cost else 0
|
|
118
|
+
worst = max(worst, run)
|
|
119
|
+
return d[len(a), len(b)][0] / max(1, len(a)), worst
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def main():
|
|
123
|
+
job = json.load(sys.stdin)
|
|
124
|
+
out = sys.stdout
|
|
125
|
+
# the libraries print their own notes on stdout: keep it for the JSON answer
|
|
126
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
127
|
+
import math
|
|
128
|
+
import mlx.core as mx
|
|
129
|
+
import numpy as np
|
|
130
|
+
from huggingface_hub import snapshot_download
|
|
131
|
+
from mlx_audio.audio_io import write
|
|
132
|
+
from mlx_audio.tts.utils import load_model
|
|
133
|
+
|
|
134
|
+
model = load_model(snapshot_download(job['repo'], revision=job['revision']))
|
|
135
|
+
asr, asr_error, pinyin = None, None, None
|
|
136
|
+
if job.get('asr'):
|
|
137
|
+
try:
|
|
138
|
+
from mlx_audio.stt.utils import load_model as load_stt
|
|
139
|
+
from pypinyin import lazy_pinyin as pinyin
|
|
140
|
+
asr = load_stt(snapshot_download(job['asr']['repo'], revision=job['asr']['revision']))
|
|
141
|
+
except Exception as e: # offline before the first download, say: check by length only
|
|
142
|
+
asr_error = f'{type(e).__name__}: {e}'.splitlines()[0][:300]
|
|
143
|
+
|
|
144
|
+
def hear(audio, line):
|
|
145
|
+
"""(what the recognizer heard, mismatch), or (None, None) for a line it can't check."""
|
|
146
|
+
chinese = line['lang'] == 'chinese'
|
|
147
|
+
if asr is None or not (chinese or (line['lang'] == 'auto' and not re.search(r'[㐀-鿿]', line['text']))):
|
|
148
|
+
return None, None
|
|
149
|
+
tmp = line['file'] + '.check.wav'
|
|
150
|
+
write(tmp, audio, model.sample_rate)
|
|
151
|
+
try:
|
|
152
|
+
heard = asr.generate(tmp, language='Chinese' if chinese else 'English').text.strip()
|
|
153
|
+
finally:
|
|
154
|
+
os.remove(tmp)
|
|
155
|
+
return heard, mismatch(line['text'], heard, chinese, pinyin)
|
|
156
|
+
|
|
157
|
+
results, n = [], len(job['lines'])
|
|
158
|
+
for i, line in enumerate(job['lines']):
|
|
159
|
+
est = expected(line['text'])
|
|
160
|
+
best = None
|
|
161
|
+
for k in range(TRIES):
|
|
162
|
+
mx.random.seed(line['seed'] + k * 7919)
|
|
163
|
+
parts = model.generate(text=line['text'], voice=line['voice'], lang_code=line['lang'],
|
|
164
|
+
temperature=job['temperature'])
|
|
165
|
+
audio = np.concatenate([np.array(r.audio, dtype=np.float32) for r in parts])
|
|
166
|
+
seconds = len(audio) / model.sample_rate
|
|
167
|
+
fits = plausible(seconds, est)
|
|
168
|
+
heard, m = hear(audio, line) if fits else (None, None)
|
|
169
|
+
# how far off the recognizer heard it: up to 1 is fine (0 when it couldn't check)
|
|
170
|
+
worse = max(m[0] / RETRY_AT, m[1] / RETRY_RUN) if m else 0
|
|
171
|
+
# a take of the right length that says the line beats one that doesn't; then the closest length
|
|
172
|
+
rank = (not fits, worse if worse > 1 else 0, abs(math.log(max(seconds, 0.1) / est)))
|
|
173
|
+
if best is None or rank < best['rank']:
|
|
174
|
+
best = {'rank': rank, 'audio': audio, 'seconds': seconds, 'fits': fits, 'heard': heard, 'm': m}
|
|
175
|
+
if fits and worse <= 1:
|
|
176
|
+
break
|
|
177
|
+
write(line['file'], best['audio'], model.sample_rate)
|
|
178
|
+
m = best['m']
|
|
179
|
+
results.append({'seconds': best['seconds'], 'tries': k + 1, 'heard': best['heard'], 'mismatch': m and m[0],
|
|
180
|
+
'suspect': not best['fits'] or bool(m and (m[0] > WARN_AT or m[1] > WARN_RUN))})
|
|
181
|
+
print(f'line {i + 1}/{n} {best["seconds"]:.1f}s' + (f' {k + 1} tries' if k else ''), file=sys.stderr, flush=True)
|
|
182
|
+
out.write(json.dumps({'results': results, 'asr': asr_error}))
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
if __name__ == '__main__':
|
|
186
|
+
main()
|
package/src/record.mjs
CHANGED
|
@@ -55,19 +55,60 @@ function pageScript({ scrollOffset }) {
|
|
|
55
55
|
for (let p = el.parentElement; p && p !== document.body; p = p.parentElement) if (scrollable(p)) return p;
|
|
56
56
|
return docEl();
|
|
57
57
|
};
|
|
58
|
+
// `behavior: 'instant'` on every frame: with `scroll-behavior: smooth` in the page's CSS (GitHub has it),
|
|
59
|
+
// plain scrollTop writes each start a smooth scroll of their own, so the page lags behind and is still
|
|
60
|
+
// moving after the step that scrolled it is over
|
|
58
61
|
window.__demoScrollBy = (dy, ms = 1200, sc = mainScroller()) => new Promise((done) => {
|
|
59
62
|
const start = sc.scrollTop, t0 = performance.now();
|
|
60
63
|
const ease = (t) => (t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2);
|
|
61
64
|
const step = (now) => {
|
|
62
|
-
const k = Math.min(1, (now - t0) / ms);
|
|
63
|
-
sc.
|
|
65
|
+
const k = ms > 0 ? Math.min(1, Math.max(0, (now - t0) / ms)) : 1;
|
|
66
|
+
sc.scrollTo({ top: start + dy * ease(k), behavior: 'instant' });
|
|
64
67
|
if (k < 1) requestAnimationFrame(step); else done();
|
|
65
68
|
};
|
|
66
69
|
requestAnimationFrame(step);
|
|
67
70
|
});
|
|
71
|
+
// How far down a header will cover the window once `el` is scrolled to the top: bars fixed at the top,
|
|
72
|
+
// and sticky ones that stick inside a box holding `el` (GitHub's README bar sticks within the README).
|
|
73
|
+
const headerCover = (el) => {
|
|
74
|
+
let cover = 0;
|
|
75
|
+
for (const h of document.querySelectorAll('body *')) {
|
|
76
|
+
if (h.getAttribute('id')?.startsWith('__demo') || h.contains(el)) continue; // not h.id: a form with an <input name=id> has that input as its .id
|
|
77
|
+
const cs = getComputedStyle(h);
|
|
78
|
+
if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;
|
|
79
|
+
const r = h.getBoundingClientRect();
|
|
80
|
+
if (!r.height || r.height > innerHeight * 0.3 || r.width < innerWidth * 0.3 || cs.visibility === 'hidden' || cs.opacity === '0') continue;
|
|
81
|
+
if (cs.position === 'fixed' && r.top <= 1) cover = Math.max(cover, r.bottom);
|
|
82
|
+
const stick = parseFloat(cs.top);
|
|
83
|
+
if (cs.position === 'sticky' && !Number.isNaN(stick) && stick < innerHeight * 0.3 && h.parentElement?.contains(el)) cover = Math.max(cover, stick + r.height);
|
|
84
|
+
}
|
|
85
|
+
return cover;
|
|
86
|
+
};
|
|
87
|
+
// the element's top lands `scrollOffset` px down, or just below a header that would cover it there
|
|
68
88
|
window.__demoScrollTo = (el, ms) => {
|
|
69
|
-
const sc = scrollerFor(el),
|
|
70
|
-
|
|
89
|
+
const sc = scrollerFor(el), docs = sc === docEl(), top = docs ? 0 : sc.getBoundingClientRect().top;
|
|
90
|
+
const offset = docs ? Math.max(scrollOffset, headerCover(el) + 12) : scrollOffset;
|
|
91
|
+
return window.__demoScrollBy(el.getBoundingClientRect().top - top - offset, ms, sc);
|
|
92
|
+
};
|
|
93
|
+
// resolves once nothing has scrolled for a few frames (the page's own smooth scrolling included)
|
|
94
|
+
window.__demoScrollIdle = (max = 2000) => new Promise((done) => {
|
|
95
|
+
const sc = mainScroller(), pos = () => `${scrollX},${scrollY},${sc.scrollTop}`;
|
|
96
|
+
const end = performance.now() + max;
|
|
97
|
+
let last = pos(), still = 0;
|
|
98
|
+
const check = () => {
|
|
99
|
+
const now = pos();
|
|
100
|
+
still = now === last ? still + 1 : 0; last = now;
|
|
101
|
+
if (still >= 3 || performance.now() > end) done(); else requestAnimationFrame(check);
|
|
102
|
+
};
|
|
103
|
+
requestAnimationFrame(check);
|
|
104
|
+
});
|
|
105
|
+
// a slider's value and range: an <input type=range>, or an ARIA slider (Desmos: [role=slider] on its knob)
|
|
106
|
+
window.__demoSliderValue = (el) => {
|
|
107
|
+
const r = el.type === 'range' ? el : el.querySelector?.('input[type=range]');
|
|
108
|
+
if (r) return { now: Number(r.value), lo: Number(r.min || 0), hi: Number(r.max || 100), native: r };
|
|
109
|
+
const aria = el.hasAttribute('aria-valuenow') ? el : el.querySelector?.('[aria-valuenow]');
|
|
110
|
+
const now = Number(aria?.getAttribute('aria-valuenow'));
|
|
111
|
+
return aria && Number.isFinite(now) ? { now, lo: Number(aria.getAttribute('aria-valuemin') ?? 0), hi: Number(aria.getAttribute('aria-valuemax') ?? 100) } : null;
|
|
71
112
|
};
|
|
72
113
|
// Tutorial callouts: spotlight a target, show a pressed key combo.
|
|
73
114
|
window.__demoSpot = (r) => {
|
|
@@ -86,9 +127,10 @@ function pageScript({ scrollOffset }) {
|
|
|
86
127
|
Object.assign(el.style, { left: `${r.x - pad}px`, top: `${r.y - pad}px`, width: `${r.width + 2 * pad}px`, height: `${r.height + 2 * pad}px` });
|
|
87
128
|
requestAnimationFrame(() => { el.style.opacity = '1'; });
|
|
88
129
|
};
|
|
89
|
-
window.__demoSpotOff = () => {
|
|
130
|
+
window.__demoSpotOff = (instant) => {
|
|
90
131
|
const el = document.getElementById('__demo_spot');
|
|
91
|
-
if (el
|
|
132
|
+
if (el && instant) el.remove();
|
|
133
|
+
else if (el) { el.style.opacity = '0'; setTimeout(() => el.remove(), 300); }
|
|
92
134
|
};
|
|
93
135
|
window.__demoKeys = (label) => {
|
|
94
136
|
const el = document.createElement('div');
|
|
@@ -424,7 +466,7 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
424
466
|
await sleep(280);
|
|
425
467
|
return true;
|
|
426
468
|
}
|
|
427
|
-
const spotOff = () => page.evaluate(() => window.__demoSpotOff()).catch(() => {});
|
|
469
|
+
const spotOff = (instant) => page.evaluate((i) => window.__demoSpotOff(i), !!instant).catch(() => {});
|
|
428
470
|
const keyLabel = (k) => k.split('+').map((p) => ({ Meta: '⌘', ControlOrMeta: '⌘', Control: 'Ctrl', Shift: '⇧', Alt: '⌥', Enter: '↵ Enter', Escape: 'Esc', Backspace: '⌫', Tab: '⇥ Tab' })[p] ?? (p.length === 1 ? p.toUpperCase() : p)).join(' ');
|
|
429
471
|
|
|
430
472
|
async function runAction(action) {
|
|
@@ -473,6 +515,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
473
515
|
case 'scrollTo': {
|
|
474
516
|
const h = await locate(page, spec).elementHandle({ timeout: opt.timeout ?? 15000 });
|
|
475
517
|
await page.evaluate(([el, ms]) => window.__demoScrollTo(el, ms), [h, opt.ms ?? 1200]);
|
|
518
|
+
// where it landed: a zoom on the same target frames it there, not where it was before the scroll
|
|
519
|
+
const b = await h.boundingBox().catch(() => null);
|
|
520
|
+
if (b) boxes.set(keyOf(spec), b);
|
|
476
521
|
break;
|
|
477
522
|
}
|
|
478
523
|
case 'scroll': await page.evaluate(([dy, ms]) => window.__demoScrollBy(dy, ms), [value.y ?? value, value.ms ?? 1000]); break;
|
|
@@ -570,25 +615,28 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
570
615
|
}
|
|
571
616
|
case 'slide': {
|
|
572
617
|
// slide: { target: <slider>, to: 0.2 } — drag a slider to a position along its track (0 = start,
|
|
573
|
-
// 1 = end),
|
|
574
|
-
|
|
618
|
+
// 1 = end), or { target, value: 3 } — to a value, for sliders that report their range; found from
|
|
619
|
+
// the element instead of pixel positions that break when the layout moves
|
|
620
|
+
if (!value?.target || (typeof value?.to !== 'number' && typeof value?.value !== 'number')) throw new Error('slide needs { target: <slider>, to: 0..1 } or { target: <slider>, value: <number> }');
|
|
575
621
|
const p = await pointOf(value.target, value);
|
|
576
|
-
const
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
622
|
+
const s = await locate(page, value.target).evaluate((el) => {
|
|
623
|
+
// a knob (Desmos's [role=slider] is its 38 px thumb): slide along its track, the nearest wide parent
|
|
624
|
+
let track = el;
|
|
625
|
+
const w = el.getBoundingClientRect().width;
|
|
626
|
+
for (let a = el.parentElement, i = 0; w < 60 && a && i < 4; a = a.parentElement, i++) if (a.getBoundingClientRect().width >= 3 * w) { track = a; break; }
|
|
627
|
+
const t = track.getBoundingClientRect(), v = window.__demoSliderValue(el);
|
|
628
|
+
let from = v ? (v.now - v.lo) / ((v.hi - v.lo) || 1) : null;
|
|
629
|
+
if (from == null) {
|
|
630
|
+
// custom sliders: start where the knob is drawn, so the value doesn't jump first
|
|
631
|
+
const knob = [...track.querySelectorAll('[role=slider], [class*=thumb], [class*=handle], [class*=knob]')]
|
|
632
|
+
.map((k) => k.getBoundingClientRect()).find((k) => k.width > 0 && k.width < t.width * 0.5);
|
|
633
|
+
from = knob ? (knob.x + knob.width / 2 - t.x) / (t.width || 1) : 0.5;
|
|
584
634
|
}
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
}).catch(() => 0.5);
|
|
591
|
-
const x0 = b.x + b.width * from, x1 = b.x + b.width * Math.max(0, Math.min(1, value.to)), y = b.y + b.height / 2;
|
|
635
|
+
return { box: { x: t.x, y: t.y, width: t.width, height: t.height }, from, range: v && { lo: v.lo, hi: v.hi } };
|
|
636
|
+
}).catch(() => ({ box: p.box, from: 0.5, range: null }));
|
|
637
|
+
if (typeof value.value === 'number' && !s.range) throw new Error('slide value: this slider doesn\'t report its range (no aria-valuemin/max, not an <input type=range>) — use to: 0..1');
|
|
638
|
+
const to = typeof value.value === 'number' ? (value.value - s.range.lo) / ((s.range.hi - s.range.lo) || 1) : value.to;
|
|
639
|
+
const b = s.box, x0 = b.x + b.width * s.from, x1 = b.x + b.width * Math.max(0, Math.min(1, to)), y = b.y + b.height / 2;
|
|
592
640
|
await moveTo(x0, y, value.ms ?? cfg.pace.cursor);
|
|
593
641
|
await sleep(120);
|
|
594
642
|
clicks.push(now());
|
|
@@ -596,6 +644,19 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
596
644
|
await moveTo(x1, y, value.dragMs ?? 900);
|
|
597
645
|
await sleep(120);
|
|
598
646
|
await page.mouse.up();
|
|
647
|
+
if (typeof value.value === 'number') {
|
|
648
|
+
// a native range's thumb doesn't reach the ends of its box, so the drag can land a step off: set it
|
|
649
|
+
const got = await locate(page, value.target).evaluate((el, want) => {
|
|
650
|
+
const v = window.__demoSliderValue(el), r = v?.native;
|
|
651
|
+
if (r && v.now !== want) {
|
|
652
|
+
Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set.call(r, String(want));
|
|
653
|
+
r.dispatchEvent(new Event('input', { bubbles: true })); r.dispatchEvent(new Event('change', { bubbles: true }));
|
|
654
|
+
return Number(r.value);
|
|
655
|
+
}
|
|
656
|
+
return v?.now;
|
|
657
|
+
}, value.value).catch(() => null);
|
|
658
|
+
if (got != null && Math.abs(got - value.value) > 1e-9) log(` ⚠ ${acting}: slide stopped at ${got}, not ${value.value} — the slider snaps to its step`);
|
|
659
|
+
}
|
|
599
660
|
await settle();
|
|
600
661
|
await beat();
|
|
601
662
|
break;
|
|
@@ -608,11 +669,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
608
669
|
let box = null;
|
|
609
670
|
const phraseOnly = v.text && Object.keys(v).every((k) => ['text', 'exact'].includes(k));
|
|
610
671
|
if (phraseOnly) {
|
|
672
|
+
await page.evaluate(() => window.__demoScrollIdle()).catch(() => {}); // judge "in view" where the page ends up
|
|
611
673
|
const found = await page.evaluate((t) => window.__demoFind(t), v.text);
|
|
612
|
-
if (found && !found.inView)
|
|
613
|
-
await page.evaluate((ms) => window.__demoScrollTo(window.__demoFound.startContainer.parentElement, ms), fast ? 0 : 900);
|
|
614
|
-
await sleep(fast ? 50 : 950);
|
|
615
|
-
}
|
|
674
|
+
if (found && !found.inView) await page.evaluate((ms) => window.__demoScrollTo(window.__demoFound.startContainer.parentElement, ms), fast ? 0 : 900);
|
|
616
675
|
if (found) box = await page.evaluate(([ms, color, b]) => window.__demoMark('found', { ms, color, box: b }), [ms, color, opt.box ?? false]);
|
|
617
676
|
}
|
|
618
677
|
if (!box) {
|
|
@@ -659,7 +718,9 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
659
718
|
|
|
660
719
|
// --- setup (not recorded) -----------------------------------------------------------
|
|
661
720
|
// Run a list of actions; on failure, say exactly which one and save a screenshot.
|
|
721
|
+
let acting = '';
|
|
662
722
|
async function runAll(actions, where) {
|
|
723
|
+
acting = where;
|
|
663
724
|
for (const [j, a] of actions.entries()) {
|
|
664
725
|
try {
|
|
665
726
|
// watchdog: an action that hangs (a frozen page, a stalled browser) fails with a clear message
|
|
@@ -759,9 +820,10 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
759
820
|
const n = Math.min(inspectAfter, cfg.steps.length);
|
|
760
821
|
for (const [i, s] of cfg.steps.slice(0, n).entries()) {
|
|
761
822
|
stepHighlight = false;
|
|
762
|
-
// like the recording: marks and
|
|
763
|
-
if (i > 0) await page.evaluate(() => { window.__demoUnmark(); window.__demoUnnote(); }).catch(() => {});
|
|
823
|
+
// like the recording: marks, notes and spotlights end with their step (the last step's stay in the screenshot)
|
|
824
|
+
if (i > 0) await page.evaluate(() => { window.__demoUnmark(true); window.__demoUnnote(true); window.__demoSpotOff(true); }).catch(() => {});
|
|
764
825
|
await runAll(s.actions, `step ${i + 1}`);
|
|
826
|
+
await page.evaluate(() => window.__demoScrollIdle()).catch(() => {});
|
|
765
827
|
}
|
|
766
828
|
await sleep(800);
|
|
767
829
|
const shot = path.join(cfg.work, `inspect-after-${n}.jpg`);
|
|
@@ -804,10 +866,14 @@ export async function record(cfg, { log = console.log, inspectAfter = null } = {
|
|
|
804
866
|
const zoomTarget = s.zoom?.target && !isPoint(s.zoom.target);
|
|
805
867
|
const startBox = zoomTarget ? await boxNow() : null;
|
|
806
868
|
await runAll(s.actions, `step ${i + 1}`);
|
|
869
|
+
// a scroll still running (the page's own smooth scrolling) ends in this step, not in the next one's first shot
|
|
870
|
+
await page.evaluate(() => window.__demoScrollIdle()).catch(() => {});
|
|
807
871
|
const end = now(); // before clearing: a step that is held for its narration keeps its marks
|
|
808
|
-
|
|
809
|
-
//
|
|
810
|
-
|
|
872
|
+
// docs clips start where the step starts, and a hidden step is cut: clear at once, so no fading
|
|
873
|
+
// callout leaks into the next shot
|
|
874
|
+
const instant = cfg.clips.length > 0 || !!s.hidden;
|
|
875
|
+
await spotOff(instant);
|
|
876
|
+
await page.evaluate((i) => { window.__demoUnmark(i); window.__demoUnnote(i); }, instant).catch(() => {});
|
|
811
877
|
let zoomBox = null;
|
|
812
878
|
if (s.zoom?.target) {
|
|
813
879
|
if (isPoint(s.zoom.target)) {
|