reelson 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/CHANGELOG.md +99 -0
  2. package/LICENSE +21 -0
  3. package/NOTICE.md +15 -0
  4. package/README.md +180 -0
  5. package/bin/reelson.js +649 -0
  6. package/bin/reelson.ts +676 -0
  7. package/bin/run.js +5 -0
  8. package/demo.config.example.json +51 -0
  9. package/docs/demo.webp +0 -0
  10. package/docs/prompting.md +72 -0
  11. package/docs/style-guide.md +117 -0
  12. package/examples/demo.config.json +18 -0
  13. package/examples/todo-add-item/scenario.ts +45 -0
  14. package/examples/todo-add-item/video.json +35 -0
  15. package/music/README.md +7 -0
  16. package/package.json +68 -0
  17. package/skills/reelson-compose/SKILL.md +226 -0
  18. package/skills/reelson-compose/schemas/video.schema.json +172 -0
  19. package/skills/reelson-compose/scripts/build.js +424 -0
  20. package/skills/reelson-compose/scripts/build.ts +532 -0
  21. package/skills/reelson-compose/scripts/captions.js +24 -0
  22. package/skills/reelson-compose/scripts/captions.ts +40 -0
  23. package/skills/reelson-compose/scripts/check.js +80 -0
  24. package/skills/reelson-compose/scripts/check.ts +93 -0
  25. package/skills/reelson-compose/scripts/composition.js +163 -0
  26. package/skills/reelson-compose/scripts/composition.ts +241 -0
  27. package/skills/reelson-compose/scripts/hyperframes.js +138 -0
  28. package/skills/reelson-compose/scripts/hyperframes.ts +157 -0
  29. package/skills/reelson-compose/scripts/portrait.js +150 -0
  30. package/skills/reelson-compose/scripts/portrait.ts +218 -0
  31. package/skills/reelson-compose/scripts/project.js +238 -0
  32. package/skills/reelson-compose/scripts/project.ts +324 -0
  33. package/skills/reelson-compose/scripts/studio.js +330 -0
  34. package/skills/reelson-compose/scripts/studio.ts +394 -0
  35. package/skills/reelson-compose/scripts/timeline.js +271 -0
  36. package/skills/reelson-compose/scripts/timeline.ts +514 -0
  37. package/skills/reelson-compose/scripts/tts.js +303 -0
  38. package/skills/reelson-compose/scripts/tts.ts +371 -0
  39. package/skills/reelson-compose/scripts/verify.js +190 -0
  40. package/skills/reelson-compose/scripts/verify.ts +209 -0
  41. package/skills/reelson-compose/scripts/voice.js +178 -0
  42. package/skills/reelson-compose/scripts/voice.ts +224 -0
  43. package/skills/reelson-compose/scripts/zooms.js +259 -0
  44. package/skills/reelson-compose/scripts/zooms.ts +333 -0
  45. package/skills/reelson-compose/sections/README.md +5 -0
  46. package/skills/reelson-compose/sections/intro/minimal/section.css +27 -0
  47. package/skills/reelson-compose/sections/intro/minimal/section.html +12 -0
  48. package/skills/reelson-compose/sections/intro/minimal/section.js +16 -0
  49. package/skills/reelson-compose/sections/intro/minimal/section.json +9 -0
  50. package/skills/reelson-compose/sections/intro/poster/section.css +27 -0
  51. package/skills/reelson-compose/sections/intro/poster/section.html +14 -0
  52. package/skills/reelson-compose/sections/intro/poster/section.js +28 -0
  53. package/skills/reelson-compose/sections/intro/poster/section.json +9 -0
  54. package/skills/reelson-compose/sections/intro/split/section.css +30 -0
  55. package/skills/reelson-compose/sections/intro/split/section.html +12 -0
  56. package/skills/reelson-compose/sections/intro/split/section.js +19 -0
  57. package/skills/reelson-compose/sections/intro/split/section.json +9 -0
  58. package/skills/reelson-compose/sections/outro/compact/section.css +23 -0
  59. package/skills/reelson-compose/sections/outro/compact/section.html +10 -0
  60. package/skills/reelson-compose/sections/outro/compact/section.js +11 -0
  61. package/skills/reelson-compose/sections/outro/compact/section.json +8 -0
  62. package/skills/reelson-compose/sections/outro/endcard/section.css +25 -0
  63. package/skills/reelson-compose/sections/outro/endcard/section.html +13 -0
  64. package/skills/reelson-compose/sections/outro/endcard/section.js +13 -0
  65. package/skills/reelson-compose/sections/outro/endcard/section.json +8 -0
  66. package/skills/reelson-compose/sections/outro/wordmark/section.css +11 -0
  67. package/skills/reelson-compose/sections/outro/wordmark/section.html +5 -0
  68. package/skills/reelson-compose/sections/outro/wordmark/section.js +12 -0
  69. package/skills/reelson-compose/sections/outro/wordmark/section.json +8 -0
  70. package/skills/reelson-compose/sections/recap/compact/section.css +26 -0
  71. package/skills/reelson-compose/sections/recap/compact/section.html +8 -0
  72. package/skills/reelson-compose/sections/recap/compact/section.js +23 -0
  73. package/skills/reelson-compose/sections/recap/compact/section.json +11 -0
  74. package/skills/reelson-compose/sections/recap/steps/section.css +36 -0
  75. package/skills/reelson-compose/sections/recap/steps/section.html +8 -0
  76. package/skills/reelson-compose/sections/recap/steps/section.js +27 -0
  77. package/skills/reelson-compose/sections/recap/steps/section.json +11 -0
  78. package/skills/reelson-compose/studio/index.html +81 -0
  79. package/skills/reelson-compose/studio/studio.css +142 -0
  80. package/skills/reelson-compose/studio/studio.js +1031 -0
  81. package/skills/reelson-compose/templates/README.md +193 -0
  82. package/skills/reelson-compose/templates/classic/assets/vendor/Inter-OFL.txt +93 -0
  83. package/skills/reelson-compose/templates/classic/assets/vendor/gsap.min.js +11 -0
  84. package/skills/reelson-compose/templates/classic/assets/vendor/inter-latin-ext-wght-normal.woff2 +0 -0
  85. package/skills/reelson-compose/templates/classic/assets/vendor/inter-latin-wght-normal.woff2 +0 -0
  86. package/skills/reelson-compose/templates/classic/stage.html +447 -0
  87. package/skills/reelson-compose/templates/classic/template.json +5 -0
  88. package/skills/reelson-record/SKILL.md +220 -0
  89. package/skills/reelson-record/schemas/demo.config.schema.json +269 -0
  90. package/skills/reelson-record/scripts/capture.js +142 -0
  91. package/skills/reelson-record/scripts/capture.ts +190 -0
  92. package/skills/reelson-record/scripts/config.js +148 -0
  93. package/skills/reelson-record/scripts/config.ts +264 -0
  94. package/skills/reelson-record/scripts/cursor-overlay.js +111 -0
  95. package/skills/reelson-record/scripts/cursor-overlay.ts +122 -0
  96. package/skills/reelson-record/scripts/doctor.js +128 -0
  97. package/skills/reelson-record/scripts/doctor.ts +147 -0
  98. package/skills/reelson-record/scripts/languages.js +29 -0
  99. package/skills/reelson-record/scripts/languages.ts +41 -0
  100. package/skills/reelson-record/scripts/record.js +365 -0
  101. package/skills/reelson-record/scripts/record.ts +451 -0
  102. package/skills/reelson-record/scripts/scenario.js +356 -0
  103. package/skills/reelson-record/scripts/scenario.ts +570 -0
  104. package/skills/reelson-record/scripts/validate.js +143 -0
  105. package/skills/reelson-record/scripts/validate.ts +179 -0
@@ -0,0 +1,303 @@
1
+ /**
2
+ * Text-to-speech providers for the voice-over (demo.config.json `voice.provider`):
3
+ *
4
+ * openai OpenAI's speech API (OPENAI_API_KEY) — or, with `baseURL`, any local server
5
+ * speaking the same API (Kokoro-FastAPI, Speaches, LocalAI, …)
6
+ * elevenlabs ElevenLabs (ELEVENLABS_API_KEY); `voice` is a voice id or a voice's name
7
+ * piper Piper, a local neural voice (`pipx install piper-tts`); the voice model is
8
+ * downloaded once into ~/.cache/reelson/piper
9
+ * command any local program: `command` is its argv, with {text} {out} {voice} {model}
10
+ * {speed} {language} filled in (the text also comes on stdin); it writes {out}
11
+ *
12
+ * Each speaks one line into a file ffmpeg can read; voice.ts trims and caches it.
13
+ */
14
+ import { spawnSync } from 'node:child_process';
15
+ import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
16
+ import { homedir } from 'node:os';
17
+ import { resolve } from 'node:path';
18
+ import { ReelsonError } from "./project.js";
19
+ export const PROVIDERS = ['openai', 'elevenlabs', 'piper', 'command'];
20
+ const OPENAI_URL = 'https://api.openai.com/v1';
21
+ /** The model and voice each provider uses unless demo.config.json / video.json name one. */
22
+ export function providerDefaults(provider, language) {
23
+ switch (provider) {
24
+ case 'openai':
25
+ return { model: 'gpt-4o-mini-tts', voice: 'alloy' };
26
+ case 'elevenlabs':
27
+ // "George", one of ElevenLabs' default voices; the multilingual model speaks 29 languages.
28
+ return { model: 'eleven_multilingual_v2', voice: 'JBFqnCBsd6RMkjVDRZzb' };
29
+ case 'piper':
30
+ return { model: '', voice: PIPER_VOICES[language.split('-')[0].toLowerCase()] ?? '' };
31
+ case 'command':
32
+ return { model: '', voice: '' };
33
+ }
34
+ }
35
+ /** A good Piper voice per language (https://rhasspy.github.io/piper-samples/). */
36
+ const PIPER_VOICES = {
37
+ en: 'en_US-lessac-medium',
38
+ ro: 'ro_RO-mihai-medium',
39
+ de: 'de_DE-thorsten-medium',
40
+ fr: 'fr_FR-siwis-medium',
41
+ es: 'es_ES-davefx-medium',
42
+ it: 'it_IT-paola-medium',
43
+ pl: 'pl_PL-gosia-medium',
44
+ pt: 'pt_BR-faber-medium',
45
+ ru: 'ru_RU-irina-medium',
46
+ uk: 'uk_UA-ukrainian_tts-medium',
47
+ };
48
+ /** Speaks `text` into `out` (any format ffmpeg reads). Throws a ReelsonError on failure. */
49
+ export async function speak(text, s, out) {
50
+ switch (s.provider) {
51
+ case 'openai':
52
+ return openai(text, s, out);
53
+ case 'elevenlabs':
54
+ return elevenlabs(text, s, out);
55
+ case 'piper':
56
+ return piper(text, s, out);
57
+ case 'command':
58
+ return command(text, s, out);
59
+ }
60
+ }
61
+ /** Why this provider cannot speak on this machine (a missing key or program), or null. */
62
+ export function missingSetup(s) {
63
+ switch (s.provider) {
64
+ case 'openai':
65
+ return !process.env.OPENAI_API_KEY && isOpenAI(s)
66
+ ? 'no OPENAI_API_KEY — export it or put it in a .env next to demo.config.json'
67
+ : null;
68
+ case 'elevenlabs':
69
+ return process.env.ELEVENLABS_API_KEY ? null : 'no ELEVENLABS_API_KEY — export it or put it in a .env next to demo.config.json';
70
+ case 'piper':
71
+ if (!s.voice) {
72
+ return `no Piper voice for language "${s.language}" — set voice.voice in demo.config.json (e.g. "en_US-lessac-medium")`;
73
+ }
74
+ return found('piper') ? null : 'piper not found — pipx install piper-tts';
75
+ case 'command':
76
+ if (!s.command?.length) {
77
+ return 'voice.provider "command" needs voice.command (argv with {text} and {out})';
78
+ }
79
+ return found(s.command[0]) ? null : `${s.command[0]} not found (voice.command)`;
80
+ }
81
+ }
82
+ function isOpenAI(s) {
83
+ return (s.baseURL ?? OPENAI_URL).replace(/\/+$/, '') === OPENAI_URL;
84
+ }
85
+ function found(program) {
86
+ const probe = spawnSync(program, ['--help'], { stdio: 'ignore' });
87
+ return !(probe.error && probe.error.code === 'ENOENT');
88
+ }
89
+ async function post(url, headers, body, what, out) {
90
+ let response;
91
+ try {
92
+ response = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json', ...headers }, body: JSON.stringify(body) });
93
+ }
94
+ catch (error) {
95
+ throw new ReelsonError(`voice-over: cannot reach ${new URL(url).origin} (${error.message})`);
96
+ }
97
+ if (!response.ok) {
98
+ throw new ReelsonError(`voice-over: ${new URL(url).host} answered ${response.status} for "${what}": ${(await response.text()).slice(0, 300)}`);
99
+ }
100
+ writeFileSync(out, Buffer.from(await response.arrayBuffer()));
101
+ }
102
+ async function openai(text, s, out) {
103
+ const key = process.env.OPENAI_API_KEY;
104
+ await post(`${(s.baseURL ?? OPENAI_URL).replace(/\/+$/, '')}/audio/speech`, key ? { Authorization: `Bearer ${key}` } : {}, {
105
+ model: s.model,
106
+ voice: s.voice,
107
+ input: text,
108
+ response_format: 'mp3',
109
+ ...(s.instructions ? { instructions: s.instructions } : {}),
110
+ ...(s.speed !== undefined ? { speed: s.speed } : {}),
111
+ ...s.options,
112
+ }, text, out);
113
+ }
114
+ /** ElevenLabs voice ids are 20 letters and digits; anything else is a voice's name. */
115
+ const ELEVEN_ID = /^[A-Za-z0-9]{20}$/;
116
+ const ELEVEN_URL = 'https://api.elevenlabs.io';
117
+ const elevenVoices = new Map();
118
+ async function elevenVoiceId(voice, key) {
119
+ if (ELEVEN_ID.test(voice)) {
120
+ return voice;
121
+ }
122
+ if (!elevenVoices.size) {
123
+ const response = await fetch(`${ELEVEN_URL}/v1/voices`, { headers: { 'xi-api-key': key } });
124
+ if (!response.ok) {
125
+ const why = (await response.text()).slice(0, 300);
126
+ throw new ReelsonError(response.status === 401 && why.includes('voices_read')
127
+ ? `voice-over: finding the ElevenLabs voice "${voice}" by name needs the key's Voices (read) permission — give it that, or use the voice id (ElevenLabs → Voices → ⋯ → Copy voice ID)`
128
+ : `voice-over: ElevenLabs answered ${response.status} listing voices: ${why}`);
129
+ }
130
+ const { voices } = (await response.json());
131
+ for (const v of voices) {
132
+ elevenVoices.set(v.name.toLowerCase(), v.voice_id);
133
+ // "George - Warm, Captivating Storyteller" is found as "George" too.
134
+ elevenVoices.set(v.name.split(/\s[-–]\s/)[0].toLowerCase(), v.voice_id);
135
+ }
136
+ }
137
+ const id = elevenVoices.get(voice.toLowerCase());
138
+ if (!id) {
139
+ throw new ReelsonError(`voice-over: no ElevenLabs voice named "${voice}" in your account — use its voice id, or add it in the Voice Library`);
140
+ }
141
+ return id;
142
+ }
143
+ async function elevenlabs(text, s, out) {
144
+ const key = process.env.ELEVENLABS_API_KEY;
145
+ const id = await elevenVoiceId(s.voice, key);
146
+ const { voice_settings: own, ...rest } = (s.options ?? {});
147
+ const voiceSettings = { ...(s.speed !== undefined ? { speed: s.speed } : {}), ...own };
148
+ await post(`${ELEVEN_URL}/v1/text-to-speech/${id}?output_format=mp3_44100_128`, { 'xi-api-key': key }, {
149
+ text,
150
+ model_id: s.model,
151
+ // Only the Flash / Turbo v2.5 models take a language; the others detect it.
152
+ ...(/_v2_5$/.test(s.model) ? { language_code: s.language.split('-')[0] } : {}),
153
+ ...(Object.keys(voiceSettings).length ? { voice_settings: voiceSettings } : {}),
154
+ ...rest,
155
+ }, text, out);
156
+ }
157
+ /** Where a Piper voice lives on Hugging Face (rhasspy/piper-voices), e.g. ro/ro_RO/mihai/medium/. */
158
+ export function piperVoiceURL(voice) {
159
+ const match = /^(([a-z]{2,3})_[A-Z]{2})-(.+)-(x_low|low|medium|high)$/.exec(voice);
160
+ if (!match) {
161
+ throw new ReelsonError(`voice-over: "${voice}" is not a Piper voice name (like "en_US-lessac-medium")`);
162
+ }
163
+ const [, locale, lang, speaker, quality] = match;
164
+ return `https://huggingface.co/rhasspy/piper-voices/resolve/main/${lang}/${locale}/${speaker}/${quality}/${voice}.onnx`;
165
+ }
166
+ export const PIPER_DIR = resolve(homedir(), '.cache/reelson/piper');
167
+ async function piperModel(voice, log) {
168
+ const model = resolve(PIPER_DIR, `${voice}.onnx`);
169
+ if (existsSync(model) && existsSync(`${model}.json`)) {
170
+ return model;
171
+ }
172
+ mkdirSync(PIPER_DIR, { recursive: true });
173
+ log(` voice: downloading the Piper voice ${voice} into ${PIPER_DIR}`);
174
+ const url = piperVoiceURL(voice);
175
+ for (const [from, to] of [[`${url}.json`, `${model}.json`], [url, model]]) {
176
+ const response = await fetch(from);
177
+ if (!response.ok) {
178
+ throw new ReelsonError(`voice-over: cannot download the Piper voice ${voice} (${response.status} ${from})`);
179
+ }
180
+ writeFileSync(to, Buffer.from(await response.arrayBuffer()));
181
+ }
182
+ return model;
183
+ }
184
+ async function piper(text, s, out) {
185
+ const model = await piperModel(s.voice, console.log);
186
+ const flags = Object.entries(s.options ?? {}).flatMap(([k, v]) => [`--${k}`, String(v)]);
187
+ const args = ['-m', model, '-f', out, ...(s.speed ? ['--length-scale', String(1 / s.speed)] : []), ...flags];
188
+ const run = spawnSync('piper', args, { input: text, encoding: 'utf8' });
189
+ if (run.status !== 0 || !existsSync(out)) {
190
+ throw new ReelsonError(`voice-over: piper failed for "${text}": ${(run.stderr || run.error?.message || '').trim().slice(-300)}`);
191
+ }
192
+ }
193
+ /** voice.command's argv with the placeholders filled in. */
194
+ export function commandArgs(s, text, out) {
195
+ const values = {
196
+ text,
197
+ out,
198
+ voice: s.voice,
199
+ model: s.model,
200
+ speed: String(s.speed ?? 1),
201
+ language: s.language,
202
+ };
203
+ return (s.command ?? []).map((arg) => arg.replace(/\{(text|out|voice|model|speed|language)\}/g, (_, name) => values[name]));
204
+ }
205
+ async function command(text, s, out) {
206
+ const [program, ...args] = commandArgs(s, text, out);
207
+ const run = spawnSync(program, args, { input: text, encoding: 'utf8' });
208
+ if (run.status !== 0 || !existsSync(out)) {
209
+ throw new ReelsonError(`voice-over: \`${program}\` did not write the line "${text}" (exit ${run.status}): ${(run.stderr || run.error?.message || '').trim().slice(-300)}`);
210
+ }
211
+ }
212
+ /** OpenAI's built-in voices (marin and cedar sound the most natural with gpt-4o-mini-tts). */
213
+ const OPENAI_VOICES = ['alloy', 'ash', 'ballad', 'cedar', 'coral', 'echo', 'fable', 'marin', 'nova', 'onyx', 'sage', 'shimmer', 'verse'];
214
+ /**
215
+ * The voices a provider offers: for Piper, those for `language` unless `all`; for ElevenLabs,
216
+ * the account's — or with `library`, the shared Voice Library's for `language` (usable over the
217
+ * API on a paid plan).
218
+ */
219
+ export async function listVoices(provider, language, all = false, library = false) {
220
+ switch (provider) {
221
+ case 'openai':
222
+ return OPENAI_VOICES.map((id) => ({
223
+ id,
224
+ name: id,
225
+ about: ['marin', 'cedar'].includes(id) ? 'most natural (gpt-4o-mini-tts)' : ['ballad', 'verse', 'marin', 'cedar'].includes(id) ? 'gpt-4o-mini-tts only' : '',
226
+ }));
227
+ case 'elevenlabs': {
228
+ const key = process.env.ELEVENLABS_API_KEY;
229
+ if (!key) {
230
+ throw new ReelsonError('no ELEVENLABS_API_KEY — export it or put it in a .env next to demo.config.json');
231
+ }
232
+ if (library) {
233
+ return elevenLibrary(key, language);
234
+ }
235
+ const response = await fetch(`${ELEVEN_URL}/v1/voices`, { headers: { 'xi-api-key': key } });
236
+ if (!response.ok) {
237
+ throw new ReelsonError(`ElevenLabs answered ${response.status} listing voices: ${(await response.text()).slice(0, 300)}`);
238
+ }
239
+ const { voices } = (await response.json());
240
+ return voices.map((v) => {
241
+ const labels = v.labels ?? {};
242
+ const languages = [...new Set((v.verified_languages ?? []).map((l) => l.language))];
243
+ const [name, tagline] = v.name.split(/\s[-–]\s/);
244
+ return {
245
+ id: v.voice_id,
246
+ name,
247
+ about: [
248
+ tagline,
249
+ labels.gender,
250
+ labels.age?.replace(/_/g, ' '),
251
+ labels.accent,
252
+ labels.descriptive ?? labels.description,
253
+ labels.use_case?.replace(/_/g, ' '),
254
+ v.category && v.category !== 'premade' ? v.category : '',
255
+ languages.length > 1 ? `${languages.length} languages` : '',
256
+ languages.includes(language.split('-')[0].toLowerCase()) ? `verified in "${language}"` : '',
257
+ ]
258
+ .filter(Boolean)
259
+ .join(', '),
260
+ };
261
+ });
262
+ }
263
+ case 'piper': {
264
+ const response = await fetch('https://huggingface.co/rhasspy/piper-voices/resolve/main/voices.json');
265
+ if (!response.ok) {
266
+ throw new ReelsonError(`cannot fetch Piper's voice list (${response.status})`);
267
+ }
268
+ const voices = (await response.json());
269
+ const family = language.split('-')[0].toLowerCase();
270
+ return Object.entries(voices)
271
+ .filter(([, v]) => all || v.language.family === family)
272
+ .map(([id, v]) => ({
273
+ id,
274
+ name: id,
275
+ about: [`${v.language.name_english} (${v.language.country_english})`, v.quality, v.num_speakers > 1 ? `${v.num_speakers} speakers` : '']
276
+ .filter(Boolean)
277
+ .join(', '),
278
+ }))
279
+ .sort((a, b) => a.id.localeCompare(b.id));
280
+ }
281
+ case 'command':
282
+ throw new ReelsonError('voice.provider "command": its voices are whatever your program offers');
283
+ }
284
+ }
285
+ /** The Voice Library's voices for `language`, most used first. */
286
+ async function elevenLibrary(key, language) {
287
+ const lang = language.split('-')[0].toLowerCase();
288
+ const response = await fetch(`${ELEVEN_URL}/v1/shared-voices?language=${lang}&page_size=50&sort=usage_character_count_1y`, { headers: { 'xi-api-key': key } });
289
+ if (!response.ok) {
290
+ throw new ReelsonError(`ElevenLabs answered ${response.status} listing the Voice Library: ${(await response.text()).slice(0, 300)}`);
291
+ }
292
+ const { voices } = (await response.json());
293
+ return voices.map((v) => {
294
+ const [name, tagline] = v.name.split(/\s[-–]\s/);
295
+ return {
296
+ id: v.voice_id,
297
+ name: name.trim(),
298
+ about: [tagline, v.gender, v.age?.replace(/_/g, ' '), v.accent !== 'standard' ? v.accent : '', v.use_case?.replace(/_/g, ' ')]
299
+ .filter(Boolean)
300
+ .join(', '),
301
+ };
302
+ });
303
+ }
@@ -0,0 +1,371 @@
1
+ /**
2
+ * Text-to-speech providers for the voice-over (demo.config.json `voice.provider`):
3
+ *
4
+ * openai OpenAI's speech API (OPENAI_API_KEY) — or, with `baseURL`, any local server
5
+ * speaking the same API (Kokoro-FastAPI, Speaches, LocalAI, …)
6
+ * elevenlabs ElevenLabs (ELEVENLABS_API_KEY); `voice` is a voice id or a voice's name
7
+ * piper Piper, a local neural voice (`pipx install piper-tts`); the voice model is
8
+ * downloaded once into ~/.cache/reelson/piper
9
+ * command any local program: `command` is its argv, with {text} {out} {voice} {model}
10
+ * {speed} {language} filled in (the text also comes on stdin); it writes {out}
11
+ *
12
+ * Each speaks one line into a file ffmpeg can read; voice.ts trims and caches it.
13
+ */
14
+ import { spawnSync } from 'node:child_process'
15
+ import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
16
+ import { homedir } from 'node:os'
17
+ import { resolve } from 'node:path'
18
+ import { ReelsonError } from './project.ts'
19
+
20
+ export const PROVIDERS = ['openai', 'elevenlabs', 'piper', 'command'] as const
21
+ export type Provider = (typeof PROVIDERS)[number]
22
+
23
+ export interface SpeechSettings {
24
+ provider: Provider
25
+ model: string
26
+ voice: string
27
+ /** How to speak (tone, pace) — OpenAI's gpt-4o-mini-tts; other providers ignore it. */
28
+ instructions: string
29
+ /** Speaking rate, 1 = the provider's normal pace; undefined: not sent. */
30
+ speed?: number
31
+ /** The project's language (Piper's default voice, ElevenLabs' language_code). */
32
+ language: string
33
+ /** openai: the API root (a local OpenAI-compatible server). */
34
+ baseURL?: string
35
+ /** command: argv with placeholders. */
36
+ command?: string[]
37
+ /** Extra request fields (openai, elevenlabs) or --flags (piper), passed through. */
38
+ options?: Record<string, unknown>
39
+ }
40
+
41
+ const OPENAI_URL = 'https://api.openai.com/v1'
42
+
43
+ /** The model and voice each provider uses unless demo.config.json / video.json name one. */
44
+ export function providerDefaults(provider: Provider, language: string): { model: string; voice: string } {
45
+ switch (provider) {
46
+ case 'openai':
47
+ return { model: 'gpt-4o-mini-tts', voice: 'alloy' }
48
+ case 'elevenlabs':
49
+ // "George", one of ElevenLabs' default voices; the multilingual model speaks 29 languages.
50
+ return { model: 'eleven_multilingual_v2', voice: 'JBFqnCBsd6RMkjVDRZzb' }
51
+ case 'piper':
52
+ return { model: '', voice: PIPER_VOICES[language.split('-')[0].toLowerCase()] ?? '' }
53
+ case 'command':
54
+ return { model: '', voice: '' }
55
+ }
56
+ }
57
+
58
+ /** A good Piper voice per language (https://rhasspy.github.io/piper-samples/). */
59
+ const PIPER_VOICES: Record<string, string> = {
60
+ en: 'en_US-lessac-medium',
61
+ ro: 'ro_RO-mihai-medium',
62
+ de: 'de_DE-thorsten-medium',
63
+ fr: 'fr_FR-siwis-medium',
64
+ es: 'es_ES-davefx-medium',
65
+ it: 'it_IT-paola-medium',
66
+ pl: 'pl_PL-gosia-medium',
67
+ pt: 'pt_BR-faber-medium',
68
+ ru: 'ru_RU-irina-medium',
69
+ uk: 'uk_UA-ukrainian_tts-medium',
70
+ }
71
+
72
+ /** Speaks `text` into `out` (any format ffmpeg reads). Throws a ReelsonError on failure. */
73
+ export async function speak(text: string, s: SpeechSettings, out: string): Promise<void> {
74
+ switch (s.provider) {
75
+ case 'openai':
76
+ return openai(text, s, out)
77
+ case 'elevenlabs':
78
+ return elevenlabs(text, s, out)
79
+ case 'piper':
80
+ return piper(text, s, out)
81
+ case 'command':
82
+ return command(text, s, out)
83
+ }
84
+ }
85
+
86
+ /** Why this provider cannot speak on this machine (a missing key or program), or null. */
87
+ export function missingSetup(s: SpeechSettings): string | null {
88
+ switch (s.provider) {
89
+ case 'openai':
90
+ return !process.env.OPENAI_API_KEY && isOpenAI(s)
91
+ ? 'no OPENAI_API_KEY — export it or put it in a .env next to demo.config.json'
92
+ : null
93
+ case 'elevenlabs':
94
+ return process.env.ELEVENLABS_API_KEY ? null : 'no ELEVENLABS_API_KEY — export it or put it in a .env next to demo.config.json'
95
+ case 'piper':
96
+ if (!s.voice) {
97
+ return `no Piper voice for language "${s.language}" — set voice.voice in demo.config.json (e.g. "en_US-lessac-medium")`
98
+ }
99
+ return found('piper') ? null : 'piper not found — pipx install piper-tts'
100
+ case 'command':
101
+ if (!s.command?.length) {
102
+ return 'voice.provider "command" needs voice.command (argv with {text} and {out})'
103
+ }
104
+ return found(s.command[0]) ? null : `${s.command[0]} not found (voice.command)`
105
+ }
106
+ }
107
+
108
+ function isOpenAI(s: SpeechSettings): boolean {
109
+ return (s.baseURL ?? OPENAI_URL).replace(/\/+$/, '') === OPENAI_URL
110
+ }
111
+
112
+ function found(program: string): boolean {
113
+ const probe = spawnSync(program, ['--help'], { stdio: 'ignore' })
114
+ return !(probe.error && (probe.error as NodeJS.ErrnoException).code === 'ENOENT')
115
+ }
116
+
117
+ async function post(url: string, headers: Record<string, string>, body: unknown, what: string, out: string): Promise<void> {
118
+ let response: Response
119
+ try {
120
+ response = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json', ...headers }, body: JSON.stringify(body) })
121
+ } catch (error) {
122
+ throw new ReelsonError(`voice-over: cannot reach ${new URL(url).origin} (${(error as Error).message})`)
123
+ }
124
+ if (!response.ok) {
125
+ throw new ReelsonError(`voice-over: ${new URL(url).host} answered ${response.status} for "${what}": ${(await response.text()).slice(0, 300)}`)
126
+ }
127
+ writeFileSync(out, Buffer.from(await response.arrayBuffer()))
128
+ }
129
+
130
+ async function openai(text: string, s: SpeechSettings, out: string): Promise<void> {
131
+ const key = process.env.OPENAI_API_KEY
132
+ await post(
133
+ `${(s.baseURL ?? OPENAI_URL).replace(/\/+$/, '')}/audio/speech`,
134
+ key ? { Authorization: `Bearer ${key}` } : {},
135
+ {
136
+ model: s.model,
137
+ voice: s.voice,
138
+ input: text,
139
+ response_format: 'mp3',
140
+ ...(s.instructions ? { instructions: s.instructions } : {}),
141
+ ...(s.speed !== undefined ? { speed: s.speed } : {}),
142
+ ...s.options,
143
+ },
144
+ text,
145
+ out,
146
+ )
147
+ }
148
+
149
+ /** ElevenLabs voice ids are 20 letters and digits; anything else is a voice's name. */
150
+ const ELEVEN_ID = /^[A-Za-z0-9]{20}$/
151
+ const ELEVEN_URL = 'https://api.elevenlabs.io'
152
+ const elevenVoices = new Map<string, string>()
153
+
154
+ async function elevenVoiceId(voice: string, key: string): Promise<string> {
155
+ if (ELEVEN_ID.test(voice)) {
156
+ return voice
157
+ }
158
+ if (!elevenVoices.size) {
159
+ const response = await fetch(`${ELEVEN_URL}/v1/voices`, { headers: { 'xi-api-key': key } })
160
+ if (!response.ok) {
161
+ const why = (await response.text()).slice(0, 300)
162
+ throw new ReelsonError(
163
+ response.status === 401 && why.includes('voices_read')
164
+ ? `voice-over: finding the ElevenLabs voice "${voice}" by name needs the key's Voices (read) permission — give it that, or use the voice id (ElevenLabs → Voices → ⋯ → Copy voice ID)`
165
+ : `voice-over: ElevenLabs answered ${response.status} listing voices: ${why}`,
166
+ )
167
+ }
168
+ const { voices } = (await response.json()) as { voices: { voice_id: string; name: string }[] }
169
+ for (const v of voices) {
170
+ elevenVoices.set(v.name.toLowerCase(), v.voice_id)
171
+ // "George - Warm, Captivating Storyteller" is found as "George" too.
172
+ elevenVoices.set(v.name.split(/\s[-–]\s/)[0].toLowerCase(), v.voice_id)
173
+ }
174
+ }
175
+ const id = elevenVoices.get(voice.toLowerCase())
176
+ if (!id) {
177
+ throw new ReelsonError(`voice-over: no ElevenLabs voice named "${voice}" in your account — use its voice id, or add it in the Voice Library`)
178
+ }
179
+ return id
180
+ }
181
+
182
+ async function elevenlabs(text: string, s: SpeechSettings, out: string): Promise<void> {
183
+ const key = process.env.ELEVENLABS_API_KEY as string
184
+ const id = await elevenVoiceId(s.voice, key)
185
+ const { voice_settings: own, ...rest } = (s.options ?? {}) as { voice_settings?: Record<string, unknown> }
186
+ const voiceSettings = { ...(s.speed !== undefined ? { speed: s.speed } : {}), ...own }
187
+ await post(
188
+ `${ELEVEN_URL}/v1/text-to-speech/${id}?output_format=mp3_44100_128`,
189
+ { 'xi-api-key': key },
190
+ {
191
+ text,
192
+ model_id: s.model,
193
+ // Only the Flash / Turbo v2.5 models take a language; the others detect it.
194
+ ...(/_v2_5$/.test(s.model) ? { language_code: s.language.split('-')[0] } : {}),
195
+ ...(Object.keys(voiceSettings).length ? { voice_settings: voiceSettings } : {}),
196
+ ...rest,
197
+ },
198
+ text,
199
+ out,
200
+ )
201
+ }
202
+
203
+ /** Where a Piper voice lives on Hugging Face (rhasspy/piper-voices), e.g. ro/ro_RO/mihai/medium/. */
204
+ export function piperVoiceURL(voice: string): string {
205
+ const match = /^(([a-z]{2,3})_[A-Z]{2})-(.+)-(x_low|low|medium|high)$/.exec(voice)
206
+ if (!match) {
207
+ throw new ReelsonError(`voice-over: "${voice}" is not a Piper voice name (like "en_US-lessac-medium")`)
208
+ }
209
+ const [, locale, lang, speaker, quality] = match
210
+ return `https://huggingface.co/rhasspy/piper-voices/resolve/main/${lang}/${locale}/${speaker}/${quality}/${voice}.onnx`
211
+ }
212
+
213
+ export const PIPER_DIR = resolve(homedir(), '.cache/reelson/piper')
214
+
215
+ async function piperModel(voice: string, log: (l: string) => void): Promise<string> {
216
+ const model = resolve(PIPER_DIR, `${voice}.onnx`)
217
+ if (existsSync(model) && existsSync(`${model}.json`)) {
218
+ return model
219
+ }
220
+ mkdirSync(PIPER_DIR, { recursive: true })
221
+ log(` voice: downloading the Piper voice ${voice} into ${PIPER_DIR}`)
222
+ const url = piperVoiceURL(voice)
223
+ for (const [from, to] of [[`${url}.json`, `${model}.json`], [url, model]]) {
224
+ const response = await fetch(from)
225
+ if (!response.ok) {
226
+ throw new ReelsonError(`voice-over: cannot download the Piper voice ${voice} (${response.status} ${from})`)
227
+ }
228
+ writeFileSync(to, Buffer.from(await response.arrayBuffer()))
229
+ }
230
+ return model
231
+ }
232
+
233
+ async function piper(text: string, s: SpeechSettings, out: string): Promise<void> {
234
+ const model = await piperModel(s.voice, console.log)
235
+ const flags = Object.entries(s.options ?? {}).flatMap(([k, v]) => [`--${k}`, String(v)])
236
+ const args = ['-m', model, '-f', out, ...(s.speed ? ['--length-scale', String(1 / s.speed)] : []), ...flags]
237
+ const run = spawnSync('piper', args, { input: text, encoding: 'utf8' })
238
+ if (run.status !== 0 || !existsSync(out)) {
239
+ throw new ReelsonError(`voice-over: piper failed for "${text}": ${(run.stderr || run.error?.message || '').trim().slice(-300)}`)
240
+ }
241
+ }
242
+
243
+ /** voice.command's argv with the placeholders filled in. */
244
+ export function commandArgs(s: SpeechSettings, text: string, out: string): string[] {
245
+ const values: Record<string, string> = {
246
+ text,
247
+ out,
248
+ voice: s.voice,
249
+ model: s.model,
250
+ speed: String(s.speed ?? 1),
251
+ language: s.language,
252
+ }
253
+ return (s.command ?? []).map((arg) => arg.replace(/\{(text|out|voice|model|speed|language)\}/g, (_, name: string) => values[name]))
254
+ }
255
+
256
+ async function command(text: string, s: SpeechSettings, out: string): Promise<void> {
257
+ const [program, ...args] = commandArgs(s, text, out)
258
+ const run = spawnSync(program, args, { input: text, encoding: 'utf8' })
259
+ if (run.status !== 0 || !existsSync(out)) {
260
+ throw new ReelsonError(
261
+ `voice-over: \`${program}\` did not write the line "${text}" (exit ${run.status}): ${(run.stderr || run.error?.message || '').trim().slice(-300)}`,
262
+ )
263
+ }
264
+ }
265
+
266
+ export interface VoiceInfo {
267
+ /** What goes in voice.voice. */
268
+ id: string
269
+ name: string
270
+ /** Accent, gender, style, languages — whatever the provider tells. */
271
+ about: string
272
+ }
273
+
274
+ /** OpenAI's built-in voices (marin and cedar sound the most natural with gpt-4o-mini-tts). */
275
+ const OPENAI_VOICES = ['alloy', 'ash', 'ballad', 'cedar', 'coral', 'echo', 'fable', 'marin', 'nova', 'onyx', 'sage', 'shimmer', 'verse']
276
+
277
+ /**
278
+ * The voices a provider offers: for Piper, those for `language` unless `all`; for ElevenLabs,
279
+ * the account's — or with `library`, the shared Voice Library's for `language` (usable over the
280
+ * API on a paid plan).
281
+ */
282
+ export async function listVoices(provider: Provider, language: string, all = false, library = false): Promise<VoiceInfo[]> {
283
+ switch (provider) {
284
+ case 'openai':
285
+ return OPENAI_VOICES.map((id) => ({
286
+ id,
287
+ name: id,
288
+ about: ['marin', 'cedar'].includes(id) ? 'most natural (gpt-4o-mini-tts)' : ['ballad', 'verse', 'marin', 'cedar'].includes(id) ? 'gpt-4o-mini-tts only' : '',
289
+ }))
290
+ case 'elevenlabs': {
291
+ const key = process.env.ELEVENLABS_API_KEY
292
+ if (!key) {
293
+ throw new ReelsonError('no ELEVENLABS_API_KEY — export it or put it in a .env next to demo.config.json')
294
+ }
295
+ if (library) {
296
+ return elevenLibrary(key, language)
297
+ }
298
+ const response = await fetch(`${ELEVEN_URL}/v1/voices`, { headers: { 'xi-api-key': key } })
299
+ if (!response.ok) {
300
+ throw new ReelsonError(`ElevenLabs answered ${response.status} listing voices: ${(await response.text()).slice(0, 300)}`)
301
+ }
302
+ const { voices } = (await response.json()) as {
303
+ voices: { voice_id: string; name: string; category?: string; labels?: Record<string, string>; verified_languages?: { language: string }[] }[]
304
+ }
305
+ return voices.map((v) => {
306
+ const labels = v.labels ?? {}
307
+ const languages = [...new Set((v.verified_languages ?? []).map((l) => l.language))]
308
+ const [name, tagline] = v.name.split(/\s[-–]\s/)
309
+ return {
310
+ id: v.voice_id,
311
+ name,
312
+ about: [
313
+ tagline,
314
+ labels.gender,
315
+ labels.age?.replace(/_/g, ' '),
316
+ labels.accent,
317
+ labels.descriptive ?? labels.description,
318
+ labels.use_case?.replace(/_/g, ' '),
319
+ v.category && v.category !== 'premade' ? v.category : '',
320
+ languages.length > 1 ? `${languages.length} languages` : '',
321
+ languages.includes(language.split('-')[0].toLowerCase()) ? `verified in "${language}"` : '',
322
+ ]
323
+ .filter(Boolean)
324
+ .join(', '),
325
+ }
326
+ })
327
+ }
328
+ case 'piper': {
329
+ const response = await fetch('https://huggingface.co/rhasspy/piper-voices/resolve/main/voices.json')
330
+ if (!response.ok) {
331
+ throw new ReelsonError(`cannot fetch Piper's voice list (${response.status})`)
332
+ }
333
+ const voices = (await response.json()) as Record<string, { language: { family: string; name_english: string; country_english: string }; quality: string; num_speakers: number }>
334
+ const family = language.split('-')[0].toLowerCase()
335
+ return Object.entries(voices)
336
+ .filter(([, v]) => all || v.language.family === family)
337
+ .map(([id, v]) => ({
338
+ id,
339
+ name: id,
340
+ about: [`${v.language.name_english} (${v.language.country_english})`, v.quality, v.num_speakers > 1 ? `${v.num_speakers} speakers` : '']
341
+ .filter(Boolean)
342
+ .join(', '),
343
+ }))
344
+ .sort((a, b) => a.id.localeCompare(b.id))
345
+ }
346
+ case 'command':
347
+ throw new ReelsonError('voice.provider "command": its voices are whatever your program offers')
348
+ }
349
+ }
350
+
351
+ /** The Voice Library's voices for `language`, most used first. */
352
+ async function elevenLibrary(key: string, language: string): Promise<VoiceInfo[]> {
353
+ const lang = language.split('-')[0].toLowerCase()
354
+ const response = await fetch(`${ELEVEN_URL}/v1/shared-voices?language=${lang}&page_size=50&sort=usage_character_count_1y`, { headers: { 'xi-api-key': key } })
355
+ if (!response.ok) {
356
+ throw new ReelsonError(`ElevenLabs answered ${response.status} listing the Voice Library: ${(await response.text()).slice(0, 300)}`)
357
+ }
358
+ const { voices } = (await response.json()) as {
359
+ voices: { voice_id: string; name: string; gender?: string; age?: string; accent?: string; use_case?: string; description?: string }[]
360
+ }
361
+ return voices.map((v) => {
362
+ const [name, tagline] = v.name.split(/\s[-–]\s/)
363
+ return {
364
+ id: v.voice_id,
365
+ name: name.trim(),
366
+ about: [tagline, v.gender, v.age?.replace(/_/g, ' '), v.accent !== 'standard' ? v.accent : '', v.use_case?.replace(/_/g, ' ')]
367
+ .filter(Boolean)
368
+ .join(', '),
369
+ }
370
+ })
371
+ }