rvlib-shipkit 2.3.0 → 2.4.0

package/src/trailer/sound/run.ts

@@ -0,0 +1,189 @@
+// THE one way a trailer makes a sound: a command that writes `{out}`, cached on its words and the content of each file
+// it reads. A voice, an sfx, a music bed and a MIDI file are presets of it (presets.ts). The cache is
+// `.shipkit/cache/sound/<key>.wav` with `<key>.json` (the command, its measures) and `<key>.png` (its waveform), out of
+// the trailer's cache so a trailer prune never deletes a paid line. One sound is made at a time, under nice, never
+// while the machine's load is over 16
+import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs'
+import { loadavg } from 'node:os'
+import { join, relative, resolve } from 'node:path'
+import { ffmpeg } from '../ffmpeg.ts'
+import type { SoundRun } from '../grammar.ts'
+import { measure, measureLine, SOUND_RATE, waveformPng } from './audio.ts'
+
+/** the folder every made sound of the repo is kept in */
+export const soundCache = (root: string): string => join(root, '.shipkit', 'cache', 'sound')
+
+
+export interface RunPlace {
+    root: string
+    /** the .trailer's folder, where a `reads` file is looked for first */
+    dir: string
+    /** a music cue's length: `{seconds}` is it rounded up to 10 s */
+    seconds?: number
+}
+
+/** a music cue's seconds as the command reads them: rounded up to 10, so a small edit makes nothing again */
+export const runSeconds = (s: number | undefined): number => Math.max(10, Math.ceil((s ?? 0) / 10) * 10)
+
+const q = (s: string): string => `'${s.replaceAll("'", "'\\''")}'`
+
+const escape = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
+/** the command with `{seconds}` filled, each read file found (beside the .trailer first, then from the repo root) and
+ *  named in the command by its repo relative path, as a film's `@<file>` is: the command, run at the root, reads the
+ *  file the key hashed. Then the key and the cache file */
+export function runSpec(r: SoundRun, p: RunPlace): { cmd: string; reads: { path: string; abs: string }[]; key: string; file: string } {
+    let cmd = r.cmd.replaceAll('{seconds}', String(runSeconds(p.seconds)))
+    if (!cmd.includes('{out}')) throw new Error(`a sound run writes {out}: ${r.cmd}`)
+    const reads = (r.reads ?? []).map((given) => {
+        const abs = [resolve(p.dir, given), resolve(p.root, given)].find((f) => existsSync(f))
+        if (!abs) throw new Error(`sound run reads ${given}: no such file beside the .trailer or from the repo root`)
+        const rel = relative(p.root, abs)
+        const path = rel.startsWith('..') ? abs : rel
+        if (path !== given) cmd = cmd.replace(new RegExp(`(^|[\\s'"=])${escape(given)}(?=$|[\\s'"])`, 'g'), (_, before: string) => `${before}${path}`)
+        return { path, abs }
+    })
+    const h = new Bun.CryptoHasher('sha1').update(cmd)
+    for (const x of reads) h.update(`\0${x.path}\0`).update(readFileSync(x.abs))
+    const key = h.digest('hex').slice(0, 16)
+    return { cmd, reads, key, file: join(soundCache(p.root), `${key}.wav`) }
+}
+
+let queue: Promise<unknown> = Promise.resolve()
+
+/** the sound's file, made when the cache misses: the command run in the repo root, its output kept as a 48 kHz wav */
+export function soundRun(r: SoundRun, p: RunPlace, say: (line: string) => void): Promise<{ file: string; made: boolean }> {
+    const spec = runSpec(r, p)
+    if (existsSync(spec.file)) return Promise.resolve({ file: spec.file, made: false })
+    const job = queue.then(() => make(spec, p.root, say))
+    queue = job.catch(() => undefined)
+    return job
+}
+
+/** the process holding a lock file: alive, or the lock is stale */
+function holderAlive(lock: string): boolean {
+    let pid = 0
+    try {
+        pid = Number(readFileSync(lock, 'utf8').trim())
+    } catch {
+        return true // written this instant: its pid is not in it yet
+    }
+    if (!Number.isInteger(pid) || pid <= 0) return false
+    try {
+        process.kill(pid, 0)
+        return true
+    } catch (e) {
+        return (e as NodeJS.ErrnoException).code === 'EPERM'
+    }
+}
+
+/** `<key>.lock` beside the sound, across processes: taken with O_EXCL, its pid inside; a lock of a dead process is
+ *  taken over. Null when the sound was made while this one waited */
+async function lockSound(spec: ReturnType<typeof runSpec>, say: (line: string) => void): Promise<(() => void) | null> {
+    const lock = `${spec.file.slice(0, -4)}.lock`
+    for (let said = false; ; ) {
+        if (existsSync(spec.file)) return null
+        try {
+            const fd = openSync(lock, 'wx')
+            writeSync(fd, String(process.pid))
+            closeSync(fd)
+            if (existsSync(spec.file)) {
+                rmSync(lock, { force: true })
+                return null
+            }
+            return () => rmSync(lock, { force: true })
+        } catch (e) {
+            if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e
+        }
+        if (!holderAlive(lock)) {
+            rmSync(lock, { force: true })
+            continue
+        }
+        if (!said) say(`sound: ${spec.key} is being made by another process: waiting for it`)
+        said = true
+        await Bun.sleep(200)
+    }
+}
+
+async function make(spec: ReturnType<typeof runSpec>, root: string, say: (line: string) => void): Promise<{ file: string; made: boolean }> {
+    mkdirSync(soundCache(root), { recursive: true })
+    const unlock = await lockSound(spec, say)
+    if (!unlock) return { file: spec.file, made: false }
+    try {
+        return await makeLocked(spec, root, say)
+    } finally {
+        unlock()
+    }
+}
+
+/** a command that has not answered in this long is killed, loud: a hung box never stalls a build */
+const runTimeoutS = (): number => Number(process.env.SK_SOUND_TIMEOUT_S ?? 30 * 60)
+/** a `.make-*` folder older than this was left by a killed process */
+const STALE_MAKE_MS = 60 * 60 * 1000
+
+/** a process group ended: TERM, then KILL 5 s later */
+export function killGroup(pid: number): void {
+    const kill = (sig: NodeJS.Signals): void => {
+        try {
+            process.kill(-pid, sig)
+        } catch {
+            // already gone
+        }
+    }
+    kill('SIGTERM')
+    setTimeout(() => kill('SIGKILL'), 5000).unref()
+}
+
+const prepared = new Map<string, Promise<void>>()
+
+/** once per process and repo, before the first sound it makes: the folders killed makes left swept, then a wait while
+ *  the machine's load is over 16 */
+async function prepare(root: string, say: (line: string) => void): Promise<void> {
+    const cache = soundCache(root)
+    for (const e of readdirSync(cache)) {
+        const f = join(cache, e)
+        if (e.startsWith('.make-') && Date.now() - statSync(f).mtimeMs > STALE_MAKE_MS) rmSync(f, { recursive: true, force: true })
+    }
+    for (let waited = 0; loadavg()[0]! > 16; waited += 5) {
+        if (waited === 0) say(`sound: the load is ${loadavg()[0]!.toFixed(1)}, over 16: waiting before the first sound`)
+        if (waited > 600) throw new Error('sound: the load stayed over 16 for 10 minutes')
+        await Bun.sleep(5000)
+    }
+}
+
+async function makeLocked(spec: ReturnType<typeof runSpec>, root: string, say: (line: string) => void): Promise<{ file: string; made: boolean }> {
+    let ready = prepared.get(root)
+    if (!ready) prepared.set(root, (ready = prepare(root, say)))
+    await ready
+    const dir = mkdtempSync(join(soundCache(root), '.make-'))
+    try {
+        const out = join(dir, 'out.wav')
+        say(`sound: making ${spec.key}: ${spec.cmd.length > 160 ? `${spec.cmd.slice(0, 157)}…` : spec.cmd}`)
+        const t0 = performance.now()
+        // its own process group: a timeout kills the command and every child holding its pipes
+        const proc = Bun.spawn(['nice', '-n', '15', 'bash', '-c', spec.cmd.replaceAll('{out}', q(out))], { cwd: root, stdout: 'pipe', stderr: 'pipe', detached: true })
+        let timedOut = false
+        const timer = setTimeout(() => {
+            timedOut = true
+            killGroup(proc.pid)
+        }, runTimeoutS() * 1000)
+        const [text, err, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]).finally(() => clearTimeout(timer))
+        if (timedOut) throw new Error(`sound ${spec.key} did not answer in ${runTimeoutS()} s and was killed: ${spec.cmd}`)
+        for (const l of text.split('\n')) if (/^(ok|ElevenLabs|WARN|designing)/.test(l)) say(`  ${l}`)
+        if (code !== 0 || !existsSync(out)) {
+            const tail = `${err}\n${text}`.trim().split('\n').slice(-4).join('\n  ')
+            throw new Error(`sound ${spec.key} ${code !== 0 ? `exited ${code}` : 'wrote no {out}'}: ${spec.cmd}\n  ${tail}`)
+        }
+        const wav = join(dir, 'kept.wav')
+        await ffmpeg(['-y', '-v', 'error', '-i', out, '-ac', '2', '-ar', String(SOUND_RATE), '-c:a', 'pcm_s16le', wav])
+        const m = await measure(wav)
+        if (existsSync(`${out}.png`)) renameSync(`${out}.png`, `${spec.file.slice(0, -4)}.png`)
+        else await waveformPng(wav, `${spec.file.slice(0, -4)}.png`)
+        writeFileSync(`${spec.file.slice(0, -4)}.json`, `${JSON.stringify({ cmd: spec.cmd, reads: spec.reads.map((x) => x.path), made: new Date().toISOString(), ms: Math.round(performance.now() - t0), measures: m }, null, 1)}\n`)
+        renameSync(wav, spec.file)
+        say(`sound: ${relative(root, spec.file)} in ${((performance.now() - t0) / 1000).toFixed(1)} s: ${measureLine(m)}`)
+        return { file: spec.file, made: true }
+    } finally {
+        rmSync(dir, { recursive: true, force: true })
+    }
+}