rvlib-shipkit 2.2.0 → 2.4.0

package/src/trailer/ffmpeg.ts

@@ -1,6 +1,7 @@
 // the one way the trailer spawns ffmpeg and ffprobe: a run that fails loud with ffmpeg's own words, a probe, the nine
 // frame sheet. The binaries resolve once, with the homebrew path for processes started without it
 import { spawn } from 'node:child_process'
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
 import { basename } from 'node:path'
 
 const FFMPEG = Bun.which('ffmpeg') ?? '/opt/homebrew/bin/ffmpeg'
@@ -44,6 +45,27 @@ export function hasAudio(file: string): boolean {
     return r.stdout.toString().trim() !== ''
 }
 
+/** a sound's integrated loudness (LUFS) and true peak (dBTP), as ffmpeg's ebur128 measures them */
+export async function loudness(file: string): Promise<{ lufs: number; truePeak: number }> {
+    const log = await ffmpeg(['-nostats', '-loglevel', 'info', '-i', file, '-map', '0:a', '-af', 'ebur128=peak=true', '-f', 'null', '-'])
+    const summary = log.split('Summary:').at(-1) ?? ''
+    // silence measures -inf: floored, so the value stays a number in json
+    const num = (v: string | undefined): number => (v === '-inf' ? -120 : Number(v ?? Number.NaN))
+    const lufs = num(/I:\s+(-?[\d.]+|-inf) LUFS/.exec(summary)?.[1])
+    const truePeak = num(/Peak:\s+(-?[\d.]+|-inf) dBFS/.exec(summary)?.[1])
+    if (Number.isNaN(lufs) || Number.isNaN(truePeak)) throw new Error(`ebur128 ${basename(file)}: no loudness summary in ffmpeg's log`)
+    return { lufs, truePeak }
+}
+
+/** loudness() of a file that never changes under its name (a cached render), kept beside it */
+export async function loudnessKept(file: string): Promise<{ lufs: number; truePeak: number }> {
+    const json = `${file}.loudness.json`
+    if (existsSync(json)) return JSON.parse(readFileSync(json, 'utf8')) as { lufs: number; truePeak: number }
+    const m = await loudness(file)
+    writeFileSync(json, JSON.stringify(m))
+    return m
+}
+
 /** nine frames of a clip evenly spaced, three by three on one png: how a shot is judged */
 export async function sheetOf(clip: string, seconds: number, png: string): Promise<void> {
     await ffmpeg(['-i', clip, '-vf', `fps=${(9 / seconds).toFixed(4)},scale=480:-2,tile=3x3`, '-frames:v', '1', png])

package/src/trailer/fonts.ts

@@ -0,0 +1,55 @@
+// a title's face: a font file of the repo, or a Google Fonts family fetched once per weight into the machine's cache
+// (every repo shares it). Each face is drawn under a name of its own, and one that does not register fails loud: a
+// title never falls back to another face in silence
+import { GlobalFonts } from '@napi-rs/canvas'
+import { existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+/** a `font` word that names a file; any other names a Google Fonts family */
+export const isFontFile = (w: string): boolean => /\.(ttf|otf|woff2?)$/i.test(w)
+
+export const fontCache = (): string => process.env.SHIPKIT_FONT_CACHE || join(homedir(), '.cache', 'shipkit', 'fonts')
+
+const slug = (family: string): string => family.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
+
+/** where a Google Fonts family at a weight is kept */
+export const googleFile = (family: string, weight: number): string => join(fontCache(), `${slug(family)}-${weight}.ttf`)
+
+/** a Google Fonts family at a weight, fetched when the cache lacks it: the css answer names a truetype file to a plain
+ *  agent */
+export async function googleFont(family: string, weight: number): Promise<string> {
+    const file = googleFile(family, weight)
+    if (existsSync(file)) return file
+    const url = `https://fonts.googleapis.com/css2?family=${encodeURIComponent(family).replaceAll('%20', '+')}:wght@${weight}`
+    const css = await fetch(url)
+    if (!css.ok) throw new Error(`font "${family}" weight ${weight}: Google Fonts has no such family at that weight (HTTP ${css.status}); check its name and weights on fonts.google.com, or name a font file of the repo (.ttf .otf .woff .woff2)`)
+    const src = /url\((https:[^)]+)\)/.exec(await css.text())?.[1]
+    if (!src) throw new Error(`font "${family}" weight ${weight}: Google Fonts answered with no font file (${url})`)
+    const got = await fetch(src)
+    if (!got.ok) throw new Error(`font "${family}" weight ${weight}: its file did not download (HTTP ${got.status}, ${src})`)
+    mkdirSync(fontCache(), { recursive: true })
+    const part = `${file}.${process.pid}.part`
+    writeFileSync(part, Buffer.from(await got.arrayBuffer()))
+    renameSync(part, file)
+    return file
+}
+
+const registered = new Map<string, string>()
+
+/** the name a face file is drawn under, its own: two files never share a family, and a file that does not load fails */
+export function registerFont(abs: string): string {
+    const had = registered.get(abs)
+    if (had) return had
+    if (!existsSync(abs)) throw new Error(`the font file ${abs} is missing`)
+    const name = `F${new Bun.CryptoHasher('sha1').update(abs).digest('hex').slice(0, 12)}`
+    GlobalFonts.registerFromPath(abs, name)
+    if (!GlobalFonts.has(name)) throw new Error(`the font file ${abs} does not load: a .ttf, .otf, .woff or .woff2 face`)
+    registered.set(abs, name)
+    return name
+}
+
+/** a family the machine draws, never a stand in: a name it lacks fails loud */
+export function hasFamily(family: string): boolean {
+    return family.split(',').some((f) => GlobalFonts.has(f.trim().replace(/^["']|["']$/g, '')))
+}

package/src/trailer/footage.ts

@@ -1,52 +1,96 @@
 // the takes on disk: each film line that writes its own take (a project CLI film row, a `film run`, a `film url`) filmed once into
 // `.shipkit/cache/trailer/footage/<key>.<ext>`, a stamp beside it naming the world it was filmed on. A take of an older
 // world is reported, and filmed again only on --refilm. Written aside then renamed, two at once
-import { spawnSync } from 'node:child_process'
-import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
-import { join } from 'node:path'
+import { copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { isAbsolute, join } from 'node:path'
+import type { TrailerFilmLine, TrailerUnit } from './adapter.ts'
 import { duration } from './ffmpeg.ts'
-import type { Trailer } from './grammar.ts'
-import { cliArgv, isTake, runCommand, type Stamp, type Take, type TakeFilm, takeIsStill, takeName, takeSeconds } from './plan.ts'
+import type { Shot, Trailer } from './grammar.ts'
+import { cliArgv, isTake, recipeText, recipeWords, runCommand, type Stamp, specOf, type Take, type TakeFilm, type TakeSpec, takeIsStill, takeName, takeSeconds } from './plan.ts'
+import { statesOf } from './states.ts'
 
-export const trailerCache = (root: string, part: 'footage' | 'pieces' | 'titles' | 'sound'): string => join(root, '.shipkit', 'cache', 'trailer', part)
+export const trailerCache = (root: string, part: 'footage' | 'pieces' | 'titles' | 'sound' | 'score'): string => join(root, '.shipkit', 'cache', 'trailer', part)
 
-/** every shot's take, with its stamp when it was filmed */
-export function takesOf(t: Trailer, root: string): Take[] {
+/** a recipe file a `@<file>` word names: its repo relative path, its text */
+export interface Recipe {
+    path: string
+    text: string
+}
+
+/** what a take reads past its line: the recipe files, the repo's trailer unit */
+export interface TakeContext {
+    recipes: ReadonlyMap<string, Recipe>
+    unit: TrailerUnit | null
+}
+
+/** a film line as the unit sees it */
+export function filmLineOf(shot: Shot, f: Extract<TakeFilm, { kind: 'cli' }>, c: Pick<TakeContext, 'recipes'>): TrailerFilmLine {
+    const files = Object.fromEntries(recipeWords(f.argv).flatMap((w) => (c.recipes.has(w) ? [[w, c.recipes.get(w)!.text]] : [])))
+    return { line: f.line, shot: shot.name, argv: f.argv, output: f.output, files }
+}
+
+/** a take's spec and the words it runs: its row's defaults, the unit's `take` over them, each `@<file>` resolved */
+export function specFor(shot: Shot, f: TakeFilm, c: TakeContext): { spec: TakeSpec; argv?: string[] } {
+    // a run keys the files it says it reads, by their content
+    if (f.kind === 'run') return { spec: specOf(f, Object.fromEntries((f.reads ?? []).map((p) => [`reads:${p}`, c.recipes.get(`reads:${p}`)?.text ?? '']))) }
+    if (f.kind !== 'cli') return { spec: specOf(f) }
+    const line = filmLineOf(shot, f, c)
+    // a declared file word keys its content, and reaches the row as its repo relative path
+    const declared = Object.fromEntries((f.files ?? []).flatMap((i) => (c.recipes.has(`file:${f.argv[i]}`) ? [[`file:${f.argv[i]}`, c.recipes.get(`file:${f.argv[i]}`)!.text]] : [])))
+    const base = specOf(f, { ...Object.fromEntries(Object.entries(line.files).map(([w, text]) => [w, recipeText(text)])), ...declared })
+    const got = c.unit?.take?.({ ...line, version: base.version, length: base.length })
+    const spec: TakeSpec = got
+        ? { ...base, version: got.version ?? base.version, length: got.length ?? base.length, world: got.world ?? null, recipe: got.recipe === undefined ? base.recipe : { recipe: got.recipe, files: (base.recipe as { files?: unknown }).files ?? null } }
+        : base
+    const files = new Set(f.files ?? [])
+    return { spec, argv: f.argv.map((w, i) => (files.has(i) ? (c.recipes.get(`file:${w}`)?.path ?? w) : c.recipes.has(w) ? `@${c.recipes.get(w)!.path}` : w)) }
+}
+
+/** every shot's and footage block's take, with its stamp when it was filmed */
+export function takesOf(t: Trailer, root: string, c: TakeContext = { recipes: new Map(), unit: null }): Take[] {
     const dir = trailerCache(root, 'footage')
-    return t.shots.flatMap((shot) => {
+    return [...t.shots, ...t.footage].flatMap((shot) => {
         const film = shot.film
-        if (!isTake(film)) return []
-        const file = join(dir, takeName(shot, film, t))
+        if (!isTake(film) || shot.broken) return []
+        const { spec, argv } = specFor(shot, film, c)
+        const file = join(dir, takeName(shot, film, t, spec))
         const stampFile = `${file}.json`
         const stamp = existsSync(file) && existsSync(stampFile) ? (JSON.parse(readFileSync(stampFile, 'utf8')) as Stamp) : null
-        return [{ shot, film, file, stamp }]
+        return [{ shot, film, file, stamp, spec, ...(argv ? { argv } : {}) }]
     })
 }
 
-const git = (root: string, ...args: string[]): string | null => {
-    const r = spawnSync('git', args, { cwd: root, encoding: 'utf8' })
-    return r.status === 0 ? r.stdout : null
+/** a git run's output, null when it fails; no size limit: a big repo lists many files */
+const git = (root: string, args: string[], env?: Record<string, string>): string | null => {
+    const r = Bun.spawnSync(['git', ...args], { cwd: root, stdout: 'pipe', stderr: 'pipe', ...(env ? { env: { ...process.env, ...env } } : {}) })
+    return r.exitCode === 0 ? r.stdout.toString() : null
 }
 
-/** the world the takes show: the repo's committed tree and what the working tree changes, trailer folders and the
- *  shipkit cache left out; a repo with no git: `no-git` */
-export function worldStamp(root: string): string {
-    const tree = git(root, 'rev-parse', 'HEAD^{tree}')
-    if (tree === null) return 'no-git'
-    const h = new Bun.CryptoHasher('sha1').update(tree)
-    const dirty = (git(root, 'status', '--porcelain=v1', '-z', '--untracked-files=all', '--', '.', ':(exclude)trailer', ':(exclude).shipkit') ?? '')
-        .split('\0')
-        .filter(Boolean)
-        .map((l) => l.slice(3))
-        .sort()
-    for (const f of dirty) {
-        h.update(f)
-        const abs = join(root, f)
-        try {
-            h.update(readFileSync(abs))
-        } catch {}
+/**
+ * the world the takes show: every file of the repo as it stands, by its content (`trailer/` and `.shipkit/` left out),
+ * or only the `paths` the trailer unit's `stamp` names; a repo with no git: `no-git`. Content addressed: the working
+ * tree is staged into a throwaway copy of the index and the staged entries hashed, so the same content dirty or
+ * committed is one world, and an edit of the cut alone moves nothing
+ */
+export function worldStamp(root: string, paths?: readonly string[]): string {
+    const real = git(root, ['rev-parse', '--git-path', 'index'])
+    if (real === null) return 'no-git'
+    const spec = paths ? ['--', ...paths] : ['--', '.', ':(exclude)trailer', ':(exclude).shipkit']
+    if (paths) for (const p of paths) if (!existsSync(join(root, p))) throw new Error(`the trailer unit's stamp names a path the repo does not hold: ${p}`)
+    const index = join(tmpdir(), `sk-world-${process.pid}-${Math.random().toString(36).slice(2)}`)
+    try {
+        // a copy of the real index keeps git's stat cache: only the files that changed are read again
+        const from = isAbsolute(real.trim()) ? real.trim() : join(root, real.trim())
+        if (existsSync(from)) copyFileSync(from, index)
+        const env = { GIT_INDEX_FILE: index }
+        if (git(root, ['add', '-A', ...spec], env) === null) throw new Error('git add into the world\'s throwaway index failed')
+        const staged = git(root, ['ls-files', '-s', ...spec], env)
+        if (staged === null) throw new Error('git ls-files of the world\'s throwaway index failed')
+        return new Bun.CryptoHasher('sha1').update(staged).digest('hex').slice(0, 12)
+    } finally {
+        rmSync(index, { force: true })
     }
-    return h.digest('hex').slice(0, 12)
 }
 
 export interface FilmContext {
@@ -55,6 +99,8 @@ export interface FilmContext {
     world: string
     /** the project CLI's argv, null when the repo has none */
     cli: readonly string[] | null
+    /** a clip take that drops its picture is filmed again once, then refused (the unit's `holes`) */
+    holes?: boolean
     say: (line: string) => void
 }
 
@@ -64,34 +110,57 @@ export async function filmTake(tk: Take, c: FilmContext): Promise<Take> {
     const part = tk.file.replace(/(\.\w+)$/, '.part$1')
     rmSync(part, { force: true })
     const t0 = performance.now()
-    const seconds = takeSeconds(tk.shot, tk.film)
+    const seconds = takeSeconds(tk.shot, tk.film, tk.spec.length)
     const f = tk.film
-    const said = f.kind === 'url' ? await urlTake(tk, f, part, seconds, c) : await commandTake(tk, f, part, seconds, c)
-    renameSync(part, tk.file)
     const still = takeIsStill(f)
-    const stamp: Stamp = { world: c.world, filmedAt: new Date().toISOString(), seconds: still ? 0 : duration(tk.file), note: said.split('\n')[0]?.slice(0, 200) ?? '' }
+    let said = ''
+    for (let attempt = 1; ; attempt++) {
+        said = f.kind === 'url' || f.kind === 'page' ? await urlTake(tk, f, part, seconds, c) : await commandTake(tk, f, part, seconds, c)
+        if (still || !c.holes) break
+        const holes = await (await import('./holes.ts')).takeHoles(part)
+        if (holes.length === 0) break
+        const where = `the picture drops out on frames ${holes[0]} to ${holes.at(-1)}`
+        if (attempt >= 2) {
+            rmSync(part, { force: true })
+            throw new Error(`shot ${tk.shot.name} (line ${f.line}): ${where}, twice`)
+        }
+        c.say(`  ${tk.shot.name}: ${where}, filmed again`)
+    }
+    renameSync(part, tk.file)
+    const states = statesOf(said)
+    const first = (said.split('\n')[0] ?? '').replace(/\s·\sstates \S+/, '').replace(part, '').trim()
+    const stamp: Stamp = {
+        world: tk.spec.world ?? c.world,
+        filmedAt: new Date().toISOString(),
+        seconds: still ? 0 : duration(tk.file),
+        note: first.slice(0, 200),
+        ...(states ? { states } : {}),
+        ...(tk.spec.world ? { fixed: true as const } : {}),
+    }
     writeFileSync(`${tk.file}.json`, JSON.stringify(stamp, null, 2))
     c.say(`  filmed ${tk.shot.name}: ${still ? 'a still' : `${stamp.seconds.toFixed(2)} s`} in ${((performance.now() - t0) / 1000).toFixed(1)} s`)
     return { ...tk, stamp }
 }
 
 /** a take a command writes at `part` (a `film run`, a project CLI row); what it said, or a throw with its last lines */
-async function commandTake(tk: Take, f: Exclude<TakeFilm, { kind: 'url' }>, part: string, seconds: number | null, c: FilmContext): Promise<string> {
+async function commandTake(tk: Take, f: Exclude<TakeFilm, { kind: 'url' | 'page' }>, part: string, seconds: number | null, c: FilmContext): Promise<string> {
     if (f.kind === 'cli' && c.cli === null) throw new Error(`shot ${tk.shot.name} (line ${f.line}): film ${f.argv.join(' ')} needs the project CLI (the cli script of package.json)`)
-    const argv = f.kind === 'run' ? ['sh', '-c', runCommand(f, { out: part, seconds, size: c.t.size, fps: c.t.fps })] : cliArgv(c.cli!, f, part, seconds)
-    const p = Bun.spawn(argv, { cwd: c.root, stdout: 'pipe', stderr: 'pipe' })
+    const argv = f.kind === 'run' ? ['sh', '-c', runCommand(f, { out: part, seconds, size: c.t.size, fps: c.t.fps })] : cliArgv(c.cli!, f, part, seconds, tk.argv)
+    // the cut's frame and rate: a row that films at the cut's size reads them
+    const env = { ...process.env, SK_TRAILER_SIZE: `${c.t.size[0]}x${c.t.size[1]}`, SK_TRAILER_FPS: String(c.t.fps) }
+    const p = Bun.spawn(argv, { cwd: c.root, stdout: 'pipe', stderr: 'pipe', env })
     const [out, err, code] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text(), p.exited])
     const said = `${out}${err}`.trim()
     if (code !== 0 || !existsSync(part)) {
         rmSync(part, { force: true })
-        const shown = f.kind === 'run' ? f.cmd : [...(c.cli ?? []), ...f.argv].join(' ')
+        const shown = f.kind === 'run' ? f.cmd : [...(c.cli ?? []), ...(tk.argv ?? f.argv)].join(' ')
         throw new Error(`shot ${tk.shot.name} (line ${f.line}): ${shown} ${code !== 0 ? `exited ${code}` : `wrote no file at its --out`}${said ? `\n${said.split('\n').slice(-8).join('\n')}` : ''}`)
     }
     return said
 }
 
 /** a `film url` take through the house shot service; a throw names the shot, its line and the service's reason */
-async function urlTake(tk: Take, f: Extract<TakeFilm, { kind: 'url' }>, part: string, seconds: number | null, c: FilmContext): Promise<string> {
+async function urlTake(tk: Take, f: Extract<TakeFilm, { kind: 'url' | 'page' }>, part: string, seconds: number | null, c: FilmContext): Promise<string> {
     const { filmUrl } = await import('./url.ts')
     try {
         const note = await filmUrl(f, { root: c.root, base: c.t.base, size: c.t.size, fps: c.t.fps, seconds, out: part, say: c.say })
@@ -99,6 +168,6 @@ async function urlTake(tk: Take, f: Extract<TakeFilm, { kind: 'url' }>, part: st
         return note
     } catch (e) {
         rmSync(part, { force: true })
-        throw new Error(`shot ${tk.shot.name} (line ${f.line}): film url ${f.route}: ${(e as Error).message}`)
+        throw new Error(`shot ${tk.shot.name} (line ${f.line}): film ${f.kind} ${f.route}: ${(e as Error).message}`)
     }
 }