videobook 0.0.0-stage → 0.2.0

package/dist/rhythm/track.js

@@ -0,0 +1,320 @@
+import { RHYTHM_ANALYZER_ID, RHYTHM_ANALYZER_VERSION, RHYTHM_LATENCY_SAMPLES, RHYTHM_SCHEMA_VERSION, } from "./contract.js";
+import { energyCurve } from "./energy.js";
+import { beatPositions } from "./timing.js";
+const MIN_BPM = 55;
+const MAX_BPM = 200;
+const TIGHTNESS = 80;
+const LOW_CONFIDENCE = 0.42;
+export function trackBeats(envelope) {
+    const hopSeconds = envelope.hopSamples / envelope.sampleRate;
+    const latencySeconds = RHYTHM_LATENCY_SAMPLES / envelope.sampleRate;
+    const onsets = pickOnsets(envelope.strength, hopSeconds);
+    const tempo = estimateTempo(envelope.strength, hopSeconds, onsets);
+    const tracked = tempo.periodHops === null
+        ? []
+        : beatTrack(envelope.strength, tempo.periodHops);
+    const beatTimes = tracked
+        .map((frame) => frame * hopSeconds)
+        .filter((time) => time >= 0);
+    const confidences = beatConfidences(envelope.strength, tracked);
+    const global = globalConfidence(confidences, envelope.strength, tracked);
+    const weakBeats = global < LOW_CONFIDENCE;
+    const noBeats = tempo.bpm === null || global <= 0 || beatTimes.length < (weakBeats ? 8 : 2);
+    const lowConfidence = noBeats || weakBeats || beatTimes.length < 8 || tempo.ambiguity !== "stable";
+    const beats = noBeats
+        ? []
+        : beatTimes.map((timeSeconds, index) => ({
+            timeSeconds,
+            confidence: confidences[index] ?? null,
+        }));
+    const positions = beatPositions(beats, tempo.bpm);
+    const detectedMeter = noBeats || weakBeats ? null : detectMeter(envelope.strength, tracked, positions);
+    const assignedMeter = detectedMeter ?? (noBeats ? null : { beatsPerBar: 4, confidence: 0, phase: 0 });
+    const meter = assignedMeter
+        ? { beatsPerBar: assignedMeter.beatsPerBar, confidence: assignedMeter.confidence }
+        : null;
+    const bars = assignedMeter ? barStarts(beats, positions, assignedMeter) : null;
+    return {
+        schemaVersion: RHYTHM_SCHEMA_VERSION,
+        analyzerId: RHYTHM_ANALYZER_ID,
+        analyzerVersion: RHYTHM_ANALYZER_VERSION,
+        sampleRate: envelope.sampleRate,
+        hopSamples: envelope.hopSamples,
+        windowSamples: envelope.windowSamples,
+        latencySeconds,
+        beats,
+        onsets,
+        bpm: noBeats ? null : tempo.bpm,
+        bpmConfidence: tempo.confidence,
+        tempoAmbiguity: tempo.ambiguity,
+        meter: noBeats ? null : meter,
+        bars: noBeats ? null : bars,
+        barSource: noBeats ? null : detectedMeter ? "detected" : "estimated",
+        noBeats,
+        lowConfidence,
+        energy: energyCurve(envelope),
+    };
+}
+function pickOnsets(strength, hopSeconds) {
+    if (strength.length < 3)
+        return [];
+    let mean = 0;
+    for (const value of strength)
+        mean += value;
+    mean /= strength.length;
+    let variance = 0;
+    for (const value of strength)
+        variance += (value - mean) ** 2;
+    const deviation = Math.sqrt(variance / strength.length);
+    const threshold = Math.max(0.08, mean + deviation * 0.4);
+    const minGap = Math.max(1, Math.round(0.05 / hopSeconds));
+    const onsets = [];
+    let last = -minGap;
+    for (let index = 1; index < strength.length - 1; index += 1) {
+        const value = strength[index];
+        if (value < threshold)
+            continue;
+        if (value < strength[index - 1] || value < strength[index + 1])
+            continue;
+        if (index - last < minGap) {
+            const previous = onsets[onsets.length - 1];
+            if (previous && value > previous.strength) {
+                onsets[onsets.length - 1] = { timeSeconds: index * hopSeconds, strength: value };
+                last = index;
+            }
+            continue;
+        }
+        onsets.push({ timeSeconds: index * hopSeconds, strength: value });
+        last = index;
+    }
+    return onsets;
+}
+function estimateTempo(strength, hopSeconds, onsets) {
+    const minLag = Math.max(2, Math.round(60 / MAX_BPM / hopSeconds));
+    const maxLag = Math.min(strength.length - 2, Math.round(60 / MIN_BPM / hopSeconds));
+    if (maxLag <= minLag + 1 || onsets.length < 2) {
+        return { bpm: null, confidence: null, ambiguity: null, periodHops: null };
+    }
+    const correlation = autocorrelate(strength, minLag, maxLag);
+    let bestLag = minLag;
+    let best = -Infinity;
+    for (let lag = minLag; lag <= maxLag; lag += 1) {
+        const value = correlation[lag - minLag];
+        if (value > best) {
+            best = value;
+            bestLag = lag;
+        }
+    }
+    const onsetPeriod = onsetInterval(onsets, hopSeconds, minLag, maxLag);
+    const resolved = resolveOctave(bestLag, correlation, minLag, onsetPeriod);
+    const bpm = 60 / (resolved.lag * hopSeconds);
+    const mean = correlation.reduce((sum, value) => sum + value, 0) / correlation.length;
+    const peakiness = best <= 0 ? 0 : (best - mean) / best;
+    const confidence = best <= 0 ? 0 : Math.max(0, Math.min(1, peakiness));
+    if (peakiness < 0.55) {
+        return { bpm: null, confidence, ambiguity: null, periodHops: null };
+    }
+    return {
+        bpm,
+        confidence,
+        ambiguity: resolved.ambiguity,
+        periodHops: resolved.lag,
+    };
+}
+function autocorrelate(strength, minLag, maxLag) {
+    const values = new Float64Array(maxLag - minLag + 1);
+    let energy = 0;
+    for (const value of strength)
+        energy += value * value;
+    const norm = Math.sqrt(energy) || 1;
+    for (let lag = minLag; lag <= maxLag; lag += 1) {
+        let sum = 0;
+        for (let index = 0; index + lag < strength.length; index += 1) {
+            sum += strength[index] * strength[index + lag];
+        }
+        values[lag - minLag] = sum / norm;
+    }
+    return values;
+}
+function onsetInterval(onsets, hopSeconds, minLag, maxLag) {
+    const gaps = [];
+    for (let index = 1; index < onsets.length; index += 1) {
+        const hops = Math.round((onsets[index].timeSeconds - onsets[index - 1].timeSeconds) / hopSeconds);
+        if (hops >= minLag && hops <= maxLag)
+            gaps.push(hops);
+    }
+    if (gaps.length === 0)
+        return null;
+    gaps.sort((left, right) => left - right);
+    return gaps[Math.floor(gaps.length / 2)];
+}
+function resolveOctave(lag, correlation, minLag, onsetPeriod) {
+    const candidates = [lag, Math.round(lag / 2), lag * 2].filter((value) => {
+        const index = value - minLag;
+        return value >= minLag && index >= 0 && index < correlation.length;
+    });
+    const unique = [...new Set(candidates)];
+    let chosen = lag;
+    let bestScore = -Infinity;
+    for (const candidate of unique) {
+        const auto = correlation[candidate - minLag] ?? 0;
+        const onsetScore = onsetPeriod === null
+            ? 0
+            : 1 - Math.min(1, Math.abs(candidate - onsetPeriod) / onsetPeriod);
+        const score = auto + onsetScore * Math.max(auto, 1);
+        if (score > bestScore) {
+            bestScore = score;
+            chosen = candidate;
+        }
+    }
+    let ambiguity = "stable";
+    const half = correlation[Math.round(chosen / 2) - minLag];
+    const double = correlation[chosen * 2 - minLag];
+    const self = correlation[chosen - minLag] ?? 0;
+    if (half !== undefined && self > 0 && half > self * 0.92)
+        ambiguity = "half";
+    else if (double !== undefined && self > 0 && double > self * 0.92)
+        ambiguity = "double";
+    if (chosen !== lag && chosen < lag)
+        ambiguity = "double";
+    if (chosen !== lag && chosen > lag)
+        ambiguity = "half";
+    return { lag: chosen, ambiguity };
+}
+function beatTrack(strength, period) {
+    const count = strength.length;
+    const score = new Float64Array(count);
+    const back = new Int32Array(count);
+    back.fill(-1);
+    const minLag = Math.max(1, Math.round(period * 0.55));
+    const maxLag = Math.max(minLag + 1, Math.round(period * 1.7));
+    for (let index = 0; index < count; index += 1) {
+        let best = strength[index];
+        let link = -1;
+        const start = Math.max(0, index - maxLag);
+        const end = index - minLag;
+        for (let previous = start; previous <= end; previous += 1) {
+            const lag = index - previous;
+            const penalty = TIGHTNESS * (Math.log(lag / period) ** 2);
+            const candidate = score[previous] + strength[index] - penalty;
+            if (candidate > best) {
+                best = candidate;
+                link = previous;
+            }
+        }
+        score[index] = link < 0 ? strength[index] : best;
+        back[index] = link;
+    }
+    let cursor = -1;
+    let bestScore = -Infinity;
+    const tail = Math.max(0, count - Math.round(period * 2));
+    for (let index = tail; index < count; index += 1) {
+        if (score[index] > bestScore) {
+            bestScore = score[index];
+            cursor = index;
+        }
+    }
+    const frames = [];
+    const guard = new Set();
+    while (cursor >= 0 && !guard.has(cursor)) {
+        guard.add(cursor);
+        if (strength[cursor] > 0.02)
+            frames.push(cursor);
+        cursor = back[cursor];
+    }
+    frames.reverse();
+    return thinBeats(frames, period);
+}
+function thinBeats(frames, period) {
+    const minimum = Math.max(1, Math.round(period * 0.55));
+    const kept = [];
+    for (const frame of frames) {
+        const previous = kept[kept.length - 1];
+        if (previous === undefined || frame - previous >= minimum)
+            kept.push(frame);
+        else
+            kept[kept.length - 1] = frame;
+    }
+    return kept;
+}
+function beatConfidences(strength, frames) {
+    return frames.map((frame) => {
+        const value = strength[frame] ?? 0;
+        return value > 0 ? Math.max(0, Math.min(1, value)) : null;
+    });
+}
+function globalConfidence(confidences, strength, frames) {
+    if (frames.length < 2)
+        return 0;
+    let sum = 0;
+    let known = 0;
+    for (const value of confidences) {
+        if (value === null)
+            continue;
+        sum += value;
+        known += 1;
+    }
+    if (known === 0)
+        return 0;
+    const mean = sum / known;
+    let noise = 0;
+    let samples = 0;
+    for (let index = 0; index < strength.length; index += 8) {
+        noise += strength[index];
+        samples += 1;
+    }
+    const floor = samples ? noise / samples : 0;
+    return Math.max(0, Math.min(1, mean - floor));
+}
+function detectMeter(strength, frames, positions) {
+    if (frames.length < 8)
+        return null;
+    let best = null;
+    for (const beatsPerBar of [3, 4, 5, 6]) {
+        if (frames.length < beatsPerBar * 4)
+            continue;
+        let bestPhase = 0;
+        let bestMean = -Infinity;
+        let other = 0;
+        for (let phase = 0; phase < beatsPerBar; phase += 1) {
+            let sum = 0;
+            let count = 0;
+            for (let index = 0; index < frames.length; index += 1) {
+                if (positions[index] % beatsPerBar !== phase)
+                    continue;
+                sum += strength[frames[index]] ?? 0;
+                count += 1;
+            }
+            const mean = count ? sum / count : 0;
+            if (mean > bestMean) {
+                other = bestMean;
+                bestMean = mean;
+                bestPhase = phase;
+            }
+            else if (mean > other) {
+                other = mean;
+            }
+        }
+        const confidence = bestMean <= 0 ? 0 : (bestMean - Math.max(0, other)) / bestMean;
+        if (confidence < 0.18)
+            continue;
+        if (!best || confidence > best.confidence) {
+            best = { beatsPerBar, confidence, phase: bestPhase };
+        }
+    }
+    if (!best || best.confidence < 0.4)
+        return null;
+    return best;
+}
+function barStarts(beats, positions, meter) {
+    const bars = [];
+    for (let index = 0; index < beats.length; index += 1) {
+        if (positions[index] % meter.beatsPerBar !== meter.phase)
+            continue;
+        const beat = beats[index];
+        if (!beat)
+            continue;
+        bars.push({ timeSeconds: beat.timeSeconds, confidence: meter.confidence });
+    }
+    return bars.length ? bars : null;
+}

package/dist/rhythm/visual-fit.js

@@ -0,0 +1,4 @@
+export function fittedTransform(fit = "fit") {
+    return { fit, positionX: 0, positionY: 0, scaleX: 1, scaleY: 1, anchorX: 0.5, anchorY: 0.5,
+        rotationDegrees: 0, cropTop: 0, cropRight: 0, cropBottom: 0, cropLeft: 0, opacity: 1, blendMode: "normal" };
+}

package/dist/rhythm/worker.js

@@ -0,0 +1,25 @@
+import { analyzeAudioFile } from "./analyze.js";
+const controller = new AbortController();
+process.once("SIGTERM", () => controller.abort());
+process.once("disconnect", () => controller.abort());
+async function send(message) {
+    if (!process.connected || !process.send)
+        return;
+    await new Promise((resolve, reject) => {
+        process.send(message, (error) => error ? reject(error) : resolve());
+    });
+}
+process.once("message", async (message) => {
+    try {
+        const analysis = await analyzeAudioFile(message.ffmpegPath, message.inputPath, controller.signal, message.streamIndex);
+        await send({ analysis });
+    }
+    catch (error) {
+        if (process.connected)
+            await send({ error: error instanceof Error ? error.message : String(error) });
+    }
+    finally {
+        if (process.connected)
+            process.disconnect?.();
+    }
+});

package/dist/scene-detect.js

@@ -0,0 +1,200 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { FFMPEG } from "./ffmpeg-binary.js";
+const execFileAsync = promisify(execFile);
+// Camera-cut threshold on scdet's absolute mean frame diff. ffmpeg's default
+// scdet threshold is 10; we match that for the "shot" classification. Anything
+// below this within a single shot is candidate territory for "settle".
+const SHOT_SCORE_THRESHOLD = 10;
+// scdet runs at this temporal resolution. 4 fps gives 0.25s candidate
+// granularity — enough to land action endings cleanly while keeping the
+// detection pass fast on long clips.
+const SCDET_FPS = 4;
+// Settle: within a single shot, look for the falling edge of motion. The
+// frame must drop below the segment's 25th percentile after sustained motion
+// above the 75th percentile, AND stay below p25 for at least this many
+// seconds. Tuned so a short pause inside an action doesn't produce a false
+// settle, but a real "the man finished standing up" landing does.
+const SETTLE_HOLD_S = 0.5;
+// Parse scdet's stderr stream. ffmpeg 6's scdet logs one line per processed
+// frame in the form:
+//   [scdet @ 0x...] lavfi.scd.score: 4.798, lavfi.scd.time: 0.25
+// Score is the absolute mean frame difference (Y-plane), time is seconds.
+// The very first line always has score=0 (no previous frame) — we keep it
+// in the timeline; deriveCandidates ignores it via the threshold.
+function parseScdetOutput(stderr) {
+    const re = /lavfi\.scd\.score:\s*([0-9.]+),\s*lavfi\.scd\.time:\s*([0-9.]+)/g;
+    const out = [];
+    let m;
+    while ((m = re.exec(stderr)) !== null) {
+        const score = parseFloat(m[1]);
+        const t = parseFloat(m[2]);
+        if (Number.isFinite(score) && Number.isFinite(t))
+            out.push({ t, score });
+    }
+    return out;
+}
+export function deriveCameraCandidates(samples, scoreFloor = 6, clusterSeconds = 0.5) {
+    const peaks = samples.filter((sample, index) => {
+        if (sample.score < scoreFloor)
+            return false;
+        const previous = samples[index - 1]?.score ?? Number.NEGATIVE_INFINITY;
+        const next = samples[index + 1]?.score ?? Number.NEGATIVE_INFINITY;
+        return sample.score >= previous && sample.score >= next;
+    });
+    const clustered = [];
+    for (const peak of peaks) {
+        const previous = clustered.at(-1);
+        if (!previous || peak.t - previous.t > clusterSeconds) {
+            clustered.push({ ...peak });
+            continue;
+        }
+        if (peak.score > previous.score) {
+            clustered[clustered.length - 1] = { ...peak };
+        }
+    }
+    return clustered;
+}
+// Two-tier candidate derivation from a per-frame scdet timeline:
+//
+// 1. shot   — any frame whose scdet score is ≥ SHOT_SCORE_THRESHOLD. These
+//             are camera/cut changes. ffmpeg's default scdet threshold is the
+//             same value; matching it keeps behavior aligned with the upstream
+//             definition of "scene change".
+//
+// 2. settle — within each run of consecutive non-shot frames (a single shot),
+//             find a falling edge: a frame where the score drops below the
+//             segment's 25th percentile after sustained motion above the 75th
+//             percentile, and stays below p25 for ≥ SETTLE_HOLD_S. The first
+//             frame of the sustained-low region is the action's natural end.
+//
+// Settle scoring is (above-mean − below-mean), so deeper drops rank higher.
+// Empty / quiet shots produce no settle candidates.
+function deriveCandidates(samples) {
+    if (samples.length === 0)
+        return [];
+    const out = [];
+    const shotIdx = [];
+    for (let i = 0; i < samples.length; i++) {
+        if (samples[i].score >= SHOT_SCORE_THRESHOLD) {
+            shotIdx.push(i);
+            out.push({ t: samples[i].t, score: samples[i].score, kind: "shot" });
+        }
+    }
+    const segments = [];
+    let segStart = 0;
+    for (const i of shotIdx) {
+        if (i > segStart)
+            segments.push([segStart, i]);
+        segStart = i + 1;
+    }
+    if (segStart < samples.length)
+        segments.push([segStart, samples.length]);
+    for (const [lo, hi] of segments) {
+        const seg = samples.slice(lo, hi);
+        if (seg.length < 3)
+            continue;
+        const sorted = [...seg.map((s) => s.score)].sort((a, b) => a - b);
+        const p25 = sorted[Math.floor(sorted.length * 0.25)];
+        const p75 = sorted[Math.floor(sorted.length * 0.75)];
+        if (p75 <= p25)
+            continue;
+        let aboveSum = 0;
+        let aboveCount = 0;
+        for (let j = 0; j < seg.length; j++) {
+            if (seg[j].score >= p75) {
+                aboveSum += seg[j].score;
+                aboveCount++;
+            }
+            if (aboveCount > 0 && seg[j].score <= p25) {
+                const start = seg[j].t;
+                let k = j;
+                while (k < seg.length && seg[k].score <= p25 && seg[k].t - start < SETTLE_HOLD_S)
+                    k++;
+                if (k < seg.length && seg[k].t - start >= SETTLE_HOLD_S) {
+                    let belowSum = 0;
+                    let belowCount = 0;
+                    for (let m = j; m < k; m++) {
+                        belowSum += seg[m].score;
+                        belowCount++;
+                    }
+                    const aboveMean = aboveSum / aboveCount;
+                    const belowMean = belowCount > 0 ? belowSum / belowCount : 0;
+                    out.push({ t: seg[j].t, score: Math.max(0, aboveMean - belowMean), kind: "settle" });
+                    j = k;
+                    aboveSum = 0;
+                    aboveCount = 0;
+                }
+            }
+        }
+    }
+    out.sort((a, b) => a.t - b.t);
+    return out;
+}
+// Run a single ffmpeg scdet pass over the input video at downscaled
+// 320×180 / SCDET_FPS, parse the per-frame score timeline from stderr, and
+// return shot + settle candidates. ffmpeg writes scdet's metadata to stderr
+// (not stdout), so we capture stderr from the invocation. A non-zero exit is
+// expected for `-f null -` runs only when ffmpeg actually fails — successful
+// runs return cleanly with the metadata in stderr.
+export async function detectSceneSamples(videoPath, options = {}) {
+    let stderr = "";
+    const startSeconds = Math.max(0, options.startSeconds ?? 0);
+    const args = [
+        "-hide_banner",
+        "-nostats",
+        "-loglevel", "info",
+    ];
+    if (startSeconds > 0)
+        args.push("-ss", String(startSeconds));
+    args.push("-i", videoPath);
+    if (typeof options.durationSeconds === "number"
+        && options.durationSeconds > 0) {
+        args.push("-t", String(options.durationSeconds));
+    }
+    const filters = ["scale=320:180"];
+    if (typeof options.fps === "number" && options.fps > 0) {
+        filters.push(`fps=${options.fps}`);
+    }
+    filters.push("scdet=threshold=0");
+    args.push("-vf", filters.join(","), "-an", "-f", "null", "-");
+    try {
+        const result = await execFileAsync(FFMPEG, args, {
+            maxBuffer: 64 * 1024 * 1024,
+            ...(options.signal ? { signal: options.signal } : {}),
+        });
+        stderr = result.stderr;
+    }
+    catch (error) {
+        if (options.signal?.aborted) {
+            throw options.signal.reason instanceof Error
+                ? options.signal.reason
+                : new Error("Scene detection was cancelled");
+        }
+        const e = error;
+        throw new Error(`scdet pass failed: ${e.message}`);
+    }
+    return parseScdetOutput(stderr).map((sample) => ({
+        ...sample,
+        t: sample.t + startSeconds,
+    }));
+}
+export async function refineCameraCandidate(videoPath, candidate, signal) {
+    const startSeconds = Math.max(0, candidate.t - 0.75);
+    const samples = await detectSceneSamples(videoPath, {
+        startSeconds,
+        durationSeconds: 1.5,
+        signal,
+    });
+    const best = samples
+        .filter((sample) => Math.abs(sample.t - candidate.t) <= 0.75)
+        .sort((left, right) => right.score - left.score)[0];
+    return best ? { t: best.t, score: best.score } : candidate;
+}
+export async function detectSceneCandidates(videoPath, signal) {
+    const samples = await detectSceneSamples(videoPath, {
+        fps: SCDET_FPS,
+        signal,
+    });
+    return deriveCandidates(samples);
+}

package/dist/scene-transition.js

@@ -0,0 +1,12 @@
+export const MIN_TRANSITION_SECONDS = 0.05;
+export function fixedTransitionSeconds(firstDuration, secondDuration, requested) {
+    const available = Math.min(firstDuration, secondDuration);
+    const transition = requested ?? Math.max(MIN_TRANSITION_SECONDS, Math.min(0.35, available / 2));
+    if (!Number.isFinite(transition) || transition < MIN_TRANSITION_SECONDS) {
+        throw new Error(`Transition must be at least ${MIN_TRANSITION_SECONDS} seconds`);
+    }
+    if (!Number.isFinite(available) || transition >= available) {
+        throw new Error("Transition must be shorter than both video clips");
+    }
+    return transition;
+}