videobook 0.0.0-stage → 0.2.0

package/dist/rhythm/section-labels.js

@@ -0,0 +1,168 @@
+import { INTELLIGENCE_MODEL, buildIntelligenceResponses, firstResponseText, intelligenceRequest, } from "../openai-intelligence.js";
+import { getApiKey } from "../api-key.js";
+import { assessToolCallEstimate, logUnavailableCostEstimate } from "../jobs/cost-estimate-assessment.js";
+import { energySegments } from "./energy.js";
+import { sectionLabelsSchema, } from "./model.js";
+import { cachedMusicAnalysis, musicSource, readSectionLabels, sectionLabelsCurrent } from "./music-analysis.js";
+import { SECTION_DETECTOR_ID, SECTION_DETECTOR_VERSION, SECTION_LABELER_ID, SECTION_LABELER_VERSION } from "./contract.js";
+const LABEL_PROMPT = [
+    "You label pre-detected section boundaries in a song for a video editor.",
+    "Each candidate boundary was detected from the song's energy curve; its id, position and detection quality are given.",
+    "Assign exactly one label per candidate id, choosing from: intro, verse, chorus, drop, break, bridge, outro.",
+    "You cannot move, add, or remove boundaries: label only the listed candidate ids.",
+    "Labels are editorial suggestions, not facts. Judge from the energy arc around each boundary, not from lyrics you imagine.",
+    'Answer with JSON {"labels": [{candidate_id, label, confidence, rationale}]} covering every candidate id exactly once.',
+    "confidence is 0-1. rationale is one short sentence grounded in the reported energy data.",
+].join(" ");
+const LABELS_JSON_SCHEMA = {
+    type: "object",
+    required: ["labels"],
+    additionalProperties: false,
+    properties: {
+        labels: {
+            type: "array",
+            items: {
+                type: "object",
+                required: ["candidate_id", "label", "confidence", "rationale"],
+                additionalProperties: false,
+                properties: {
+                    candidate_id: { type: "string" },
+                    label: { type: "string", enum: ["intro", "verse", "chorus", "drop", "break", "bridge", "outro"] },
+                    confidence: { type: "number" },
+                    rationale: { type: "string" },
+                },
+            },
+        },
+    },
+};
+export async function labelMusicSections(engine, artifactId, analysisId, streamId, signal) {
+    const source = await musicSource(engine, artifactId, streamId);
+    if (source.analysisId !== analysisId)
+        throw new Error("The audio source changed; analyze it again");
+    const analysis = await cachedMusicAnalysis(engine, source);
+    if (!analysis || analysis.analysisId !== analysisId)
+        throw new Error("Run analyze_music before requesting section labels");
+    if (!analysis.energy || !analysis.sections)
+        throw new Error("The analysis has no energy curve; analyze the current audio again");
+    const stored = await readSectionLabels(engine, artifactId, analysisId);
+    if (stored && stored.status === "ready" && sectionLabelsCurrent(stored, analysis))
+        return stored;
+    const next = await buildLabels(engine, analysis, signal);
+    const saved = await engine.metadata.artifacts.write(artifactId, `music.sections.${analysisId}`, next);
+    if (!saved.ok)
+        throw new Error(saved.error.message);
+    return next;
+}
+async function buildLabels(engine, analysis, signal) {
+    const header = {
+        schemaVersion: 1,
+        analysisId: analysis.analysisId,
+        artifactId: analysis.artifactId,
+        objectHash: analysis.objectHash,
+        streamId: analysis.streamId,
+        labeler: { id: SECTION_LABELER_ID, version: SECTION_LABELER_VERSION, model: "" },
+        boundaries: {
+            algorithmId: analysis.sections?.algorithmId ?? SECTION_DETECTOR_ID,
+            algorithmVersion: analysis.sections?.algorithmVersion ?? SECTION_DETECTOR_VERSION,
+        },
+        createdAt: Date.now(),
+    };
+    const apiKey = getApiKey("OPENAI_API_KEY");
+    if (!apiKey) {
+        return sectionLabelsSchema.parse({
+            ...header,
+            status: "unavailable",
+            reason: "OPENAI_API_KEY is not set; section labels stay optional",
+            labels: [],
+        });
+    }
+    signal.throwIfAborted();
+    await adviseCostEstimate(engine, analysis);
+    const response = await requestLabels(apiKey, analysis, signal);
+    return sectionLabelsSchema.parse({
+        ...header,
+        labeler: { id: SECTION_LABELER_ID, version: SECTION_LABELER_VERSION, model: INTELLIGENCE_MODEL },
+        status: "ready",
+        reason: null,
+        labels: response,
+    });
+}
+async function requestLabels(apiKey, analysis, signal) {
+    if (!analysis.energy || !analysis.sections)
+        throw new Error("The analysis has no energy curve");
+    const segments = energySegments(analysis.energy, analysis.sections, analysis.durationSeconds);
+    const tierFor = (timeSeconds) => segments.find((segment) => timeSeconds >= segment.startSeconds && timeSeconds < segment.endSeconds);
+    const lines = analysis.sections.candidates.map((candidate) => {
+        const segment = tierFor(candidate.timeSeconds);
+        const mean = segment ? `segment ${segment.startSeconds.toFixed(1)}-${segment.endSeconds.toFixed(1)}s energy ${segment.tier} (mean ${segment.meanEnergy.toFixed(2)})` : "segment energy unknown";
+        return `${candidate.id}: ${candidate.timeSeconds.toFixed(1)}s, detection quality ${candidate.quality.toFixed(2)}, ${mean}`;
+    });
+    const content = [{
+            type: "input_text",
+            text: [
+                `Song duration ${analysis.durationSeconds.toFixed(1)}s${analysis.bpm ? `, tempo about ${analysis.bpm.toFixed(0)} BPM` : ""}.`,
+                `Detected section boundary candidates:`,
+                ...lines,
+            ].join("\n"),
+        }];
+    const responses = buildIntelligenceResponses(apiKey, { timeout: 120_000 });
+    const response = await responses(intelligenceRequest({
+        instructions: LABEL_PROMPT,
+        input: [{ role: "user", content }],
+        text: { format: { type: "json_schema", name: "section_labels", schema: LABELS_JSON_SCHEMA, strict: true } },
+        max_output_tokens: 2048,
+    }), signal);
+    const wanted = new Map(analysis.sections.candidates.map((candidate, index) => [candidate.id, index]));
+    const seen = new Set();
+    const labels = [];
+    for (const entry of parseLabelResponse(firstResponseText(response))) {
+        if (typeof entry !== "object" || entry === null)
+            continue;
+        const row = entry;
+        const id = typeof row.candidate_id === "string" ? row.candidate_id : "";
+        const label = row.label;
+        if (!wanted.has(id) || seen.has(id))
+            continue;
+        if (typeof label !== "string" || !["intro", "verse", "chorus", "drop", "break", "bridge", "outro"].includes(label))
+            continue;
+        seen.add(id);
+        labels.push({
+            candidateId: id,
+            label: label,
+            confidence: Math.min(1, Math.max(0, typeof row.confidence === "number" && Number.isFinite(row.confidence) ? row.confidence : 0)),
+            rationale: typeof row.rationale === "string" ? row.rationale.slice(0, 500) : "",
+        });
+    }
+    if (labels.length !== wanted.size)
+        throw new Error("The section labeler did not cover every boundary; retry labeling");
+    return labels;
+}
+function parseLabelResponse(text) {
+    const parsed = JSON.parse(text);
+    if (typeof parsed !== "object" || parsed === null)
+        return [];
+    const labels = parsed.labels;
+    return Array.isArray(labels) ? labels : [];
+}
+async function adviseCostEstimate(engine, analysis) {
+    try {
+        const assessment = await assessToolCallEstimate(engine, "analyze_video", { asset_id: analysis.artifactId });
+        if (!assessment.available) {
+            await logUnavailableCostEstimate({
+                assessment,
+                engine,
+                correlation: {
+                    bookId: engine.book.get().bookId,
+                    artifactId: analysis.artifactId,
+                    tool: "label_music_sections",
+                    provider: "openai",
+                    model: INTELLIGENCE_MODEL,
+                    stage: "label-music-sections",
+                },
+            });
+        }
+    }
+    catch {
+        return;
+    }
+}

package/dist/rhythm/sequence-rhythm.js

@@ -0,0 +1,67 @@
+import { randomUUID } from "node:crypto";
+import { z } from "zod";
+import { withNotebookMutation } from "../notebook-mutations.js";
+import { rhythmSettingsSchema } from "./model.js";
+import { soundtrackClip } from "./timing.js";
+import { musicSource } from "./music-analysis.js";
+const documentSchema = z.object({
+    revision: z.string(), settings: rhythmSettingsSchema,
+    past: z.array(rhythmSettingsSchema).max(20), future: z.array(rhythmSettingsSchema).max(20),
+});
+async function rhythmDocument(engine, sequenceId) {
+    if (!engine.sequences.get(sequenceId).ok)
+        throw new Error("The sequence no longer exists");
+    const saved = await engine.metadata.book.read(`music.rhythm.${sequenceId}`);
+    if (saved.ok)
+        return documentSchema.parse(saved.value);
+    return { revision: "none", settings: { masterClipId: null, correction: null }, past: [], future: [] };
+}
+function state(document) {
+    return { revision: document.revision, settings: document.settings, canUndo: document.past.length > 0, canRedo: document.future.length > 0 };
+}
+export async function readSequenceRhythm(engine, sequenceId) {
+    return state(await rhythmDocument(engine, sequenceId));
+}
+export async function updateSequenceRhythm(engine, sequenceId, expectedRevision, update) {
+    return withNotebookMutation(`rhythm:${engine.book.get().bookId}:${sequenceId}`, async () => {
+        const current = await rhythmDocument(engine, sequenceId);
+        if (current.revision !== expectedRevision)
+            throw new Error("The beat map changed. Reload it before editing.");
+        const sequence = engine.sequences.get(sequenceId);
+        if (!sequence.ok)
+            throw new Error(sequence.error.message);
+        const settings = update === "undo" ? current.past.at(-1) : update === "redo" ? current.future.at(-1) : rhythmSettingsSchema.parse(update);
+        if (!settings)
+            throw new Error(`Nothing to ${update}`);
+        const master = soundtrackClip(sequence.value, settings);
+        if (settings.masterClipId && !master)
+            throw new Error("The selected soundtrack clip is no longer available");
+        if (settings.correction) {
+            if (!master || settings.correction.objectHash !== master.source.range.objectHash)
+                throw new Error("The soundtrack changed. Reset its beat map before editing.");
+            const source = await musicSource(engine, master.source.artifactId, master.source.range.streamId);
+            if (source.objectHash !== settings.correction.objectHash)
+                throw new Error("The audio source changed");
+            if (sequence.value.tracks.find((track) => track.trackId === master.trackId)?.locked)
+                throw new Error("Unlock the soundtrack track before correcting its beat map");
+            const streams = engine.streams.get(master.source.range.streamId);
+            if (!streams.ok || streams.value.objectHash !== settings.correction.objectHash)
+                throw new Error("The audio source changed");
+            const duration = streams.value.durationTicks * streams.value.timeBase.numerator / streams.value.timeBase.denominator;
+            if (settings.correction.markers.some((marker) => marker.timeSeconds >= duration))
+                throw new Error("Beat markers must be inside the source audio");
+            const ids = new Set(settings.correction.markers.map((marker) => marker.id));
+            if (ids.size !== settings.correction.markers.length)
+                throw new Error("Beat marker IDs must be unique");
+            settings.correction.markers.sort((left, right) => left.timeSeconds - right.timeSeconds);
+        }
+        const next = { revision: randomUUID(), settings,
+            past: update === "undo" ? current.past.slice(0, -1) : [...current.past, current.settings].slice(-20),
+            future: update === "redo" ? current.future.slice(0, -1) : update === "undo" ? [...current.future, current.settings].slice(-20) : [],
+        };
+        const saved = await engine.metadata.book.write(`music.rhythm.${sequenceId}`, next);
+        if (!saved.ok)
+            throw new Error(saved.error.message);
+        return state(next);
+    });
+}

package/dist/rhythm/synthesize.js

@@ -0,0 +1,103 @@
+import { RHYTHM_SAMPLE_RATE } from "./contract.js";
+export function synthesizeClicks(options) {
+    const sampleRate = options.sampleRate ?? RHYTHM_SAMPLE_RATE;
+    const lead = options.leadSilenceSeconds ?? 0;
+    const samples = new Float32Array(Math.ceil(options.seconds * sampleRate));
+    const beatSeconds = [];
+    const interval = 60 / options.bpm;
+    for (let time = lead; time < options.seconds - 0.05; time += interval) {
+        beatSeconds.push(time);
+        addClick(samples, sampleRate, time, 1);
+    }
+    return { sampleRate, samples, beatSeconds, barSeconds: null };
+}
+export function synthesizePattern(options) {
+    const sampleRate = options.sampleRate ?? RHYTHM_SAMPLE_RATE;
+    const samples = new Float32Array(Math.ceil(options.seconds * sampleRate));
+    const beatSeconds = [];
+    const barSeconds = [];
+    const interval = 60 / options.bpm;
+    let index = 0;
+    for (let time = 0.1; time < options.seconds - 0.05; time += interval) {
+        beatSeconds.push(time);
+        if (index % options.beatsPerBar === 0)
+            barSeconds.push(time);
+        const accent = index % options.accentEvery === 0 ? 1 : 0.55;
+        addClick(samples, sampleRate, time, accent);
+        if (options.syncopated)
+            addClick(samples, sampleRate, time + interval * 0.5, 0.35);
+        index += 1;
+    }
+    if (options.harmonic)
+        addFormant(samples, sampleRate);
+    return {
+        sampleRate,
+        samples,
+        beatSeconds,
+        barSeconds,
+    };
+}
+export function synthesizeTempoChange(options) {
+    const sampleRate = options.sampleRate ?? RHYTHM_SAMPLE_RATE;
+    const samples = new Float32Array(Math.ceil(options.seconds * sampleRate));
+    const beatSeconds = [];
+    let time = 0.1;
+    let bpm = options.firstBpm;
+    while (time < options.seconds - 0.05) {
+        if (time >= options.changeAtSeconds)
+            bpm = options.secondBpm;
+        beatSeconds.push(time);
+        addClick(samples, sampleRate, time, 1);
+        time += 60 / bpm;
+    }
+    return { sampleRate, samples, beatSeconds, barSeconds: null };
+}
+export function synthesizeSpeechLike(seconds, sampleRate = RHYTHM_SAMPLE_RATE) {
+    const samples = new Float32Array(Math.ceil(seconds * sampleRate));
+    let cursor = 0.05;
+    let salt = 17;
+    while (cursor < seconds - 0.1) {
+        salt = (salt * 1_103_515_245 + 12_345) % 2_147_483_648;
+        const gap = 0.07 + (salt % 430) / 1_000;
+        const length = 0.03 + (salt % 90) / 1_000;
+        addNoiseBurst(samples, sampleRate, cursor, length);
+        cursor += gap;
+    }
+    return { sampleRate, samples, beatSeconds: [], barSeconds: null };
+}
+export function synthesizeSilence(seconds, sampleRate = RHYTHM_SAMPLE_RATE) {
+    return {
+        sampleRate,
+        samples: new Float32Array(Math.ceil(seconds * sampleRate)),
+        beatSeconds: [],
+        barSeconds: null,
+    };
+}
+function addClick(samples, sampleRate, timeSeconds, gain) {
+    const start = Math.max(0, Math.floor(timeSeconds * sampleRate));
+    const length = Math.floor(0.012 * sampleRate);
+    for (let index = 0; index < length && start + index < samples.length; index += 1) {
+        const envelope = Math.exp(-index / (0.0022 * sampleRate));
+        const tone = Math.sin((2 * Math.PI * 1_400 * index) / sampleRate);
+        const noise = ((index * 17) % 11) / 11 - 0.5;
+        samples[start + index] = Math.max(-1, Math.min(1, samples[start + index] + gain * envelope * (0.7 * tone + 0.6 * noise)));
+    }
+}
+function addFormant(samples, sampleRate) {
+    for (let index = 0; index < samples.length; index += 1) {
+        const time = index / sampleRate;
+        const vowel = 0.18 * (Math.sin(2 * Math.PI * 220 * time)
+            + 0.5 * Math.sin(2 * Math.PI * 440 * time)
+            + 0.25 * Math.sin(2 * Math.PI * 880 * time));
+        samples[index] = Math.max(-1, Math.min(1, samples[index] * 0.75 + vowel));
+    }
+}
+function addNoiseBurst(samples, sampleRate, timeSeconds, lengthSeconds) {
+    const start = Math.floor(timeSeconds * sampleRate);
+    const length = Math.floor(lengthSeconds * sampleRate);
+    for (let index = 0; index < length && start + index < samples.length; index += 1) {
+        const envelope = Math.sin((Math.PI * index) / length);
+        const noise = Math.sin(index * 12.9898) * 43_758.5453;
+        samples[start + index] = (noise - Math.floor(noise) - 0.5) * envelope;
+    }
+}

package/dist/rhythm/timing.js

@@ -0,0 +1,98 @@
+export function soundtrackClip(sequence, settings) {
+    const audioTracks = new Set(sequence.tracks.filter((track) => track.kind === "audio").map((track) => track.trackId));
+    const clips = sequence.clips.filter((clip) => audioTracks.has(clip.trackId) && clip.source.kind === "timed");
+    return (settings.masterClipId ? clips.find((clip) => clip.clipId === settings.masterClipId) : clips.length === 1 ? clips[0] : null) ?? null;
+}
+export function effectiveMarkers(clip, settings, analysis) {
+    if (!clip)
+        return [];
+    if (settings.correction)
+        return settings.correction.objectHash === clip.source.range.objectHash ? settings.correction.markers : [];
+    if (!analysis || analysis.objectHash !== clip.source.range.objectHash || analysis.streamId !== clip.source.range.streamId)
+        return [];
+    const bars = new Set(analysis.bars?.map((bar) => Math.round(bar.timeSeconds * 1000000)));
+    return analysis.beats.map((beat, index) => ({
+        id: `${analysis.analysisId.slice(0, 12)}-${index}`, timeSeconds: beat.timeSeconds,
+        bar: bars.has(Math.round(beat.timeSeconds * 1000000)), manual: false,
+    }));
+}
+export function beatPositions(beats, bpm) {
+    if (!beats.length)
+        return [];
+    const period = bpm !== null && Number.isFinite(bpm) && bpm > 0 ? 60 / bpm : null;
+    const positions = [0];
+    for (let index = 1; index < beats.length; index += 1) {
+        const gap = beats[index].timeSeconds - beats[index - 1].timeSeconds;
+        positions.push(positions[index - 1] + (period === null ? 1 : Math.max(1, Math.round(gap / period))));
+    }
+    return positions;
+}
+export function groupBeatBars(markers, anchorId, beatsPerBar, bpm) {
+    const ordered = [...markers].sort((left, right) => left.timeSeconds - right.timeSeconds);
+    const anchor = ordered.findIndex(marker => marker.id === anchorId);
+    if (anchor < 0 || !Number.isInteger(beatsPerBar) || beatsPerBar < 1 || beatsPerBar > 16)
+        throw new Error("Choose a first bar beat and 1–16 beats per bar");
+    const positions = beatPositions(ordered, bpm);
+    return ordered.map((marker, index) => ({
+        ...marker, bar: (positions[index] - positions[anchor]) % beatsPerBar === 0, manual: true,
+    }));
+}
+export function sourceTimeAtFrame(clip, frame, fps) {
+    const range = clip.source.range;
+    const base = range.timeBase.numerator / range.timeBase.denominator;
+    const elapsed = (frame - clip.timelineStartFrame) / fps * clip.speed.numerator / clip.speed.denominator;
+    return clip.reverse ? (range.startTick + range.durationTicks) * base - elapsed : range.startTick * base + elapsed;
+}
+export function frameAtSourceTime(clip, seconds, fps) {
+    const range = clip.source.range;
+    const base = range.timeBase.numerator / range.timeBase.denominator;
+    const elapsed = clip.reverse ? (range.startTick + range.durationTicks) * base - seconds : seconds - range.startTick * base;
+    return Math.round(clip.timelineStartFrame + elapsed * fps * clip.speed.denominator / clip.speed.numerator);
+}
+export function timelineMarkers(clip, markers, sequence) {
+    if (!clip || !clip.enabled)
+        return [];
+    const fps = sequence.frameRate.numerator / sequence.frameRate.denominator;
+    const range = clip.source.range;
+    const base = range.timeBase.numerator / range.timeBase.denominator;
+    const startSeconds = range.startTick * base;
+    const endSeconds = (range.startTick + range.durationTicks) * base;
+    const seen = new Set();
+    return markers.filter((marker) => marker.timeSeconds >= startSeconds && marker.timeSeconds < endSeconds)
+        .map((marker) => ({ ...marker, frame: frameAtSourceTime(clip, marker.timeSeconds, fps) }))
+        .filter((marker) => marker.frame >= clip.timelineStartFrame && marker.frame < clip.timelineStartFrame + clip.durationFrames)
+        .sort((left, right) => left.frame - right.frame || Number(right.bar) - Number(left.bar))
+        .filter((marker) => { if (seen.has(marker.frame))
+        return false; seen.add(marker.frame); return true; });
+}
+export function nearestBeat(frame, markers, mode, threshold = Infinity) {
+    if (mode === "off")
+        return Math.round(frame);
+    let best = Math.round(frame);
+    let distance = threshold;
+    for (const marker of markers) {
+        if (mode === "bars" && !marker.bar)
+            continue;
+        const delta = Math.abs(marker.frame - frame);
+        if (delta < distance || delta === distance && marker.frame < best) {
+            best = marker.frame;
+            distance = delta;
+        }
+    }
+    return best;
+}
+export function clipWaveform(clip, waveform, sourceDuration, fps, samples = 256) {
+    if (clip.source.kind !== "timed" || !sourceDuration || !waveform.length)
+        return [];
+    const timed = clip;
+    return Array.from({ length: samples }, (_, index) => {
+        const start = sourceTimeAtFrame(timed, clip.timelineStartFrame + index / samples * clip.durationFrames, fps);
+        const end = sourceTimeAtFrame(timed, clip.timelineStartFrame + (index + 1) / samples * clip.durationFrames, fps);
+        const first = Math.max(0, Math.floor(Math.min(start, end) / sourceDuration * waveform.length));
+        const last = Math.min(waveform.length, Math.max(first + 1, Math.ceil(Math.max(start, end) / sourceDuration * waveform.length)));
+        let peak = 0;
+        for (let bucket = first; bucket < last; bucket++)
+            peak = Math.max(peak, waveform[bucket] ?? 0);
+        return peak;
+    });
+}