vivid-editor-core 0.2.0 → 0.3.1

package/dist/ai-orchestrator.d.ts

@@ -156,7 +156,11 @@ export interface DetachAudioCommand {
  * `audioClipId` — id of an audio clip already in timeline. Must have a
  * cached BeatAnalysis (otherwise the orchestrator returns an error).
  *
- * `beatInterval` — how many beats between cuts. 4 = 1 cut per measure.
+ * `beatInterval` — how many beats (or peaks, with grid 'peaks') between
+ *  cuts. 4 = 1 cut per measure.
+ *
+ * `grid` — 'beats' (default, constant BPM grid) or 'peaks' (detected
+ *  transients from the server analysis; `minStrength` filters weak ones).
  *
  * `mode`:
  *  - "redistribute" (default): take all visual clips, slice them up so
@@ -177,6 +181,15 @@ export interface CutToBeatCommand {
         beatInterval: number;
         mode?: 'redistribute' | 'snap';
         syncClips?: 'all' | string[];
+        /**
+         * Which grid to cut on. 'beats' (default) = the constant BPM grid.
+         * 'peaks' = detected transients (hits, accents) — cuts land on what the
+         * ear actually hears, also on music without a steady tempo or on voice.
+         * Needs `peaks` in the BeatAnalysis (server-side analysis).
+         */
+        grid?: 'beats' | 'peaks';
+        /** grid 'peaks' only: ignore peaks weaker than this (0–1, default 0). */
+        minStrength?: number;
     };
 }
 /** Set keyframe animations on one property of a canvas object or text overlay. */

package/dist/ai-orchestrator.js

@@ -309,9 +309,19 @@ export function executeAiCommands(commands, snapshot, actions) {
                     }
                     const analysis = snapshot.audioBeatsByAssetId?.[audioClip.assetId];
                     if (!analysis || analysis === 'analyzing' || analysis === 'failed') {
-                        errors.push(`CUT_TO_BEAT: audio "${audioClip.name}" has no beat analysis (status: ${analysis ?? 'unknown'}). Wait a moment after dropping audio for analysis to complete.`);
+                        errors.push(`CUT_TO_BEAT: audio "${audioClip.name}" has no beat analysis (status: ${analysis ?? 'unknown'}). In the browser wait a moment after dropping the audio; via the API run POST /api/ai/audio-analysis on the asset first.`);
                         break;
                     }
+                    const grid = cmd.payload.grid ?? 'beats';
+                    let gridMs = analysis.beats;
+                    if (grid === 'peaks') {
+                        if (!analysis.peaks || analysis.peaks.length === 0) {
+                            errors.push(`CUT_TO_BEAT: audio "${audioClip.name}" has no peak analysis — peaks come from the server analysis (POST /api/ai/audio-analysis, mode 'peaks' or 'both'); use grid 'beats' or analyse the asset first.`);
+                            break;
+                        }
+                        const minStrength = Math.max(0, Math.min(1, cmd.payload.minStrength ?? 0));
+                        gridMs = analysis.peaks.filter((p) => p.strength >= minStrength).map((p) => p.ms).sort((a, b) => a - b);
+                    }
                     // Filter visual clips: those participating in the cut.
                     const allVisualClips = snapshot.timelineClips.filter((c) => c.mediaType === 'video' || c.mediaType === 'image');
                     const targetVisuals = syncClipsArg === 'all'
@@ -326,11 +336,11 @@ export function executeAiCommands(commands, snapshot, actions) {
                     const audioEnd = audioClip.startMs + audioClip.durationMs;
                     // Beats from analysis are offsets from start of source audio.
                     // sourceOffsetMs accounts for trims at audio's head.
-                    const beatsAbsolute = analysis.beats
+                    const beatsAbsolute = gridMs
                         .map((bMs) => audioStart + (bMs - audioClip.sourceOffsetMs))
                         .filter((t) => t >= audioStart && t <= audioEnd);
                     if (beatsAbsolute.length < beatInterval + 1) {
-                        errors.push(`CUT_TO_BEAT: not enough beats (${beatsAbsolute.length}) for beatInterval ${beatInterval}`);
+                        errors.push(`CUT_TO_BEAT: not enough ${grid} (${beatsAbsolute.length}) for beatInterval ${beatInterval}`);
                         break;
                     }
                     // Cut points: every Nth beat, including audio start (first beat) and end.
@@ -338,6 +348,10 @@ export function executeAiCommands(commands, snapshot, actions) {
                     for (let i = 0; i < beatsAbsolute.length; i += beatInterval) {
                         cutPoints.push(beatsAbsolute[i]);
                     }
+                    // Pictures from the first note: the first beat is often a few ms in
+                    // (60 ms on a MiniMax track), which left a black flash at the start.
+                    if (mode === 'redistribute')
+                        cutPoints[0] = audioStart;
                     // Always end at audio end so the last segment fills the music.
                     if (cutPoints[cutPoints.length - 1] < audioEnd - 50)
                         cutPoints.push(audioEnd);
@@ -356,69 +370,70 @@ export function executeAiCommands(commands, snapshot, actions) {
                         const newCanvasObjects = [];
                         const trackId = targetVisuals[0].trackId;
                         const now = Date.now();
-                        // Walk the tape with a moving cursor; for each cut interval,
-                        // peel off a segment of the tape equal to (next - current).
+                        // Every cut interval gets exactly one piece of its full length, so
+                        // nothing is left black between two cuts or before the end of the
+                        // music. Walking the tape with a cursor:
+                        //  - the rest of the current segment covers the interval → take it;
+                        //  - it covers at least MIN_RATE of it → take it slowed down a little
+                        //    (a still image just stays on screen longer);
+                        //  - it is shorter → drop that leftover and try the next segment;
+                        //  - the tape runs out → start again from the first segment.
+                        // If no segment can fill an interval even after a full loop, the
+                        // longest one is slowed down as far as the player allows.
+                        const MIN_RATE = 0.8;
+                        const EPS = 1;
                         let tapeIdx = 0;
-                        let tapeCursorInSegment = 0;
+                        let cursor = 0;
+                        const advance = () => { tapeIdx = (tapeIdx + 1) % tape.length; cursor = 0; };
+                        const pick = (segmentMs) => {
+                            for (let tries = 0; tries <= tape.length; tries++) {
+                                const seg = tape[tapeIdx];
+                                const avail = seg.durationMs - cursor;
+                                const still = seg.sourceClip.mediaType === 'image';
+                                if (avail >= segmentMs - EPS || (avail > EPS && (still || avail >= segmentMs * MIN_RATE))) {
+                                    const used = Math.min(avail, segmentMs);
+                                    const piece = {
+                                        seg,
+                                        offsetMs: seg.offsetMs + cursor,
+                                        rate: still || used >= segmentMs - EPS ? undefined : used / segmentMs,
+                                    };
+                                    cursor += used;
+                                    if (cursor >= seg.durationMs - EPS)
+                                        advance();
+                                    return piece;
+                                }
+                                advance();
+                            }
+                            const seg = tape.reduce((a, b) => (b.durationMs > a.durationMs ? b : a));
+                            tapeIdx = tape.indexOf(seg);
+                            advance();
+                            return { seg, offsetMs: seg.offsetMs, rate: Math.max(0.25, seg.durationMs / segmentMs) };
+                        };
                         for (let i = 0; i < cutPoints.length - 1; i++) {
                             const segmentMs = cutPoints[i + 1] - cutPoints[i];
                             if (segmentMs <= 0)
                                 continue;
-                            // The piece we lay down references the current tape segment,
-                            // possibly clipping its tail if the tape segment is shorter
-                            // than the beat interval (then we step to next tape segment).
-                            let remaining = segmentMs;
-                            let placed = false;
-                            while (remaining > 0 && tapeIdx < tape.length) {
-                                const seg = tape[tapeIdx];
-                                const available = seg.durationMs - tapeCursorInSegment;
-                                if (available <= 0) {
-                                    tapeIdx++;
-                                    tapeCursorInSegment = 0;
-                                    continue;
-                                }
-                                const take = Math.min(available, remaining);
-                                if (!placed) {
-                                    // First piece of this beat segment — emit a clip starting at cutPoints[i].
-                                    const newId = `clip-cb-${now}-${i}`;
-                                    newClips.push({
-                                        id: newId,
-                                        assetId: seg.sourceClip.assetId,
-                                        name: seg.sourceClip.name,
-                                        url: seg.sourceClip.url,
-                                        startMs: cutPoints[i],
-                                        durationMs: take,
-                                        originalDurationMs: seg.sourceClip.originalDurationMs,
-                                        sourceOffsetMs: seg.offsetMs + tapeCursorInSegment,
-                                        mediaType: seg.sourceClip.mediaType,
-                                        trackId,
-                                    });
-                                    newCanvasObjects.push({
-                                        id: `obj-${newId}`,
-                                        clipId: newId,
-                                        x: 0, y: 0, w: 1, h: 1,
-                                        rotation: 0,
-                                    });
-                                    placed = true;
-                                }
-                                else {
-                                    // Subsequent piece within the same beat — extend the
-                                    // previous clip's duration if and only if it's the same
-                                    // source asset; otherwise start a fresh clip nested in
-                                    // the same beat. For simplicity, we only support a
-                                    // single asset per beat segment (truncate trailing tape
-                                    // and snap to next beat).
-                                    break;
-                                }
-                                tapeCursorInSegment += take;
-                                remaining -= take;
-                                if (tapeCursorInSegment >= seg.durationMs) {
-                                    tapeIdx++;
-                                    tapeCursorInSegment = 0;
-                                }
-                            }
-                            if (tapeIdx >= tape.length)
-                                break; // tape exhausted, stop emitting
+                            const { seg, offsetMs, rate } = pick(segmentMs);
+                            const newId = `clip-cb-${now}-${i}`;
+                            newClips.push({
+                                id: newId,
+                                assetId: seg.sourceClip.assetId,
+                                name: seg.sourceClip.name,
+                                url: seg.sourceClip.url,
+                                startMs: cutPoints[i],
+                                durationMs: segmentMs,
+                                originalDurationMs: seg.sourceClip.originalDurationMs,
+                                sourceOffsetMs: offsetMs,
+                                mediaType: seg.sourceClip.mediaType,
+                                trackId,
+                                ...(rate !== undefined ? { playbackRate: rate } : {}),
+                            });
+                            newCanvasObjects.push({
+                                id: `obj-${newId}`,
+                                clipId: newId,
+                                x: 0, y: 0, w: 1, h: 1,
+                                rotation: 0,
+                            });
                         }
                         // Preserve audio + non-target clips intact.
                         const preservedClips = snapshot.timelineClips.filter((c) => c.mediaType === 'audio' || c.mediaType === 'color' ||

package/dist/store.d.ts

@@ -262,4 +262,5 @@ export interface EditorState {
     reset: () => void;
 }
 export type EditorStore = StoreApi<EditorState>;
+export declare function uniqueId(prefix: string): string;
 export declare function createEditorStore(deps?: EditorStoreDeps): EditorStore;

package/dist/store.js

@@ -99,6 +99,17 @@ function timelineContentMs(clips) {
 function clamp(v, min, max) {
     return Math.max(min, Math.min(max, v));
 }
+/**
+ * Timestamp-based ids collide when several clips are created in the same
+ * millisecond — which never happens from the UI but always does in a
+ * headless batch (vivid-mcp: ADD_CLIP × 3 in one call). The sequence suffix
+ * keeps them unique; the `clip-`/`obj-` prefixes stay as before.
+ */
+let idSeq = 0;
+export function uniqueId(prefix) {
+    idSeq = (idSeq + 1) % 1_000_000;
+    return `${prefix}-${Date.now()}-${idSeq.toString(36)}`;
+}
 export function createEditorStore(deps = {}) {
     return createStore()((set, get) => ({
         // ── Initial state ──
@@ -247,7 +258,7 @@ export function createEditorStore(deps = {}) {
             const availableMs = MAX_TIMELINE_MS - startMs;
             if (availableMs <= 0)
                 return;
-            const clipId = `clip-${Date.now()}`;
+            const clipId = uniqueId('clip');
             const clip = {
                 id: clipId,
                 assetId: asset.id,
@@ -465,7 +476,7 @@ export function createEditorStore(deps = {}) {
             if (!clip)
                 return;
             state.pushUndo();
-            const newId = `clip-${Date.now()}`;
+            const newId = uniqueId('clip');
             const newStart = clip.startMs + clip.durationMs; // place right after original
             const newClip = {
                 ...clip,
@@ -475,7 +486,7 @@ export function createEditorStore(deps = {}) {
             // Also duplicate canvas object if it's a visual clip
             const canvasObj = state.canvasObjects.find((o) => o.clipId === id);
             const newObjs = canvasObj
-                ? [...state.canvasObjects, { ...canvasObj, id: `obj-${Date.now()}`, clipId: newId }]
+                ? [...state.canvasObjects, { ...canvasObj, id: uniqueId('obj'), clipId: newId }]
                 : state.canvasObjects;
             set({
                 timelineClips: [...state.timelineClips, newClip],
@@ -541,7 +552,7 @@ export function createEditorStore(deps = {}) {
             }
             if (startMs === null || targetTrackId === null)
                 return;
-            const clipId = `clip-color-${Date.now()}`;
+            const clipId = uniqueId('clip-color');
             const clip = {
                 id: clipId,
                 assetId: `color-${color.replace('#', '')}`,
@@ -598,7 +609,7 @@ export function createEditorStore(deps = {}) {
             // Ensure an audio track exists for this video's track
             const audioTrackId = clip.trackId.replace('V', 'A');
             state.ensureAudioTrack(clip.trackId);
-            const audioClipId = `audio-detach-${Date.now()}`;
+            const audioClipId = uniqueId('audio-detach');
             const audioClip = {
                 id: audioClipId,
                 assetId: clip.assetId,

package/dist/types/beat.d.ts

@@ -10,7 +10,20 @@ export interface BeatAnalysis {
     /** Downbeats — every 4th beat (1, 5, 9, ...) — in ms */
     downbeats: number[];
     /** Source of the BPM value */
-    source: 'detected' | 'filename-hint' | 'detected+hint-validated' | 'envelope-fallback';
+    source: 'detected' | 'filename-hint' | 'detected+hint-validated' | 'envelope-fallback' | 'server';
     /** Duration of the analyzed audio, in ms */
     analyzedDurationMs: number;
+    /**
+     * Transient peaks (drum hits, accents, word starts) from the server-side
+     * analysis (POST /api/ai/audio-analysis), ms from the start of the source
+     * audio. Strength is 0–1 relative to the loudest onset. Optional: the
+     * browser-only analyzer does not compute them.
+     */
+    peaks?: AudioPeak[];
+    /** 0–1 peakedness of the tempo autocorrelation (server analysis only). */
+    bpmConfidence?: number;
+}
+export interface AudioPeak {
+    ms: number;
+    strength: number;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vivid-editor-core",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "VIVID video editor core: project file schema, editor store and AI command orchestrator — shared by the vividai.tv editor and vivid-mcp.",
   "type": "module",
   "main": "dist/index.js",