vidspotai-shared 1.0.67 → 1.0.69-dev.0

package/lib/services/agent/regenCore.js

@@ -0,0 +1,487 @@
+"use strict";
+/**
+ * Stage 7 (Session H) — shared core for single-target regen.
+ *
+ * Mirrors `runRegenTarget` in apps/firebase-functions/src/controllers/
+ * agent.controller.ts so both the synchronous Function endpoints AND the
+ * new AGENT_REGEN_JOBS worker call the same code path. Until Stage 8
+ * deletes the in-controller copy, this file is the authoritative
+ * implementation; the controller copy stays for backwards-compatibility
+ * with the existing sync endpoints (they keep working unchanged).
+ *
+ * Pure logic — never reads from / writes to Firestore. The caller threads
+ * project state (the bulk executor reuses one in-memory snapshot across
+ * targets) and persists once. Per-target cache mints go via `ctx.cacheMints`
+ * which the caller writes back as `providerTaskCache.<clipId>` dot-paths.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeRegenTarget = executeRegenTarget;
+const node_crypto_1 = require("node:crypto");
+const modelRouter_1 = require("./modelRouter");
+const executor_1 = require("./executor");
+const taskPoller_1 = require("./taskPoller");
+const planner_1 = require("./planner");
+const llmCallerRegistry_1 = require("./llmCallerRegistry");
+const recorder_1 = require("./eval/recorder");
+const musicSelect_1 = require("./musicSelect");
+const planToProject_1 = require("../editor/planToProject");
+const uploadAudioBuffer_1 = require("../gcp/uploadAudioBuffer");
+const pricing_1 = require("../credits/pricing");
+const logger_1 = require("../../utils/logger");
+const isoNow = () => new Date().toISOString();
+/**
+ * Run a single regen target against an in-memory project snapshot.
+ * Behavioural parity with each per-component HTTP handler in the
+ * controller so the bulk executor and the per-component path produce
+ * identical projectJson / plan mutations.
+ */
+async function executeRegenTarget(target, project, ctx, options = {}) {
+    const { component, sceneIndex } = target;
+    const currentProjectJson = project.projectJson;
+    const plan = project.plan;
+    if (!plan)
+        return { ok: false, status: 422, error: "plan required" };
+    if (!currentProjectJson) {
+        return {
+            ok: false,
+            status: 409,
+            error: "project not yet rendered — run /execute first",
+        };
+    }
+    const scene = typeof sceneIndex === "number"
+        ? plan.scenes.find((s) => s.sceneIndex === sceneIndex)
+        : undefined;
+    if (component !== "music" && !scene) {
+        return { ok: false, status: 404, error: "scene not in plan" };
+    }
+    const startRunTrace = () => {
+        const runId = (0, node_crypto_1.randomUUID)();
+        const trace = (0, recorder_1.startTrace)({
+            agentRunId: runId,
+            rngSeed: runId,
+            modelRouting: Object.fromEntries(Object.entries(modelRouter_1.DEFAULT_MODEL_ROUTING.primary).map(([k, v]) => [
+                k,
+                `${v.provider}/${v.modelId}`,
+            ])),
+        });
+        return { runId, trace };
+    };
+    switch (component) {
+        case "visual": {
+            const estimate = (0, pricing_1.estimateRegenCost)({ component: "visual", scene });
+            const { runId, trace } = startRunTrace();
+            const executor = new executor_1.Executor({
+                runTool: recorder_1.runToolRecorded,
+                onTaskMint: async (clipId, entry) => {
+                    ctx.cacheMints[clipId] = entry;
+                },
+            });
+            const slicedPlan = { ...plan, scenes: [scene] };
+            const outcomes = await executor.run(slicedPlan, {
+                userId: ctx.userId,
+                agentRunId: runId,
+            });
+            const outcome = outcomes[0];
+            if (!outcome) {
+                await (0, recorder_1.endTrace)(runId);
+                return { ok: false, status: 500, error: "executor returned no outcome" };
+            }
+            if (!outcome.ok) {
+                await (0, recorder_1.endTrace)(runId);
+                return {
+                    ok: false,
+                    status: 502,
+                    error: outcome.error?.message ?? "scene execution failed",
+                    code: outcome.error?.code,
+                };
+            }
+            if (outcome.result?.pendingTaskId && outcome.result?.pendingModelKey) {
+                const poller = new taskPoller_1.TaskPoller();
+                const pollResults = await poller.pollAll([
+                    {
+                        ref: String(sceneIndex),
+                        task: outcome.result.pendingTaskId,
+                        modelKey: outcome.result.pendingModelKey,
+                        outputFilename: `${ctx.projectId}-scene${sceneIndex}-regen`,
+                    },
+                ]);
+                const r = pollResults[0];
+                if (r.status === "completed" && r.videoUrl) {
+                    outcome.result.visualUrl = r.videoUrl;
+                }
+                else {
+                    await (0, recorder_1.endTrace)(runId);
+                    return {
+                        ok: false,
+                        status: 502,
+                        error: r.errorMessage ?? "scene regeneration timed out",
+                        code: r.status === "timed-out" ? "POLL_TIMED_OUT" : "PROVIDER_FAILED",
+                    };
+                }
+            }
+            const visualUrl = outcome.result?.visualUrl;
+            if (!visualUrl) {
+                await (0, recorder_1.endTrace)(runId);
+                return { ok: false, status: 502, error: "no visualUrl after regeneration" };
+            }
+            const clipId = `clip-${sceneIndex}`;
+            const existing = currentProjectJson.clips.find((c) => c.id === clipId);
+            if (!existing) {
+                await (0, recorder_1.endTrace)(runId);
+                return { ok: false, status: 404, error: `clip ${clipId} not in projectJson` };
+            }
+            const updatedClips = currentProjectJson.clips.map((c) => {
+                if (c.id !== clipId)
+                    return c;
+                const patched = {
+                    ...c,
+                    src: visualUrl,
+                    provenance: {
+                        ...(c.provenance ?? {}),
+                        regeneratedAt: isoNow(),
+                    },
+                };
+                return patched;
+            });
+            const baseProject = {
+                ...currentProjectJson,
+                clips: updatedClips,
+                updatedAt: isoNow(),
+            };
+            const nextProjectJson = (0, pricing_1.appendRegenHistory)(baseProject, {
+                ts: isoNow(),
+                action: "regenerate",
+                component: "visual",
+                clipId,
+                sceneIndex,
+                modelKey: scene.preferredModel,
+                estimatedCreditsSpent: estimate.amountCredits,
+                estimatedUsd: estimate.amountUsd,
+            });
+            await (0, recorder_1.endTrace)(runId);
+            return {
+                ok: true,
+                nextProjectJson,
+                runId,
+                toolCalls: trace.toolCalls,
+                cost: estimate,
+                details: { clipId, visualUrl },
+            };
+        }
+        case "voiceover": {
+            const newLine = (options.overrideLine ?? scene.voiceoverLine ?? "").trim();
+            if (!newLine) {
+                return {
+                    ok: false,
+                    status: 422,
+                    error: "no voiceoverLine on scene and none provided",
+                };
+            }
+            const estimate = (0, pricing_1.estimateRegenCost)({
+                component: "voiceover",
+                scene,
+                voiceoverLine: newLine,
+                captionsAlsoChange: options.captionsAlsoChange,
+            });
+            const { runId, trace } = startRunTrace();
+            const voInput = { text: newLine };
+            if (plan.voiceId)
+                voInput.voiceId = plan.voiceId;
+            const voOutcome = await (0, recorder_1.runToolRecorded)("generate_voiceover", voInput, {
+                userId: ctx.userId,
+                agentRunId: runId,
+                idempotencyKey: `${runId}:vo:${sceneIndex}`,
+            });
+            if (!voOutcome.ok) {
+                await (0, recorder_1.endTrace)(runId);
+                return {
+                    ok: false,
+                    status: 502,
+                    error: voOutcome.error?.message ?? "voiceover generation failed",
+                    code: voOutcome.error?.code,
+                };
+            }
+            const buffer = Buffer.from(voOutcome.output.audioBase64, "base64");
+            const destination = `agent-projects/${ctx.projectId}/vo/scene-${sceneIndex}-${Date.now()}.mp3`;
+            const voUrl = await (0, uploadAudioBuffer_1.uploadAudioBufferToGcs)(buffer, destination);
+            const clipId = `scene-${sceneIndex}-vo`;
+            const existingVo = currentProjectJson.clips.find((c) => c.id === clipId);
+            const overrideProvided = !!options.overrideLine;
+            const updatedClips = existingVo
+                ? currentProjectJson.clips.map((c) => {
+                    if (c.id !== clipId)
+                        return c;
+                    const patched = {
+                        ...c,
+                        src: voUrl,
+                        ...(overrideProvided ? { name: `VO ${sceneIndex + 1}` } : {}),
+                        provenance: {
+                            ...(c.provenance ?? {}),
+                            prompt: newLine,
+                            source: "elevenlabs",
+                            regeneratedAt: isoNow(),
+                        },
+                        metadata: {
+                            ...(c.metadata ?? {}),
+                            line: newLine,
+                        },
+                    };
+                    return patched;
+                })
+                : currentProjectJson.clips;
+            const baseProject = {
+                ...currentProjectJson,
+                clips: updatedClips,
+                updatedAt: isoNow(),
+            };
+            const nextProjectJson = (0, pricing_1.appendRegenHistory)(baseProject, {
+                ts: isoNow(),
+                action: "regenerate",
+                component: "voiceover",
+                clipId,
+                sceneIndex,
+                estimatedCreditsSpent: estimate.amountCredits,
+                estimatedUsd: estimate.amountUsd,
+            });
+            let nextPlan;
+            if (overrideProvided &&
+                (options.overrideLine ?? "").trim() !== (scene.voiceoverLine ?? "").trim()) {
+                nextPlan = {
+                    ...plan,
+                    scenes: plan.scenes.map((s) => s.sceneIndex === sceneIndex ? { ...s, voiceoverLine: newLine } : s),
+                };
+            }
+            await (0, recorder_1.endTrace)(runId);
+            return {
+                ok: true,
+                nextPlan,
+                nextProjectJson,
+                runId,
+                toolCalls: trace.toolCalls,
+                cost: estimate,
+                details: { clipId, sceneIndex, voiceoverUrl: voUrl },
+            };
+        }
+        case "captions": {
+            const estimate = (0, pricing_1.estimateRegenCost)({ component: "captions" });
+            const clipId = `scene-${sceneIndex}-caption`;
+            const voClipId = `scene-${sceneIndex}-vo`;
+            const voClip = currentProjectJson.clips.find((c) => c.id === voClipId);
+            const audioUrl = voClip?.src;
+            if (!audioUrl) {
+                return {
+                    ok: false,
+                    status: 422,
+                    error: `no voiceover clip ${voClipId} on project — regenerate voiceover first`,
+                };
+            }
+            const { runId, trace } = startRunTrace();
+            const asrOutcome = await (0, recorder_1.runToolRecorded)("generate_captions", { audioUrl, wordTimestamps: true }, {
+                userId: ctx.userId,
+                agentRunId: runId,
+                idempotencyKey: `${runId}:captions:${sceneIndex}`,
+            });
+            if (!asrOutcome.ok) {
+                await (0, recorder_1.endTrace)(runId);
+                return {
+                    ok: false,
+                    status: 502,
+                    error: asrOutcome.error?.message ?? "caption generation failed",
+                    code: asrOutcome.error?.code,
+                };
+            }
+            const words = asrOutcome.output.words.map((w) => ({
+                word: w.text,
+                start: w.start,
+                end: w.end,
+            }));
+            const text = asrOutcome.output.text;
+            const existingCaption = currentProjectJson.clips.find((c) => c.id === clipId);
+            if (!existingCaption) {
+                await (0, recorder_1.endTrace)(runId);
+                return {
+                    ok: false,
+                    status: 404,
+                    error: `no caption clip ${clipId} on project — re-run /execute to build the captions track`,
+                };
+            }
+            const updatedClips = currentProjectJson.clips.map((c) => {
+                if (c.id !== clipId)
+                    return c;
+                const patched = {
+                    ...c,
+                    text,
+                    words,
+                    metadata: {
+                        ...(c.metadata ?? {}),
+                        regeneratedAt: isoNow(),
+                    },
+                };
+                return patched;
+            });
+            const baseProject = {
+                ...currentProjectJson,
+                clips: updatedClips,
+                updatedAt: isoNow(),
+            };
+            const nextProjectJson = (0, pricing_1.appendRegenHistory)(baseProject, {
+                ts: isoNow(),
+                action: "regenerate",
+                component: "captions",
+                clipId,
+                sceneIndex,
+                estimatedCreditsSpent: estimate.amountCredits,
+                estimatedUsd: estimate.amountUsd,
+            });
+            await (0, recorder_1.endTrace)(runId);
+            return {
+                ok: true,
+                nextProjectJson,
+                runId,
+                toolCalls: trace.toolCalls,
+                cost: estimate,
+                details: { clipId, sceneIndex, wordCount: words.length },
+            };
+        }
+        case "overlays": {
+            if (!project.brief) {
+                return { ok: false, status: 422, error: "brief required" };
+            }
+            const estimate = (0, pricing_1.estimateRegenCost)({ component: "overlays" });
+            const visualClip = currentProjectJson.clips.find((c) => c.id === `scene-${sceneIndex}-visual`);
+            const offsetMs = visualClip?.display?.from ?? 0;
+            const sceneDurationMs = visualClip?.display
+                ? visualClip.display.to - visualClip.display.from
+                : scene.durationMs;
+            const isClosing = sceneIndex === plan.scenes.length - 1;
+            const { runId, trace } = startRunTrace();
+            const planner = new planner_1.Planner({ llm: (0, llmCallerRegistry_1.getLlmCaller)() });
+            let newOverlays;
+            try {
+                newOverlays = await planner.regenerateOverlaysForScene(scene, project.brief, {
+                    persona: plan.persona,
+                    bible: plan.bible,
+                    isClosingScene: isClosing,
+                    instruction: options.instruction,
+                });
+            }
+            catch (err) {
+                logger_1.logger.error("regenCore: overlays regen failed", {
+                    runId,
+                    projectId: ctx.projectId,
+                    sceneIndex,
+                    err: err instanceof Error ? err.stack ?? err.message : String(err),
+                });
+                await (0, recorder_1.endTrace)(runId);
+                return { ok: false, status: 502, error: err.message };
+            }
+            const replacementClips = (0, planToProject_1.buildOverlayClipsForScene)(sceneIndex, offsetMs, sceneDurationMs, newOverlays, currentProjectJson.size.height);
+            const overlayPrefix = `scene-${sceneIndex}-overlay-`;
+            const survivors = currentProjectJson.clips.filter((c) => !c.id.startsWith(overlayPrefix));
+            const updatedClips = [...survivors, ...replacementClips];
+            const baseProject = {
+                ...currentProjectJson,
+                clips: updatedClips,
+                updatedAt: isoNow(),
+            };
+            const nextProjectJson = (0, pricing_1.appendRegenHistory)(baseProject, {
+                ts: isoNow(),
+                action: "regenerate",
+                component: "overlays",
+                sceneIndex,
+                estimatedCreditsSpent: estimate.amountCredits,
+                estimatedUsd: estimate.amountUsd,
+            });
+            const nextPlan = {
+                ...plan,
+                scenes: plan.scenes.map((s) => s.sceneIndex === sceneIndex ? { ...s, overlays: newOverlays } : s),
+            };
+            await (0, recorder_1.endTrace)(runId);
+            return {
+                ok: true,
+                nextPlan,
+                nextProjectJson,
+                runId,
+                toolCalls: trace.toolCalls,
+                cost: estimate,
+                details: { sceneIndex, overlayCount: newOverlays.length },
+            };
+        }
+        case "music": {
+            const estimate = (0, pricing_1.estimateRegenCost)({ component: "music" });
+            const planForReselect = {
+                ...plan,
+                music: plan.music
+                    ? {
+                        ...plan.music,
+                        url: undefined,
+                        durationSec: undefined,
+                        source: undefined,
+                        attribution: undefined,
+                        onsets: undefined,
+                    }
+                    : plan.music,
+            };
+            let refreshedPlan;
+            try {
+                refreshedPlan = await (0, musicSelect_1.selectMusicForPlan)(planForReselect);
+            }
+            catch (err) {
+                logger_1.logger.error("regenCore: music regen failed", {
+                    projectId: ctx.projectId,
+                    sceneIndex,
+                    err: err instanceof Error ? err.stack ?? err.message : String(err),
+                });
+                return { ok: false, status: 502, error: err.message };
+            }
+            const newMusicUrl = refreshedPlan.music?.url;
+            const existingMusic = currentProjectJson.clips.find((c) => c.id === "music");
+            let updatedClips;
+            if (newMusicUrl && existingMusic) {
+                updatedClips = currentProjectJson.clips.map((c) => {
+                    if (c.id !== "music")
+                        return c;
+                    const patched = {
+                        ...c,
+                        src: newMusicUrl,
+                        provenance: {
+                            ...(c.provenance ?? {}),
+                            source: refreshedPlan.music?.source ?? "music",
+                            regeneratedAt: isoNow(),
+                        },
+                    };
+                    return patched;
+                });
+            }
+            else if (!newMusicUrl && existingMusic) {
+                updatedClips = currentProjectJson.clips.filter((c) => c.id !== "music");
+            }
+            else {
+                updatedClips = currentProjectJson.clips;
+            }
+            const baseProject = {
+                ...currentProjectJson,
+                clips: updatedClips,
+                updatedAt: isoNow(),
+            };
+            const nextProjectJson = (0, pricing_1.appendRegenHistory)(baseProject, {
+                ts: isoNow(),
+                action: "regenerate",
+                component: "music",
+                estimatedCreditsSpent: estimate.amountCredits,
+                estimatedUsd: estimate.amountUsd,
+            });
+            return {
+                ok: true,
+                nextPlan: refreshedPlan,
+                nextProjectJson,
+                toolCalls: [],
+                cost: estimate,
+                details: {
+                    musicUrl: newMusicUrl ?? null,
+                    source: refreshedPlan.music?.source ?? null,
+                },
+            };
+        }
+    }
+}

package/lib/services/agent/runHelpers.d.ts

@@ -0,0 +1,44 @@
+import { VideoPlan } from "../../schemas/videoPlan.schema";
+import { SceneRenderOutcome } from "../editor/planToProject";
+import { SceneOutcome } from "./executor";
+/**
+ * Map an Executor SceneOutcome to the SceneRenderOutcome shape planToProject
+ * expects. Reads `voiceoverUrl` (set by resolveVoiceoverUrl before this
+ * runs) — not the base64, which would blow Firestore's 1 MiB doc cap.
+ *
+ * Lifted from agent.controller.ts so both the slimmed controller and the
+ * Stage 1 agentRunWorker can import it.
+ */
+export declare function outcomeToRenderOutcome(outcome: SceneOutcome): SceneRenderOutcome | undefined;
+/**
+ * Stage 6.e — voiceover persistence. If the scene's executor outcome has
+ * base64 audio (ElevenLabs / generate_voiceover tool output), upload it to
+ * Firebase Storage and mutate the outcome in place so downstream
+ * planToProject calls see a real URL. Idempotent: a second call with the
+ * same outcome (already URL-resolved) is a no-op. Cleans the base64 off
+ * the outcome after upload so the Firestore write doesn't ship the buffer.
+ *
+ * C14: critical for keeping the agentRuns doc under the 1 MiB limit.
+ */
+export declare function resolveVoiceoverUrl(outcome: SceneOutcome, projectId: string): Promise<void>;
+/**
+ * Stage 3 — write the initial Project shell after the planner approves a
+ * plan. All scene clips exist with placeholder URLs so editor + drawer
+ * surfaces light up immediately; the executor will fill in real URLs
+ * scene-by-scene via the incremental write below.
+ *
+ * C23: critical for keeping the editor / drawer from sitting on an empty
+ * timeline for 20+ minutes while the worker runs.
+ */
+export declare function writeAgentProjectShell(projectId: string, plan: VideoPlan, agentRunId: string): Promise<void>;
+/**
+ * Stage 3 — write a partially-completed Project after each scene
+ * finishes. Rebuilds the full Project from plan + accumulated outcomes
+ * so far (planToProject is pure + cheap), then writes once. Best-effort:
+ * errors logged but not propagated so a Firestore blip doesn't crash
+ * the worker mid-run.
+ *
+ * C24 / C25: per-scene incremental update + soft-fail behavior.
+ */
+export declare function writeAgentProjectIncremental(projectId: string, plan: VideoPlan, outcomesSoFar: Record<number, SceneOutcome>, agentRunId: string): Promise<void>;
+//# sourceMappingURL=runHelpers.d.ts.map

package/lib/services/agent/runHelpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runHelpers.d.ts","sourceRoot":"","sources":["../../../src/services/agent/runHelpers.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAE3D,OAAO,EAAiB,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAI5E,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AA6C1C;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,YAAY,GACpB,kBAAkB,GAAG,SAAS,CAkBhC;AAED;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAoCf;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,EACf,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED;;;;;;;;GAQG;AACH,wBAAsB,4BAA4B,CAChD,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,EACf,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,EAC3C,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CAuBf"}