vidspotai-shared 1.0.84 → 1.0.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/globals/aiModels/providers/runway.d.ts.map +1 -1
- package/lib/globals/aiModels/providers/runway.js +12 -7
- package/lib/globals/types.d.ts +10 -0
- package/lib/globals/types.d.ts.map +1 -1
- package/lib/globals/types.js +11 -0
- package/lib/models/agent.model.d.ts +129 -1
- package/lib/models/agent.model.d.ts.map +1 -1
- package/lib/schemas/agentRunJob.schema.d.ts +21 -0
- package/lib/schemas/agentRunJob.schema.d.ts.map +1 -1
- package/lib/schemas/agentRunJob.schema.js +35 -3
- package/lib/services/agent/executor/core.d.ts.map +1 -1
- package/lib/services/agent/executor/core.js +21 -1
- package/lib/services/agent/executor/types.d.ts +10 -0
- package/lib/services/agent/executor/types.d.ts.map +1 -1
- package/lib/services/agent/index.d.ts +3 -0
- package/lib/services/agent/index.d.ts.map +1 -1
- package/lib/services/agent/index.js +3 -0
- package/lib/services/agent/providerFallback/chains.d.ts +2 -0
- package/lib/services/agent/providerFallback/chains.d.ts.map +1 -1
- package/lib/services/agent/providerFallback/chains.js +1 -0
- package/lib/services/agent/recovery/degradeLadder.d.ts +74 -0
- package/lib/services/agent/recovery/degradeLadder.d.ts.map +1 -0
- package/lib/services/agent/recovery/degradeLadder.js +133 -0
- package/lib/services/agent/repair/engine.d.ts +38 -0
- package/lib/services/agent/repair/engine.d.ts.map +1 -0
- package/lib/services/agent/repair/engine.js +175 -0
- package/lib/services/agent/repair/index.d.ts +20 -0
- package/lib/services/agent/repair/index.d.ts.map +1 -0
- package/lib/services/agent/repair/index.js +49 -0
- package/lib/services/agent/repair/strategies/llmRepair.strategy.d.ts +6 -0
- package/lib/services/agent/repair/strategies/llmRepair.strategy.d.ts.map +1 -0
- package/lib/services/agent/repair/strategies/llmRepair.strategy.js +210 -0
- package/lib/services/agent/repair/strategies/paramRepair.strategy.d.ts +3 -0
- package/lib/services/agent/repair/strategies/paramRepair.strategy.d.ts.map +1 -0
- package/lib/services/agent/repair/strategies/paramRepair.strategy.js +151 -0
- package/lib/services/agent/repair/types.d.ts +92 -0
- package/lib/services/agent/repair/types.d.ts.map +1 -0
- package/lib/services/agent/repair/types.js +2 -0
- package/lib/services/agent/runPlanning.d.ts +96 -0
- package/lib/services/agent/runPlanning.d.ts.map +1 -0
- package/lib/services/agent/runPlanning.js +233 -0
- package/lib/services/aiGen/providers/runway/types.d.ts +14 -0
- package/lib/services/aiGen/providers/runway/types.d.ts.map +1 -1
- package/lib/services/aiGen/providers/runway/types.js +22 -1
- package/lib/services/firestore.service.d.ts +7 -1
- package/lib/services/firestore.service.d.ts.map +1 -1
- package/lib/services/firestore.service.js +8 -0
- package/package.json +1 -1
- package/lib/services/agent/executor.d.ts +0 -141
- package/lib/services/agent/executor.d.ts.map +0 -1
- package/lib/services/agent/executor.js +0 -561
- package/lib/services/agent/llmCallerGateway.d.ts +0 -61
- package/lib/services/agent/llmCallerGateway.d.ts.map +0 -1
- package/lib/services/agent/llmCallerGateway.js +0 -368
|
@@ -1,561 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Executor = void 0;
|
|
4
|
-
exports.classifySceneFailure = classifySceneFailure;
|
|
5
|
-
const voices_1 = require("../../globals/ttsModels/voices");
|
|
6
|
-
const logger_1 = require("../../utils/logger");
|
|
7
|
-
const referenceImageRenderer_1 = require("./referenceImageRenderer");
|
|
8
|
-
const toolRegistry_1 = require("./toolRegistry");
|
|
9
|
-
const ttsDuration_1 = require("./ttsDuration");
|
|
10
|
-
const chains_1 = require("./providerFallback/chains");
|
|
11
|
-
const providerTaskCache_1 = require("./providerTaskCache");
|
|
12
|
-
/**
|
|
13
|
-
* AG-37: planners (LLMs) keep inventing voice IDs like "female-young-adult-en"
|
|
14
|
-
* or "narrator-1" that look reasonable but don't exist in any catalog. When
|
|
15
|
-
* those flow into ElevenLabs the API returns a 400 and the silent-fail VO
|
|
16
|
-
* outcome path drops the VO entirely — final video ships muted. Strip
|
|
17
|
-
* unknown IDs here so the provider falls back to its default voice instead.
|
|
18
|
-
*/
|
|
19
|
-
const ELEVENLABS_VOICE_IDS = new Set(voices_1.ELEVENLABS_VOICES.map((v) => v.id));
|
|
20
|
-
function sanitizeVoiceId(voiceId) {
|
|
21
|
-
if (!voiceId)
|
|
22
|
-
return undefined;
|
|
23
|
-
if (ELEVENLABS_VOICE_IDS.has(voiceId))
|
|
24
|
-
return voiceId;
|
|
25
|
-
// Heuristic for non-ElevenLabs providers (openai = "alloy"/"echo"/..., minimax = "male-qn-qingse").
|
|
26
|
-
// Only strip strings that look like LLM-invented descriptive labels (contain a hyphen + recognisable word).
|
|
27
|
-
if (/^(female|male|narrator|presenter|voice)[-_]/i.test(voiceId)) {
|
|
28
|
-
logger_1.logger.warn("executor: stripping invalid voiceId (LLM hallucination)", { voiceId });
|
|
29
|
-
return undefined;
|
|
30
|
-
}
|
|
31
|
-
return voiceId;
|
|
32
|
-
}
|
|
33
|
-
/**
|
|
34
|
-
* AG-31: map a brief-level voice style adjective to ElevenLabs voice_settings.
|
|
35
|
-
* `style` is the exaggeration param (use sparingly above 0.5 — artifacts);
|
|
36
|
-
* `stability` is the consistency knob (low = more emotive, high = monotone);
|
|
37
|
-
* `speed` is rate (0.7-1.2). Defaults match ElevenLabs' own (style=0,
|
|
38
|
-
* stability=0.5, speed=1.0). Tuning per persona is conservative — we'd rather
|
|
39
|
-
* be slightly flat than artifact-laden.
|
|
40
|
-
*/
|
|
41
|
-
function resolveVoiceSettings(style) {
|
|
42
|
-
switch (style) {
|
|
43
|
-
case "calm":
|
|
44
|
-
return { style: 0.15, stability: 0.7, speed: 0.95 };
|
|
45
|
-
case "warm":
|
|
46
|
-
return { style: 0.25, stability: 0.6, speed: 1.0 };
|
|
47
|
-
case "casual":
|
|
48
|
-
return { style: 0.3, stability: 0.5, speed: 1.0 };
|
|
49
|
-
case "excited":
|
|
50
|
-
return { style: 0.55, stability: 0.35, speed: 1.05 };
|
|
51
|
-
case "energetic":
|
|
52
|
-
return { style: 0.6, stability: 0.35, speed: 1.08 };
|
|
53
|
-
case "happy":
|
|
54
|
-
return { style: 0.5, stability: 0.4, speed: 1.05 };
|
|
55
|
-
case "serious":
|
|
56
|
-
return { style: 0.2, stability: 0.75, speed: 0.95 };
|
|
57
|
-
case "dramatic":
|
|
58
|
-
return { style: 0.65, stability: 0.4, speed: 0.95 };
|
|
59
|
-
case "whisper":
|
|
60
|
-
return { style: 0.2, stability: 0.85, speed: 0.9 };
|
|
61
|
-
case "neutral":
|
|
62
|
-
case undefined:
|
|
63
|
-
default:
|
|
64
|
-
return {};
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* Inspect a failed scene to decide the next orchestration step.
|
|
69
|
-
*
|
|
70
|
-
* - "replan" : agent should re-plan the scene (safety / bad input)
|
|
71
|
-
* - "degrade" : try a cheaper strategy (e.g. T2/T3 chain exhausted on quota)
|
|
72
|
-
* - "fail" : terminal — surface to user (auth / chain-exhausted-unknown)
|
|
73
|
-
* - "retry" : transient — caller may re-run the scene as-is later
|
|
74
|
-
*/
|
|
75
|
-
function classifySceneFailure(outcome) {
|
|
76
|
-
if (outcome.ok)
|
|
77
|
-
return null;
|
|
78
|
-
const c = outcome.error?.classification;
|
|
79
|
-
if (!c)
|
|
80
|
-
return "fail";
|
|
81
|
-
if (outcome.error?.needsReplan)
|
|
82
|
-
return "replan"; // safety / input
|
|
83
|
-
if (c === "auth")
|
|
84
|
-
return "fail";
|
|
85
|
-
if (c === "quota")
|
|
86
|
-
return "degrade";
|
|
87
|
-
if (c === "rate_limit" || c === "transient")
|
|
88
|
-
return "retry";
|
|
89
|
-
if (c === "capability")
|
|
90
|
-
return "degrade";
|
|
91
|
-
return "fail";
|
|
92
|
-
}
|
|
93
|
-
/**
|
|
94
|
-
* OD-T7 / AG-42 — give the planner's `preferredModel` a chance to actually run
|
|
95
|
-
* when the scene duration is just slightly off from what the model accepts.
|
|
96
|
-
*
|
|
97
|
-
* Decision tree:
|
|
98
|
-
* 1. Model has no duration constraint → use planned duration
|
|
99
|
-
* 2. Planned duration already legal → use planned duration
|
|
100
|
-
* 3. Nearest legal is within snap tolerance → snap, log info
|
|
101
|
-
* 4. Outside tolerance + explicit preferredModel → log warn ("dropping
|
|
102
|
-
* planner pick — too far from any allowed value"); use planned duration
|
|
103
|
-
* anyway so the capability filter drops it and the chain fallback runs
|
|
104
|
-
* (the withFallback warn AG-42 also logs there now)
|
|
105
|
-
*
|
|
106
|
-
* We intentionally do NOT throw — throwing would crash a scene mid-run and
|
|
107
|
-
* lose all the other planner output. The graceful fallback chain already
|
|
108
|
-
* exists; this helper just keeps the planner's pick alive when the drift is
|
|
109
|
-
* small, and makes the drop visible when it isn't.
|
|
110
|
-
*/
|
|
111
|
-
function resolveDurationForPreferredModel(modelKey, plannedDurationSec, explicit, sceneIndex) {
|
|
112
|
-
const snap = (0, chains_1.snapDurationForModel)(modelKey, plannedDurationSec);
|
|
113
|
-
if (!snap) {
|
|
114
|
-
if (explicit) {
|
|
115
|
-
logger_1.logger.warn("executor: preferredModel duration outside snap tolerance — chain fallback will pick", {
|
|
116
|
-
sceneIndex,
|
|
117
|
-
modelKey,
|
|
118
|
-
plannedDurationSec,
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
return plannedDurationSec;
|
|
122
|
-
}
|
|
123
|
-
if (snap.driftSec > 0.01) {
|
|
124
|
-
logger_1.logger.info("executor: snapped duration to fit preferredModel", {
|
|
125
|
-
sceneIndex,
|
|
126
|
-
modelKey,
|
|
127
|
-
from: snap.requestedSec,
|
|
128
|
-
to: snap.snappedSec,
|
|
129
|
-
driftSec: Number(snap.driftSec.toFixed(2)),
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
return snap.snappedSec;
|
|
133
|
-
}
|
|
134
|
-
class Executor {
|
|
135
|
-
constructor(opts = {}) {
|
|
136
|
-
this.concurrency = opts.concurrency ?? 4;
|
|
137
|
-
this.runTool = opts.runTool ?? toolRegistry_1.runTool;
|
|
138
|
-
this.onSceneComplete = opts.onSceneComplete;
|
|
139
|
-
this.taskCache = opts.taskCache;
|
|
140
|
-
this.onTaskMint = opts.onTaskMint;
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Stage 7 slice 1 helper. Returns cached pending-task output if the
|
|
144
|
-
* upstream input hash matches and entry is within TTL; null otherwise.
|
|
145
|
-
* Logs the hit so trace shows where $$ was saved.
|
|
146
|
-
*/
|
|
147
|
-
checkTaskCache(clipId, tool, input) {
|
|
148
|
-
const inputsHash = (0, providerTaskCache_1.hashTaskInputs)(input);
|
|
149
|
-
const hit = (0, providerTaskCache_1.getCachedTask)(this.taskCache, clipId, inputsHash);
|
|
150
|
-
if (hit) {
|
|
151
|
-
logger_1.logger.info("executor: task-cache hit", {
|
|
152
|
-
clipId,
|
|
153
|
-
tool,
|
|
154
|
-
modelKey: hit.modelKey,
|
|
155
|
-
taskId: hit.taskId,
|
|
156
|
-
ageMs: Date.now() - hit.createdAt,
|
|
157
|
-
});
|
|
158
|
-
}
|
|
159
|
-
return { hit, inputsHash };
|
|
160
|
-
}
|
|
161
|
-
/**
|
|
162
|
-
* Best-effort mint. Swallows callback errors — a Firestore blip must not
|
|
163
|
-
* fail the scene (the tool already succeeded and produced a real taskId).
|
|
164
|
-
*/
|
|
165
|
-
async mintTaskCache(clipId, entry) {
|
|
166
|
-
if (!this.onTaskMint)
|
|
167
|
-
return;
|
|
168
|
-
try {
|
|
169
|
-
await this.onTaskMint(clipId, entry);
|
|
170
|
-
}
|
|
171
|
-
catch (err) {
|
|
172
|
-
logger_1.logger.warn("executor: onTaskMint callback threw — swallowed", {
|
|
173
|
-
clipId,
|
|
174
|
-
err: err.message,
|
|
175
|
-
});
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
async run(plan, ctx) {
|
|
179
|
-
const queue = [...plan.scenes];
|
|
180
|
-
const results = new Array(plan.scenes.length);
|
|
181
|
-
const workers = [];
|
|
182
|
-
let cursor = 0;
|
|
183
|
-
const worker = async () => {
|
|
184
|
-
while (true) {
|
|
185
|
-
const i = cursor++;
|
|
186
|
-
if (i >= queue.length)
|
|
187
|
-
return;
|
|
188
|
-
const scene = queue[i];
|
|
189
|
-
const outcome = await this.runScene(scene, ctx, plan.aspect, plan);
|
|
190
|
-
results[i] = outcome;
|
|
191
|
-
if (this.onSceneComplete) {
|
|
192
|
-
try {
|
|
193
|
-
await this.onSceneComplete(scene.sceneIndex, outcome);
|
|
194
|
-
}
|
|
195
|
-
catch (err) {
|
|
196
|
-
logger_1.logger.warn("executor: onSceneComplete callback threw — swallowed", {
|
|
197
|
-
sceneIndex: scene.sceneIndex,
|
|
198
|
-
err: err.message,
|
|
199
|
-
});
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
};
|
|
204
|
-
for (let i = 0; i < Math.min(this.concurrency, queue.length); i++) {
|
|
205
|
-
workers.push(worker());
|
|
206
|
-
}
|
|
207
|
-
await Promise.all(workers);
|
|
208
|
-
return results;
|
|
209
|
-
}
|
|
210
|
-
async runScene(scene, ctx, aspect, plan) {
|
|
211
|
-
const start = Date.now();
|
|
212
|
-
const idempotencyKey = `${ctx.agentRunId}:scene-${scene.sceneIndex}`;
|
|
213
|
-
const sceneCtx = { ...ctx, idempotencyKey };
|
|
214
|
-
const visualOutcome = await this.runVisual(scene, sceneCtx, aspect, plan);
|
|
215
|
-
if (!visualOutcome.ok) {
|
|
216
|
-
return {
|
|
217
|
-
scene,
|
|
218
|
-
ok: false,
|
|
219
|
-
error: visualOutcome.error,
|
|
220
|
-
durationMs: Date.now() - start,
|
|
221
|
-
};
|
|
222
|
-
}
|
|
223
|
-
let voiceoverAudioBase64;
|
|
224
|
-
let voiceoverEstimateMs;
|
|
225
|
-
let voiceoverOverBudget = false;
|
|
226
|
-
// talking-head-avatar bakes the VO into the rendered MP4; running TTS again
|
|
227
|
-
// here would just waste credits and create a duplicate audio track.
|
|
228
|
-
if (scene.voiceoverLine && scene.strategy !== "talking-head-avatar") {
|
|
229
|
-
// AG-31: brief.voice.style → ElevenLabs voice_settings.
|
|
230
|
-
// AG-28: if estimated VO > scene.durationMs, speed up TTS within natural
|
|
231
|
-
// limits (≤1.15× extra over style.speed) before falling back to a
|
|
232
|
-
// freeze-frame-pad signal for the stitcher.
|
|
233
|
-
// AG-44: per-scene voiceStyle wins over plan-level when present, so the
|
|
234
|
-
// emotional arc can shift across the timeline (frustrated → curious → excited).
|
|
235
|
-
const effectiveVoiceStyle = (scene.voiceStyle ?? plan.voiceStyle);
|
|
236
|
-
const baseSettings = resolveVoiceSettings(effectiveVoiceStyle);
|
|
237
|
-
const baseSpeed = baseSettings.speed ?? 1.0;
|
|
238
|
-
const rawEstMs = (0, ttsDuration_1.estimateTtsMs)(scene.voiceoverLine);
|
|
239
|
-
const budgetMs = scene.durationMs * 1.05;
|
|
240
|
-
// Adjusted speed: only kick in if natural speech would overrun. Cap the
|
|
241
|
-
// speedup multiplier at 1.15× style-base; faster than that sounds rushed.
|
|
242
|
-
let speed = baseSpeed;
|
|
243
|
-
if (rawEstMs > budgetMs) {
|
|
244
|
-
const needed = rawEstMs / budgetMs;
|
|
245
|
-
const cappedMultiplier = Math.min(1.15, needed);
|
|
246
|
-
speed = +(baseSpeed * cappedMultiplier).toFixed(2);
|
|
247
|
-
}
|
|
248
|
-
const adjustedEstMs = rawEstMs * (baseSpeed / speed);
|
|
249
|
-
voiceoverEstimateMs = Math.round(adjustedEstMs);
|
|
250
|
-
voiceoverOverBudget = adjustedEstMs > budgetMs;
|
|
251
|
-
const voInput = {
|
|
252
|
-
text: scene.voiceoverLine,
|
|
253
|
-
...baseSettings,
|
|
254
|
-
// Override speed when AG-28 budget forced an adjustment.
|
|
255
|
-
speed: +Math.max(0.7, Math.min(1.2, speed)).toFixed(2),
|
|
256
|
-
};
|
|
257
|
-
const sanitized = sanitizeVoiceId(plan.voiceId);
|
|
258
|
-
if (sanitized)
|
|
259
|
-
voInput.voiceId = sanitized;
|
|
260
|
-
const voOutcome = await this.runTool("generate_voiceover", voInput, { ...sceneCtx, idempotencyKey: `${idempotencyKey}:vo` });
|
|
261
|
-
if (voOutcome.ok) {
|
|
262
|
-
voiceoverAudioBase64 = voOutcome.output.audioBase64;
|
|
263
|
-
}
|
|
264
|
-
else {
|
|
265
|
-
// AG-37: don't swallow silently — final video shipping muted with no
|
|
266
|
-
// log was the original bug. Surface the error in the result so the
|
|
267
|
-
// trace shows WHY VO is missing.
|
|
268
|
-
logger_1.logger.warn("executor: voiceover tool failed", {
|
|
269
|
-
sceneIndex: scene.sceneIndex,
|
|
270
|
-
error: voOutcome.error,
|
|
271
|
-
voiceIdRequested: plan.voiceId,
|
|
272
|
-
voiceIdSent: sanitized,
|
|
273
|
-
});
|
|
274
|
-
}
|
|
275
|
-
// Don't fail the whole scene on VO miss — host can re-run just the VO.
|
|
276
|
-
}
|
|
277
|
-
return {
|
|
278
|
-
scene,
|
|
279
|
-
ok: true,
|
|
280
|
-
result: {
|
|
281
|
-
...visualOutcome.output,
|
|
282
|
-
voiceoverAudioBase64,
|
|
283
|
-
voiceoverEstimateMs,
|
|
284
|
-
voiceoverOverBudget,
|
|
285
|
-
idempotencyKey,
|
|
286
|
-
},
|
|
287
|
-
durationMs: Date.now() - start,
|
|
288
|
-
};
|
|
289
|
-
}
|
|
290
|
-
async runVisual(scene, ctx, aspect, plan) {
|
|
291
|
-
const durationSec = Math.max(2, Math.round(scene.durationMs / 1000));
|
|
292
|
-
// AG-13: collect reference images for this scene's bible entities. Empty
|
|
293
|
-
// array when bible is C0/C1 or no entities are tagged — image-gen falls
|
|
294
|
-
// back to pure text prompt.
|
|
295
|
-
const referenceImageUrls = (0, referenceImageRenderer_1.collectReferenceImageUrls)(plan.bible, scene.bibleEntityIds ?? []);
|
|
296
|
-
const wrap = (tool, outcome) => outcome.ok
|
|
297
|
-
? { ok: true, output: { tool, ...outcome.output } }
|
|
298
|
-
: { ok: false, error: outcome.error };
|
|
299
|
-
switch (scene.strategy) {
|
|
300
|
-
case "stock-video":
|
|
301
|
-
case "stock-image-ken-burns": {
|
|
302
|
-
const out = await this.runTool("search_stock", {
|
|
303
|
-
query: scene.prompt,
|
|
304
|
-
kind: scene.strategy === "stock-video" ? "video" : "image",
|
|
305
|
-
aspectRatio: aspect,
|
|
306
|
-
minDurationSec: scene.strategy === "stock-video" ? durationSec : undefined,
|
|
307
|
-
}, ctx);
|
|
308
|
-
if (!out.ok)
|
|
309
|
-
return wrap("search_stock", out);
|
|
310
|
-
const url = out.output.assets[0]?.url;
|
|
311
|
-
if (!url)
|
|
312
|
-
return {
|
|
313
|
-
ok: false,
|
|
314
|
-
error: {
|
|
315
|
-
code: "NO_STOCK_RESULTS",
|
|
316
|
-
message: `No stock results for: ${scene.prompt}`,
|
|
317
|
-
attemptedProviders: out.output.attemptedProviders,
|
|
318
|
-
},
|
|
319
|
-
};
|
|
320
|
-
return {
|
|
321
|
-
ok: true,
|
|
322
|
-
output: {
|
|
323
|
-
tool: "search_stock",
|
|
324
|
-
visualUrl: url,
|
|
325
|
-
attemptedProviders: out.output.attemptedProviders,
|
|
326
|
-
},
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
|
-
case "ai-image-static":
|
|
330
|
-
case "ai-image-motion": {
|
|
331
|
-
const out = await this.runTool("generate_image", {
|
|
332
|
-
prompt: scene.prompt,
|
|
333
|
-
modelKey: scene.preferredModel ?? "google-nano-banana",
|
|
334
|
-
aspectRatio: aspect,
|
|
335
|
-
...(referenceImageUrls.length
|
|
336
|
-
? { inputImageUrls: referenceImageUrls }
|
|
337
|
-
: {}),
|
|
338
|
-
}, ctx);
|
|
339
|
-
if (!out.ok)
|
|
340
|
-
return wrap("generate_image", out);
|
|
341
|
-
return { ok: true, output: { tool: "generate_image", visualUrl: out.output.imageUrl } };
|
|
342
|
-
}
|
|
343
|
-
case "ai-image-to-video": {
|
|
344
|
-
const animateModelKey = scene.preferredModel ?? "kling-v2.6";
|
|
345
|
-
const animateDurationSec = resolveDurationForPreferredModel(animateModelKey, durationSec, !!scene.preferredModel, scene.sceneIndex);
|
|
346
|
-
// Stage 7 slice 1: cache key spans the WHOLE strategy (both image-gen
|
|
347
|
-
// + animate). Hashing only upstream-stable inputs — prompt, model,
|
|
348
|
-
// duration, aspect, reference images — lets a re-execute skip BOTH
|
|
349
|
-
// tool calls, replaying the cached animate taskId through the poller.
|
|
350
|
-
const clipId = `scene-${scene.sceneIndex}-visual`;
|
|
351
|
-
const cacheInput = {
|
|
352
|
-
strategy: "ai-image-to-video",
|
|
353
|
-
prompt: scene.prompt,
|
|
354
|
-
modelKey: animateModelKey,
|
|
355
|
-
durationSec: animateDurationSec,
|
|
356
|
-
aspect,
|
|
357
|
-
referenceImageUrls,
|
|
358
|
-
};
|
|
359
|
-
const { hit, inputsHash } = this.checkTaskCache(clipId, "animate_image", cacheInput);
|
|
360
|
-
if (hit) {
|
|
361
|
-
return {
|
|
362
|
-
ok: true,
|
|
363
|
-
output: {
|
|
364
|
-
tool: "animate_image",
|
|
365
|
-
pendingTaskId: hit.taskId,
|
|
366
|
-
pendingModelKey: hit.modelKey,
|
|
367
|
-
},
|
|
368
|
-
};
|
|
369
|
-
}
|
|
370
|
-
const img = await this.runTool("generate_image", {
|
|
371
|
-
prompt: scene.prompt,
|
|
372
|
-
modelKey: "google-nano-banana",
|
|
373
|
-
aspectRatio: aspect,
|
|
374
|
-
...(referenceImageUrls.length
|
|
375
|
-
? { inputImageUrls: referenceImageUrls }
|
|
376
|
-
: {}),
|
|
377
|
-
}, { ...ctx, idempotencyKey: `${ctx.idempotencyKey}:img` });
|
|
378
|
-
if (!img.ok)
|
|
379
|
-
return wrap("generate_image", img);
|
|
380
|
-
const animate = await this.runTool("animate_image", {
|
|
381
|
-
imageUrl: img.output.imageUrl,
|
|
382
|
-
motionHint: scene.prompt,
|
|
383
|
-
durationSec: animateDurationSec,
|
|
384
|
-
modelKey: animateModelKey,
|
|
385
|
-
}, ctx);
|
|
386
|
-
if (!animate.ok)
|
|
387
|
-
return wrap("animate_image", animate);
|
|
388
|
-
const mintedModelKey = animate.output.modelKeyUsed ?? animateModelKey;
|
|
389
|
-
await this.mintTaskCache(clipId, {
|
|
390
|
-
tool: "animate_image",
|
|
391
|
-
taskId: animate.output.taskId,
|
|
392
|
-
modelKey: mintedModelKey,
|
|
393
|
-
inputsHash,
|
|
394
|
-
createdAt: Date.now(),
|
|
395
|
-
});
|
|
396
|
-
return {
|
|
397
|
-
ok: true,
|
|
398
|
-
output: {
|
|
399
|
-
tool: "animate_image",
|
|
400
|
-
pendingTaskId: animate.output.taskId,
|
|
401
|
-
pendingModelKey: mintedModelKey,
|
|
402
|
-
attemptedProviders: animate.output.attemptedProviders,
|
|
403
|
-
},
|
|
404
|
-
};
|
|
405
|
-
}
|
|
406
|
-
case "ai-text-to-video": {
|
|
407
|
-
const t2vModelKey = scene.preferredModel ?? "kling-v2.6";
|
|
408
|
-
const t2vDurationSec = resolveDurationForPreferredModel(t2vModelKey, durationSec, !!scene.preferredModel, scene.sceneIndex);
|
|
409
|
-
const clipId = `scene-${scene.sceneIndex}-visual`;
|
|
410
|
-
const cacheInput = {
|
|
411
|
-
strategy: "ai-text-to-video",
|
|
412
|
-
prompt: scene.prompt,
|
|
413
|
-
modelKey: t2vModelKey,
|
|
414
|
-
durationSec: t2vDurationSec,
|
|
415
|
-
};
|
|
416
|
-
const { hit, inputsHash } = this.checkTaskCache(clipId, "generate_video", cacheInput);
|
|
417
|
-
if (hit) {
|
|
418
|
-
return {
|
|
419
|
-
ok: true,
|
|
420
|
-
output: {
|
|
421
|
-
tool: "generate_video",
|
|
422
|
-
pendingTaskId: hit.taskId,
|
|
423
|
-
pendingModelKey: hit.modelKey,
|
|
424
|
-
},
|
|
425
|
-
};
|
|
426
|
-
}
|
|
427
|
-
const out = await this.runTool("generate_video", {
|
|
428
|
-
prompt: scene.prompt,
|
|
429
|
-
modelKey: t2vModelKey,
|
|
430
|
-
durationSec: t2vDurationSec,
|
|
431
|
-
}, ctx);
|
|
432
|
-
if (!out.ok)
|
|
433
|
-
return wrap("generate_video", out);
|
|
434
|
-
const mintedModelKey = out.output.modelKeyUsed ?? t2vModelKey;
|
|
435
|
-
await this.mintTaskCache(clipId, {
|
|
436
|
-
tool: "generate_video",
|
|
437
|
-
taskId: out.output.taskId,
|
|
438
|
-
modelKey: mintedModelKey,
|
|
439
|
-
inputsHash,
|
|
440
|
-
createdAt: Date.now(),
|
|
441
|
-
});
|
|
442
|
-
return {
|
|
443
|
-
ok: true,
|
|
444
|
-
output: {
|
|
445
|
-
tool: "generate_video",
|
|
446
|
-
pendingTaskId: out.output.taskId,
|
|
447
|
-
pendingModelKey: mintedModelKey,
|
|
448
|
-
attemptedProviders: out.output.attemptedProviders,
|
|
449
|
-
},
|
|
450
|
-
};
|
|
451
|
-
}
|
|
452
|
-
case "user-asset": {
|
|
453
|
-
// Per-user library search. The host injects the searcher at boot
|
|
454
|
-
// (see userLibrarySearcher bootstrap); if no match, fall back to
|
|
455
|
-
// stock so the scene still renders rather than failing hard.
|
|
456
|
-
const out = await this.runTool("search_user_library", { query: scene.prompt, limit: 5 }, ctx);
|
|
457
|
-
if (out.ok) {
|
|
458
|
-
const first = out.output.assets[0];
|
|
459
|
-
if (first) {
|
|
460
|
-
return { ok: true, output: { tool: "search_user_library", visualUrl: first.url } };
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
|
-
// No hits or searcher missing — fall back to stock-video as the
|
|
464
|
-
// closest cheap substitute so the executor still produces an asset.
|
|
465
|
-
const stock = await this.runTool("search_stock", {
|
|
466
|
-
query: scene.prompt,
|
|
467
|
-
kind: "video",
|
|
468
|
-
aspectRatio: aspect,
|
|
469
|
-
minDurationSec: durationSec,
|
|
470
|
-
}, ctx);
|
|
471
|
-
if (!stock.ok)
|
|
472
|
-
return wrap("search_stock", stock);
|
|
473
|
-
const fallbackUrl = stock.output.assets[0]?.url;
|
|
474
|
-
if (!fallbackUrl) {
|
|
475
|
-
return {
|
|
476
|
-
ok: false,
|
|
477
|
-
error: {
|
|
478
|
-
code: "USER_ASSET_NO_MATCH",
|
|
479
|
-
message: `No user-library match for "${scene.prompt}" and no stock fallback.`,
|
|
480
|
-
attemptedProviders: stock.output.attemptedProviders,
|
|
481
|
-
},
|
|
482
|
-
};
|
|
483
|
-
}
|
|
484
|
-
return {
|
|
485
|
-
ok: true,
|
|
486
|
-
output: {
|
|
487
|
-
tool: "search_user_library",
|
|
488
|
-
visualUrl: fallbackUrl,
|
|
489
|
-
attemptedProviders: stock.output.attemptedProviders,
|
|
490
|
-
},
|
|
491
|
-
};
|
|
492
|
-
}
|
|
493
|
-
case "talking-head-avatar": {
|
|
494
|
-
if (!scene.avatarFaceUrl) {
|
|
495
|
-
return {
|
|
496
|
-
ok: false,
|
|
497
|
-
error: {
|
|
498
|
-
code: "AVATAR_FACE_REQUIRED",
|
|
499
|
-
message: "talking-head-avatar requires scene.avatarFaceUrl (set on the user's brand kit).",
|
|
500
|
-
},
|
|
501
|
-
};
|
|
502
|
-
}
|
|
503
|
-
if (!scene.voiceoverLine) {
|
|
504
|
-
return {
|
|
505
|
-
ok: false,
|
|
506
|
-
error: {
|
|
507
|
-
code: "AVATAR_VO_REQUIRED",
|
|
508
|
-
message: "talking-head-avatar requires scene.voiceoverLine for the lipsync audio.",
|
|
509
|
-
},
|
|
510
|
-
};
|
|
511
|
-
}
|
|
512
|
-
const avatarMode = scene.tier === "T3" ? "pro" : "std";
|
|
513
|
-
const clipId = `scene-${scene.sceneIndex}-visual`;
|
|
514
|
-
const cacheInput = {
|
|
515
|
-
strategy: "talking-head-avatar",
|
|
516
|
-
inputImageUrl: scene.avatarFaceUrl,
|
|
517
|
-
ttsText: scene.voiceoverLine,
|
|
518
|
-
mode: avatarMode,
|
|
519
|
-
};
|
|
520
|
-
const { hit, inputsHash } = this.checkTaskCache(clipId, "generate_avatar_video", cacheInput);
|
|
521
|
-
if (hit) {
|
|
522
|
-
return {
|
|
523
|
-
ok: true,
|
|
524
|
-
output: {
|
|
525
|
-
tool: "generate_avatar_video",
|
|
526
|
-
pendingTaskId: hit.taskId,
|
|
527
|
-
pendingModelKey: hit.modelKey,
|
|
528
|
-
},
|
|
529
|
-
};
|
|
530
|
-
}
|
|
531
|
-
const out = await this.runTool("generate_avatar_video", {
|
|
532
|
-
inputImageUrl: scene.avatarFaceUrl,
|
|
533
|
-
ttsText: scene.voiceoverLine,
|
|
534
|
-
mode: avatarMode,
|
|
535
|
-
}, ctx);
|
|
536
|
-
if (!out.ok)
|
|
537
|
-
return wrap("generate_avatar_video", out);
|
|
538
|
-
await this.mintTaskCache(clipId, {
|
|
539
|
-
tool: "generate_avatar_video",
|
|
540
|
-
taskId: out.output.taskId,
|
|
541
|
-
// Avatar tool hardcodes Kling Avatar internally; poller needs the
|
|
542
|
-
// actual model key to route via getAiGenProviderService (no special
|
|
543
|
-
// resolver exists). AG-20: was "generate_avatar_video" which would
|
|
544
|
-
// throw at factory lookup the moment the avatar path was polled.
|
|
545
|
-
modelKey: "kling-avatar",
|
|
546
|
-
inputsHash,
|
|
547
|
-
createdAt: Date.now(),
|
|
548
|
-
});
|
|
549
|
-
return {
|
|
550
|
-
ok: true,
|
|
551
|
-
output: {
|
|
552
|
-
tool: "generate_avatar_video",
|
|
553
|
-
pendingTaskId: out.output.taskId,
|
|
554
|
-
pendingModelKey: "kling-avatar",
|
|
555
|
-
},
|
|
556
|
-
};
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
}
|
|
560
|
-
}
|
|
561
|
-
exports.Executor = Executor;
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import { ChatRequest, ChatResponse, LlmCaller, StructuredRequest, StructuredResponse } from "./llmCaller";
|
|
3
|
-
/**
|
|
4
|
-
* Vercel AI Gateway LlmCaller — OpenAI-compatible /v1/chat/completions surface.
|
|
5
|
-
* Single endpoint, multi-provider routing via "{provider}/{modelId}" model id.
|
|
6
|
-
*
|
|
7
|
-
* No SDK dependency: pure fetch keeps the shared package light. Structured
|
|
8
|
-
* output uses JSON-schema response_format (supported by gateway across
|
|
9
|
-
* Anthropic / OpenAI / Google providers — gateway normalises it).
|
|
10
|
-
*
|
|
11
|
-
* Reliability features (added Phase 1 of AGENT_TEST_PLAN):
|
|
12
|
-
* - F1 transport-level retry: each HTTP call retries 3× with exp backoff
|
|
13
|
-
* (250 / 1000 / 4000ms) on transient errors (network drops, 408/429/5xx).
|
|
14
|
-
* - F2 model fallback: when a request supplies `fallbackModel` and the
|
|
15
|
-
* primary exhausts retries on a transient error, we attempt the fallback
|
|
16
|
-
* once (also with its own retry budget). Schema-validation failures and
|
|
17
|
-
* other deterministic errors do NOT trigger fallback.
|
|
18
|
-
*/
|
|
19
|
-
export interface GatewayConfig {
|
|
20
|
-
/** Full base url, e.g. "https://ai-gateway.vercel.sh/v1". */
|
|
21
|
-
baseUrl: string;
|
|
22
|
-
/** Bearer token. */
|
|
23
|
-
apiKey: string;
|
|
24
|
-
/** Default request timeout in ms. */
|
|
25
|
-
timeoutMs?: number;
|
|
26
|
-
/** Optional fetch impl override (tests). */
|
|
27
|
-
fetchImpl?: typeof fetch;
|
|
28
|
-
/** Max retry attempts per model on transient errors. Default 3. */
|
|
29
|
-
maxRetries?: number;
|
|
30
|
-
/** Base backoff in ms (exponential: base, base*4, base*16). Default 250. */
|
|
31
|
-
retryBaseMs?: number;
|
|
32
|
-
}
|
|
33
|
-
export declare class GatewayLlmCaller implements LlmCaller {
|
|
34
|
-
private readonly cfg;
|
|
35
|
-
private readonly fetchImpl;
|
|
36
|
-
private readonly timeoutMs;
|
|
37
|
-
private readonly maxRetries;
|
|
38
|
-
private readonly retryBaseMs;
|
|
39
|
-
constructor(cfg: GatewayConfig);
|
|
40
|
-
chat(req: ChatRequest): Promise<ChatResponse>;
|
|
41
|
-
structured<T extends z.ZodTypeAny>(req: StructuredRequest<T>): Promise<StructuredResponse<T>>;
|
|
42
|
-
structuredStream<T extends z.ZodTypeAny>(req: StructuredRequest<T>): {
|
|
43
|
-
tokens: AsyncIterable<string>;
|
|
44
|
-
result: Promise<StructuredResponse<T>>;
|
|
45
|
-
};
|
|
46
|
-
private createStreamingIterator;
|
|
47
|
-
private modelId;
|
|
48
|
-
private encodeMessage;
|
|
49
|
-
private toDataUri;
|
|
50
|
-
private usage;
|
|
51
|
-
/**
|
|
52
|
-
* POST /chat/completions with transient-error retry (F1).
|
|
53
|
-
* Throws TransientLlmError after exhausting retries on transient failures —
|
|
54
|
-
* caller catches it to trigger model fallback (F2). Non-transient errors
|
|
55
|
-
* (4xx other than 408/425/429, malformed responses) propagate as plain Error.
|
|
56
|
-
*/
|
|
57
|
-
private postWithRetry;
|
|
58
|
-
private withRetry;
|
|
59
|
-
private postOnce;
|
|
60
|
-
}
|
|
61
|
-
//# sourceMappingURL=llmCallerGateway.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"llmCallerGateway.d.ts","sourceRoot":"","sources":["../../../src/services/agent/llmCallerGateway.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,WAAW,EACX,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,aAAa,CAAC;AAGrB;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,WAAW,aAAa;IAC5B,6DAA6D;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,oBAAoB;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,mEAAmE;IACnE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AA2CD,qBAAa,gBAAiB,YAAW,SAAS;IAMpC,OAAO,CAAC,QAAQ,CAAC,GAAG;IALhC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAER,GAAG,EAAE,aAAa;IAOzC,IAAI,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAkC7C,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC,UAAU,EACrC,GAAG,EAAE,iBAAiB,CAAC,CAAC,CAAC,GACxB,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAgEjC,gBAAgB,CAAC,CAAC,SAAS,CAAC,CAAC,UAAU,EACrC,GAAG,EAAE,iBAAiB,CAAC,CAAC,CAAC,GACxB;QACD,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;QAC9B,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;KACxC;YAqBc,uBAAuB;IA4ItC,OAAO,CAAC,OAAO;YAID,aAAa;YAqBb,SAAS;IAcvB,OAAO,CAAC,KAAK;IAQb;;;;;OAKG;YACW,aAAa;YAIb,SAAS;YA+BT,QAAQ;CA6BvB"}
|