vidspotai-shared 1.0.87 → 1.0.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/globals/creditUnit.d.ts +17 -0
- package/lib/globals/creditUnit.d.ts.map +1 -1
- package/lib/globals/creditUnit.js +18 -1
- package/lib/globals/index.d.ts +1 -0
- package/lib/globals/index.d.ts.map +1 -1
- package/lib/globals/index.js +1 -0
- package/lib/globals/types.d.ts +11 -2
- package/lib/globals/types.d.ts.map +1 -1
- package/lib/globals/types.js +9 -0
- package/lib/models/agent.model.d.ts +18 -0
- package/lib/models/agent.model.d.ts.map +1 -1
- package/lib/models/video.model.d.ts +11 -0
- package/lib/models/video.model.d.ts.map +1 -1
- package/lib/schemas/agentRunJob.schema.d.ts +1 -0
- package/lib/schemas/agentRunJob.schema.d.ts.map +1 -1
- package/lib/schemas/agentRunJob.schema.js +9 -0
- package/lib/services/agent/beatSnap.d.ts +3 -0
- package/lib/services/agent/beatSnap.d.ts.map +1 -1
- package/lib/services/agent/beatSnap.js +5 -0
- package/lib/services/agent/executor/core.d.ts +6 -0
- package/lib/services/agent/executor/core.d.ts.map +1 -1
- package/lib/services/agent/executor/core.js +45 -0
- package/lib/services/agent/executor/types.d.ts +29 -0
- package/lib/services/agent/executor/types.d.ts.map +1 -1
- package/lib/services/agent/executor/visual.d.ts.map +1 -1
- package/lib/services/agent/executor/visual.js +7 -0
- package/lib/services/agent/index.d.ts +1 -0
- package/lib/services/agent/index.d.ts.map +1 -1
- package/lib/services/agent/index.js +1 -0
- package/lib/services/agent/resnapVo.d.ts +51 -0
- package/lib/services/agent/resnapVo.d.ts.map +1 -0
- package/lib/services/agent/resnapVo.js +102 -0
- package/lib/services/aiGen/providers/alibaba/alibaba.d.ts +1 -1
- package/lib/services/aiGen/providers/alibaba/alibaba.d.ts.map +1 -1
- package/lib/services/aiGen/providers/alibaba/alibaba.js +2 -2
- package/lib/services/aiGen/providers/bytedance/bytedance.service.d.ts.map +1 -1
- package/lib/services/aiGen/providers/bytedance/bytedance.service.js +8 -0
- package/lib/services/aiGen/providers/google/google.service.d.ts +1 -1
- package/lib/services/aiGen/providers/google/google.service.d.ts.map +1 -1
- package/lib/services/aiGen/providers/google/google.service.js +71 -16
- package/lib/services/aiGen/providers/google/googleErrors.d.ts +28 -0
- package/lib/services/aiGen/providers/google/googleErrors.d.ts.map +1 -1
- package/lib/services/aiGen/providers/google/googleErrors.js +75 -2
- package/lib/services/aiGen/providers/google/googleKeyPool.d.ts +8 -1
- package/lib/services/aiGen/providers/google/googleKeyPool.d.ts.map +1 -1
- package/lib/services/aiGen/providers/google/googleKeyPool.js +77 -9
- package/lib/services/aiGen/providers/kling/klingImage.service.d.ts +1 -1
- package/lib/services/aiGen/providers/kling/klingImage.service.d.ts.map +1 -1
- package/lib/services/aiGen/providers/kling/klingImage.service.js +3 -3
- package/lib/services/aiGen/providers/minimax/minimax.service.d.ts +1 -1
- package/lib/services/aiGen/providers/minimax/minimax.service.d.ts.map +1 -1
- package/lib/services/aiGen/providers/minimax/minimax.service.js +16 -7
- package/lib/services/aiGen/providers/openai/openai.service.d.ts +1 -1
- package/lib/services/aiGen/providers/openai/openai.service.d.ts.map +1 -1
- package/lib/services/aiGen/providers/openai/openai.service.js +13 -3
- package/lib/services/aiGen/providers/runway/runway.service.d.ts +1 -1
- package/lib/services/aiGen/providers/runway/runway.service.d.ts.map +1 -1
- package/lib/services/aiGen/providers/runway/runway.service.js +3 -3
- package/lib/services/aiGen/providers/types.d.ts +8 -0
- package/lib/services/aiGen/providers/types.d.ts.map +1 -1
- package/lib/services/bullmq.service.d.ts.map +1 -1
- package/lib/services/bullmq.service.js +13 -5
- package/lib/services/freePlan/index.d.ts +38 -0
- package/lib/services/freePlan/index.d.ts.map +1 -0
- package/lib/services/freePlan/index.js +55 -0
- package/lib/services/index.d.ts +1 -0
- package/lib/services/index.d.ts.map +1 -1
- package/lib/services/index.js +1 -0
- package/lib/utils/errors.d.ts +23 -0
- package/lib/utils/errors.d.ts.map +1 -1
- package/lib/utils/errors.js +75 -1
- package/lib/utils/helpers.d.ts +8 -1
- package/lib/utils/helpers.d.ts.map +1 -1
- package/lib/utils/helpers.js +9 -2
- package/package.json +1 -1
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { VideoPlan } from "../../schemas/videoPlan.schema";
|
|
2
|
+
/**
|
|
3
|
+
* Post-VO re-snap (Issues A + B from the duration/VO audit, 2026-06-17).
|
|
4
|
+
*
|
|
5
|
+
* Beat-snap (`snapPlanToBeats`) runs ONCE at plan time against a loose 3-wps VO
|
|
6
|
+
* *estimate*, because the real VO doesn't exist yet. By finalize the real audio
|
|
7
|
+
* has been probed (ffprobe → `voiceoverActualMs`), so two residual quality gaps
|
|
8
|
+
* are now fixable against the truth:
|
|
9
|
+
*
|
|
10
|
+
* A. Trailing silence — VO shorter than its scene leaves dead air over a
|
|
11
|
+
* continuing visual (benign with music, reads as a hang without it).
|
|
12
|
+
* B. Beat drift — a VO longer than planned fires the `max(planned, vo)` stretch
|
|
13
|
+
* backstop in planToProject, pushing every later cut off the onset it was
|
|
14
|
+
* snapped to. Cumulative across overrunning scenes.
|
|
15
|
+
*
|
|
16
|
+
* This pure pass recomputes each scene boundary against the REAL per-scene VO
|
|
17
|
+
* length. The onsets the plan was snapped against persist on `plan.music.onsets`,
|
|
18
|
+
* so we re-anchor every boundary to an ABSOLUTE onset — drift cannot accumulate
|
|
19
|
+
* (scene N+1's cut lands on an onset regardless of what N did).
|
|
20
|
+
*
|
|
21
|
+
* Three desiderata are over-constrained (a VO almost never ends exactly on an
|
|
22
|
+
* onset, so beat-aligned and zero-silence conflict), so the guarantee is a
|
|
23
|
+
* deterministic, bounded outcome with a fixed priority order:
|
|
24
|
+
*
|
|
25
|
+
* (I) VO-safe — `out.durationMs >= actualVoMs` for every scene. HARD, always.
|
|
26
|
+
* (II) on-beat — boundaries land on onsets / bar grid where music exists.
|
|
27
|
+
* (III) bounded silence — trailing silence ≤ {one onset gap | one bar | TAIL_MS}.
|
|
28
|
+
*
|
|
29
|
+
* Mode is picked automatically (same precedence as beat-snap): onsets → bpm grid
|
|
30
|
+
* → no-music floor. `opts.skip` returns the plan untouched (regen/frozen safety —
|
|
31
|
+
* re-snapping non-frozen neighbours would move offsets and desync a locked clip).
|
|
32
|
+
*/
|
|
33
|
+
/** Breathing room appended after the spoken line before the scene cuts. */
|
|
34
|
+
export declare const TAIL_MS = 300;
|
|
35
|
+
/** Floor for a scene in the no-music mode (prevents sub-second flashes). */
|
|
36
|
+
export declare const MIN_SCENE_MS = 1500;
|
|
37
|
+
export interface ResnapOptions {
|
|
38
|
+
/**
|
|
39
|
+
* When true the plan is returned verbatim. Set on the regen / frozen-clip
|
|
40
|
+
* path (`preserveClips.length > 0`): frozen clips keep their prior display
|
|
41
|
+
* window, and re-snapping their non-frozen neighbours would shift offsets and
|
|
42
|
+
* desync the locked clip. v1 guarantee — strictly avoid the interaction.
|
|
43
|
+
*/
|
|
44
|
+
skip?: boolean;
|
|
45
|
+
/** Breathing room after the VO (default {@link TAIL_MS}). */
|
|
46
|
+
tailMs?: number;
|
|
47
|
+
/** No-music scene floor (default {@link MIN_SCENE_MS}). */
|
|
48
|
+
minSceneMs?: number;
|
|
49
|
+
}
|
|
50
|
+
export declare function resnapPlanToActualVo(plan: VideoPlan, actualVoMsByScene: Record<number, number>, opts?: ResnapOptions): VideoPlan;
|
|
51
|
+
//# sourceMappingURL=resnapVo.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resnapVo.d.ts","sourceRoot":"","sources":["../../../src/services/agent/resnapVo.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAGzE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,2EAA2E;AAC3E,eAAO,MAAM,OAAO,MAAM,CAAC;AAC3B,4EAA4E;AAC5E,eAAO,MAAM,YAAY,OAAO,CAAC;AAIjC,MAAM,WAAW,aAAa;IAC5B;;;;;OAKG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,SAAS,EACf,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACzC,IAAI,GAAE,aAAkB,GACvB,SAAS,CAwBX"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MIN_SCENE_MS = exports.TAIL_MS = void 0;
|
|
4
|
+
exports.resnapPlanToActualVo = resnapPlanToActualVo;
|
|
5
|
+
const beatSnap_1 = require("./beatSnap");
|
|
6
|
+
/**
|
|
7
|
+
* Post-VO re-snap (Issues A + B from the duration/VO audit, 2026-06-17).
|
|
8
|
+
*
|
|
9
|
+
* Beat-snap (`snapPlanToBeats`) runs ONCE at plan time against a loose 3-wps VO
|
|
10
|
+
* *estimate*, because the real VO doesn't exist yet. By finalize the real audio
|
|
11
|
+
* has been probed (ffprobe → `voiceoverActualMs`), so two residual quality gaps
|
|
12
|
+
* are now fixable against the truth:
|
|
13
|
+
*
|
|
14
|
+
* A. Trailing silence — VO shorter than its scene leaves dead air over a
|
|
15
|
+
* continuing visual (benign with music, reads as a hang without it).
|
|
16
|
+
* B. Beat drift — a VO longer than planned fires the `max(planned, vo)` stretch
|
|
17
|
+
* backstop in planToProject, pushing every later cut off the onset it was
|
|
18
|
+
* snapped to. Cumulative across overrunning scenes.
|
|
19
|
+
*
|
|
20
|
+
* This pure pass recomputes each scene boundary against the REAL per-scene VO
|
|
21
|
+
* length. The onsets the plan was snapped against persist on `plan.music.onsets`,
|
|
22
|
+
* so we re-anchor every boundary to an ABSOLUTE onset — drift cannot accumulate
|
|
23
|
+
* (scene N+1's cut lands on an onset regardless of what N did).
|
|
24
|
+
*
|
|
25
|
+
* Three desiderata are over-constrained (a VO almost never ends exactly on an
|
|
26
|
+
* onset, so beat-aligned and zero-silence conflict), so the guarantee is a
|
|
27
|
+
* deterministic, bounded outcome with a fixed priority order:
|
|
28
|
+
*
|
|
29
|
+
* (I) VO-safe — `out.durationMs >= actualVoMs` for every scene. HARD, always.
|
|
30
|
+
* (II) on-beat — boundaries land on onsets / bar grid where music exists.
|
|
31
|
+
* (III) bounded silence — trailing silence ≤ {one onset gap | one bar | TAIL_MS}.
|
|
32
|
+
*
|
|
33
|
+
* Mode is picked automatically (same precedence as beat-snap): onsets → bpm grid
|
|
34
|
+
* → no-music floor. `opts.skip` returns the plan untouched (regen/frozen safety —
|
|
35
|
+
* re-snapping non-frozen neighbours would move offsets and desync a locked clip).
|
|
36
|
+
*/
|
|
37
|
+
/** Breathing room appended after the spoken line before the scene cuts. */
|
|
38
|
+
exports.TAIL_MS = 300;
|
|
39
|
+
/** Floor for a scene in the no-music mode (prevents sub-second flashes). */
|
|
40
|
+
exports.MIN_SCENE_MS = 1500;
|
|
41
|
+
const BEATS_PER_BAR = 4;
|
|
42
|
+
function resnapPlanToActualVo(plan, actualVoMsByScene, opts = {}) {
|
|
43
|
+
if (opts.skip)
|
|
44
|
+
return plan;
|
|
45
|
+
const tailMs = opts.tailMs ?? exports.TAIL_MS;
|
|
46
|
+
const minSceneMs = opts.minSceneMs ?? exports.MIN_SCENE_MS;
|
|
47
|
+
// Per-scene minimum (the binding floor that makes the output VO-safe). A scene
|
|
48
|
+
// that actually speaks must outlast its real VO plus a tail; a scene with no
|
|
49
|
+
// VO (silent B-roll) keeps its deliberately-planned duration as the floor so
|
|
50
|
+
// the re-snap never collapses it to zero.
|
|
51
|
+
const floorMs = (scene) => {
|
|
52
|
+
const vo = actualVoMsByScene[scene.sceneIndex] ?? 0;
|
|
53
|
+
return vo > 0 ? vo + tailMs : scene.durationMs;
|
|
54
|
+
};
|
|
55
|
+
const onsets = plan.music?.onsets;
|
|
56
|
+
if (onsets && onsets.length >= plan.scenes.length) {
|
|
57
|
+
return resnapToOnsets(plan, onsets, floorMs);
|
|
58
|
+
}
|
|
59
|
+
const bpm = (0, beatSnap_1.resolveBpm)(plan);
|
|
60
|
+
if (bpm) {
|
|
61
|
+
return resnapToBpmGrid(plan, bpm, floorMs);
|
|
62
|
+
}
|
|
63
|
+
return resnapNoMusic(plan, floorMs, minSceneMs);
|
|
64
|
+
}
|
|
65
|
+
/** Mode 1 — snap each cumulative boundary to the nearest onset ≥ (start + floor). */
|
|
66
|
+
function resnapToOnsets(plan, onsets, floorMs) {
|
|
67
|
+
const sortedOnsets = [...onsets].sort((a, b) => a - b);
|
|
68
|
+
const snappedScenes = [];
|
|
69
|
+
let cumulativeMs = 0;
|
|
70
|
+
for (const scene of plan.scenes) {
|
|
71
|
+
const minDurMs = floorMs(scene);
|
|
72
|
+
// Target the floor itself (not the planned duration): we want the FIRST
|
|
73
|
+
// onset at or past where the VO actually ends, which both trims trailing
|
|
74
|
+
// silence (A) and re-anchors the cut to an absolute onset (B).
|
|
75
|
+
const targetEndMs = cumulativeMs + minDurMs;
|
|
76
|
+
const snappedEndMs = Math.max(cumulativeMs + minDurMs, (0, beatSnap_1.nearestOnsetMs)(targetEndMs, sortedOnsets, cumulativeMs + minDurMs));
|
|
77
|
+
const newDurationMs = snappedEndMs - cumulativeMs;
|
|
78
|
+
snappedScenes.push(newDurationMs === scene.durationMs ? scene : { ...scene, durationMs: newDurationMs });
|
|
79
|
+
cumulativeMs = snappedEndMs;
|
|
80
|
+
}
|
|
81
|
+
return { ...plan, scenes: snappedScenes, totalDurationMs: cumulativeMs };
|
|
82
|
+
}
|
|
83
|
+
/** Mode 2 — round each scene to the bar grid, never below the VO floor. */
|
|
84
|
+
function resnapToBpmGrid(plan, bpm, floorMs) {
|
|
85
|
+
const barMs = (60000 / bpm) * BEATS_PER_BAR;
|
|
86
|
+
const snappedScenes = plan.scenes.map((scene) => {
|
|
87
|
+
const minMs = Math.max(barMs, (0, beatSnap_1.ceilToBar)(floorMs(scene), barMs));
|
|
88
|
+
const snapped = Math.max(minMs, (0, beatSnap_1.roundToBar)(floorMs(scene), barMs));
|
|
89
|
+
return snapped === scene.durationMs ? scene : { ...scene, durationMs: snapped };
|
|
90
|
+
});
|
|
91
|
+
const totalDurationMs = snappedScenes.reduce((sum, s) => sum + s.durationMs, 0);
|
|
92
|
+
return { ...plan, scenes: snappedScenes, totalDurationMs };
|
|
93
|
+
}
|
|
94
|
+
/** Mode 3 — no music: scene = max(VO + tail, MIN_SCENE_MS). Silence == tail. */
|
|
95
|
+
function resnapNoMusic(plan, floorMs, minSceneMs) {
|
|
96
|
+
const snappedScenes = plan.scenes.map((scene) => {
|
|
97
|
+
const snapped = Math.max(floorMs(scene), minSceneMs);
|
|
98
|
+
return snapped === scene.durationMs ? scene : { ...scene, durationMs: snapped };
|
|
99
|
+
});
|
|
100
|
+
const totalDurationMs = snappedScenes.reduce((sum, s) => sum + s.durationMs, 0);
|
|
101
|
+
return { ...plan, scenes: snappedScenes, totalDurationMs };
|
|
102
|
+
}
|
|
@@ -45,6 +45,6 @@ export declare class AlibabaService extends BaseAiGenProviderService {
|
|
|
45
45
|
* and text2image/image-synthesis (legacy Qwen).
|
|
46
46
|
*/
|
|
47
47
|
private generateImageAsync;
|
|
48
|
-
getCreditUsed({ modelKey, resolution, aspectRatio, duration, multiClip, numImages, }: CreditUsageParams): number;
|
|
48
|
+
getCreditUsed({ modelKey, resolution, aspectRatio, duration, multiClip, numImages, creditFloor, }: CreditUsageParams): number;
|
|
49
49
|
}
|
|
50
50
|
//# sourceMappingURL=alibaba.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"alibaba.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/alibaba/alibaba.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,UAAU,CAAC;AA2DlB,qBAAa,cAAe,SAAQ,wBAAwB;IAE1D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;;IAiBjC;;;;OAIG;YACW,OAAO;IAwBf,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IAiE3B,gBAAgB,CAAC,EACrB,IAAI,EACJ,cAAc,EACd,cAAyB,EACzB,QAAQ,GACT,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA0EjD;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IA4EjC;;;OAGG;YACW,iBAAiB;IAuC/B;;;OAGG;YACW,kBAAkB;IAuDhC,aAAa,CAAC,EACZ,QAAQ,EACR,UAAU,EACV,WAAW,EACX,QAAQ,EACR,SAAiB,EACjB,SAAa,
|
|
1
|
+
{"version":3,"file":"alibaba.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/alibaba/alibaba.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,UAAU,CAAC;AA2DlB,qBAAa,cAAe,SAAQ,wBAAwB;IAE1D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;;IAiBjC;;;;OAIG;YACW,OAAO;IAwBf,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IAiE3B,gBAAgB,CAAC,EACrB,IAAI,EACJ,cAAc,EACd,cAAyB,EACzB,QAAQ,GACT,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA0EjD;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IA4EjC;;;OAGG;YACW,iBAAiB;IAuC/B;;;OAGG;YACW,kBAAkB;IAuDhC,aAAa,CAAC,EACZ,QAAQ,EACR,UAAU,EACV,WAAW,EACX,QAAQ,EACR,SAAiB,EACjB,SAAa,EACb,WAAW,GACZ,EAAE,iBAAiB,GAAG,MAAM;CAoB9B"}
|
|
@@ -368,11 +368,11 @@ class AlibabaService extends baseAiGenProvider_service_1.BaseAiGenProviderServic
|
|
|
368
368
|
const urls = results.map((r) => r.url).filter((u) => !!u);
|
|
369
369
|
return { urls, taskId };
|
|
370
370
|
}
|
|
371
|
-
getCreditUsed({ modelKey, resolution, aspectRatio, duration, multiClip = false, numImages = 1, }) {
|
|
371
|
+
getCreditUsed({ modelKey, resolution, aspectRatio, duration, multiClip = false, numImages = 1, creditFloor, }) {
|
|
372
372
|
const modelConfig = aiModels_1.aiModelConfigs[modelKey];
|
|
373
373
|
// Image-gen — flat per-image cost (Kolors / Qwen-Image).
|
|
374
374
|
if (modelConfig?.cost?.fixed !== undefined) {
|
|
375
|
-
return (0, utils_1.getCreditsFromCost)(modelConfig.cost.fixed * Math.max(1, numImages), !multiClip);
|
|
375
|
+
return (0, utils_1.getCreditsFromCost)(modelConfig.cost.fixed * Math.max(1, numImages), !multiClip, creditFloor);
|
|
376
376
|
}
|
|
377
377
|
const dimensions = (0, helpers_1.getAlibabaDimensions)(resolution, aspectRatio);
|
|
378
378
|
const cost = modelConfig.cost?.table?.[dimensions]?.[duration];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bytedance.service.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/bytedance/bytedance.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,UAAU,CAAC;AAUlB,qBAAa,gBAAiB,SAAQ,wBAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,OAAO,CACsD;IAI9E,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAIb,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IAoL3B,gBAAgB,CAAC,EACrB,IAAI,EACJ,cAAc,EACd,cAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA4HjD,aAAa,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM;
|
|
1
|
+
{"version":3,"file":"bytedance.service.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/bytedance/bytedance.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,UAAU,CAAC;AAUlB,qBAAa,gBAAiB,SAAQ,wBAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,OAAO,CACsD;IAI9E,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,WAAW;IAIb,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IAoL3B,gBAAgB,CAAC,EACrB,IAAI,EACJ,cAAc,EACd,cAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA4HjD,aAAa,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM;CAyEjD"}
|
|
@@ -270,6 +270,14 @@ class ByteDanceService extends baseAiGenProvider_service_1.BaseAiGenProviderServ
|
|
|
270
270
|
}
|
|
271
271
|
getCreditUsed(params) {
|
|
272
272
|
const modelConfig = aiModels_1.aiModelConfigs[params.modelKey];
|
|
273
|
+
// Image-gen — flat per-image cost (Seedream). Must be handled before the
|
|
274
|
+
// token-based video logic below: image models carry no duration/resolution,
|
|
275
|
+
// so they'd otherwise fall through to the `return 10` fallback. Honors the
|
|
276
|
+
// image credit floor passed by the image controller.
|
|
277
|
+
if (modelConfig?.cost?.fixed !== undefined &&
|
|
278
|
+
modelConfig.type?.some((t) => t.endsWith("-to-image"))) {
|
|
279
|
+
return (0, helpers_2.getCreditsFromCost)(modelConfig.cost.fixed * Math.max(1, params.numImages ?? 1), !params.multiClip, params.creditFloor);
|
|
280
|
+
}
|
|
273
281
|
// Seedance 2.0 prices on (resolution × whether the request includes a
|
|
274
282
|
// video input), not on (audio toggle). Pick the matching per-1K rate.
|
|
275
283
|
let per1KTokens;
|
|
@@ -46,6 +46,6 @@ export declare class GoogleService extends BaseAiGenProviderService {
|
|
|
46
46
|
* ./googleMusic.
|
|
47
47
|
*/
|
|
48
48
|
generateMusic(params: MusicGenerationParams): Promise<MusicGenerationResult>;
|
|
49
|
-
getCreditUsed({ modelKey, duration, resolution, multiClip, numImages, imageSize }: CreditUsageParams): number;
|
|
49
|
+
getCreditUsed({ modelKey, duration, resolution, multiClip, numImages, imageSize, creditFloor }: CreditUsageParams): number;
|
|
50
50
|
}
|
|
51
51
|
//# sourceMappingURL=google.service.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"google.service.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/google/google.service.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,UAAU,CAAC;AAelB,qBAAa,aAAc,SAAQ,wBAAwB;IAKzD,OAAO,CAAC,EAAE,CAAc;IACxB,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAK;;IAQ/C;;;;;;;;;OASG;IACG,YAAY,CAChB,MAAM,EAAE,oBAAoB,GAC3B,OAAO,CAAC,oBAAoB,CAAC;IAoChC;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAoCvB;;;;OAIG;YACW,kBAAkB;IA+B1B,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"google.service.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/google/google.service.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,UAAU,CAAC;AAelB,qBAAa,aAAc,SAAQ,wBAAwB;IAKzD,OAAO,CAAC,EAAE,CAAc;IACxB,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAK;;IAQ/C;;;;;;;;;OASG;IACG,YAAY,CAChB,MAAM,EAAE,oBAAoB,GAC3B,OAAO,CAAC,oBAAoB,CAAC;IAoChC;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAoCvB;;;;OAIG;YACW,kBAAkB;IA+B1B,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IA0M3B,gBAAgB,CAAC,EACrB,IAAI,EACJ,cAAc,EACd,cAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAiH3C,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;YAiBnB,cAAc;IAwG5B;;;;;;OAMG;IACH;;;;OAIG;IACG,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IAIjC,aAAa,CAAC,EAAE,QAAQ,EAAE,QAAY,EAAE,UAAmB,EAAE,SAAiB,EAAE,SAAa,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,iBAAiB,GAAG,MAAM;CA8BpJ"}
|
|
@@ -236,20 +236,75 @@ class GoogleService extends baseAiGenProvider_service_1.BaseAiGenProviderService
|
|
|
236
236
|
}));
|
|
237
237
|
}
|
|
238
238
|
}
|
|
239
|
-
//
|
|
240
|
-
//
|
|
241
|
-
//
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
239
|
+
// Submit with billing/auth failover across the key pool. Each key is a
|
|
240
|
+
// separate GCP project + billing account; if the highest-priority key's
|
|
241
|
+
// account is depleted (or its auth is revoked), retry the SAME submit on
|
|
242
|
+
// the next key and circuit-break the failed one so the rest of the pipeline
|
|
243
|
+
// routes around it (see googleKeyPool.markDepleted).
|
|
244
|
+
//
|
|
245
|
+
// Cost safety: failover only fires when generateVideos THREW — i.e. no
|
|
246
|
+
// operation name came back, so that account was NOT billed for this scene.
|
|
247
|
+
// A scene is therefore charged on at most ONE account; we never pay twice.
|
|
248
|
+
const tried = new Set();
|
|
249
|
+
let lastErr;
|
|
250
|
+
// Bounded by the number of keys: one submit attempt per account, worst case.
|
|
251
|
+
for (let attempt = 0; attempt < this.keyPool.size; attempt++) {
|
|
252
|
+
const keyEntry = await this.keyPool.pickForSubmit(modelId, modelConfig.requestPerMin ?? 0, modelConfig.requestPerDay ?? 0, tried);
|
|
253
|
+
// Same key came back (single key, or all others already tried) → no point
|
|
254
|
+
// looping further; do the attempt and let any error propagate normally.
|
|
255
|
+
const isLastUsableKey = tried.has(keyEntry.id);
|
|
256
|
+
tried.add(keyEntry.id);
|
|
257
|
+
try {
|
|
258
|
+
const operation = await this.withTransientRetry("generateVideos", () => keyEntry.client.models.generateVideos(request));
|
|
259
|
+
if (!operation || !operation.name) {
|
|
260
|
+
throw new Error("Failed to initiate video generation task");
|
|
261
|
+
}
|
|
262
|
+
// Tag the task with the issuing key so poll + download re-select the
|
|
263
|
+
// same (project-scoped) client. Single-key pool → bare name (legacy).
|
|
264
|
+
return {
|
|
265
|
+
task: (0, googleApiKeys_1.encodeVeoTask)(keyEntry.id, operation.name, this.keyPool.size),
|
|
266
|
+
status: types_1.EVideoSceneStatus.TRIGGERED,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
catch (err) {
|
|
270
|
+
lastErr = err;
|
|
271
|
+
const keyFailure = (0, googleErrors_1.classifyGoogleKeyHealth)(err);
|
|
272
|
+
// Not an account-level failure (e.g. content rejection, bad params,
|
|
273
|
+
// transient RPM 429) → don't burn other accounts on it; surface as-is.
|
|
274
|
+
if (!keyFailure)
|
|
275
|
+
throw err;
|
|
276
|
+
// Account-level failure: circuit-break this key and try the next one.
|
|
277
|
+
// (No operation was created above, so this account wasn't charged.)
|
|
278
|
+
await this.keyPool.markDepleted(keyEntry.id, keyFailure.ttlSeconds, keyFailure.reason);
|
|
279
|
+
logger_1.logger.warn("Google Veo: key account failure — failing over", {
|
|
280
|
+
modelKey: params.modelKey,
|
|
281
|
+
keyId: keyEntry.id,
|
|
282
|
+
kind: keyFailure.kind,
|
|
283
|
+
attempt: attempt + 1,
|
|
284
|
+
poolSize: this.keyPool.size,
|
|
285
|
+
});
|
|
286
|
+
if (isLastUsableKey || tried.size >= this.keyPool.size) {
|
|
287
|
+
// No untried key left — EVERY Google billing account is down. This is
|
|
288
|
+
// a platform-wide outage (the 2026-06-18 Veo-depletion class), not a
|
|
289
|
+
// per-job blip: page Slack at the source with full context so on-call
|
|
290
|
+
// sees it immediately rather than relying on downstream classification.
|
|
291
|
+
logger_1.logger.error("Google Veo: ALL keys exhausted — every account failing over", {
|
|
292
|
+
modelKey: params.modelKey,
|
|
293
|
+
kind: keyFailure.kind,
|
|
294
|
+
reason: keyFailure.reason,
|
|
295
|
+
triedKeys: [...tried],
|
|
296
|
+
poolSize: this.keyPool.size,
|
|
297
|
+
});
|
|
298
|
+
// Propagate so the job surfaces a friendly error + refunds, and the
|
|
299
|
+
// job-level provider fallback can spill to another provider.
|
|
300
|
+
throw err;
|
|
301
|
+
}
|
|
302
|
+
// else: loop and pickForSubmit will skip `tried` + circuit-broken keys.
|
|
303
|
+
}
|
|
246
304
|
}
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
|
|
250
|
-
task: (0, googleApiKeys_1.encodeVeoTask)(keyEntry.id, operation.name, this.keyPool.size),
|
|
251
|
-
status: types_1.EVideoSceneStatus.TRIGGERED,
|
|
252
|
-
};
|
|
305
|
+
// Loop exhausted without returning (shouldn't happen — the throw above
|
|
306
|
+
// covers the no-key-left case) — surface the last error.
|
|
307
|
+
throw lastErr ?? new Error("Failed to initiate video generation task");
|
|
253
308
|
}
|
|
254
309
|
async checkVideoStatus({ task, outputFilename, outputFilePath = "videos", }) {
|
|
255
310
|
// Re-select the key that submitted this task (Veo operations are
|
|
@@ -472,7 +527,7 @@ class GoogleService extends baseAiGenProvider_service_1.BaseAiGenProviderService
|
|
|
472
527
|
async generateMusic(params) {
|
|
473
528
|
return (0, googleMusic_1.generateGoogleMusic)(params);
|
|
474
529
|
}
|
|
475
|
-
getCreditUsed({ modelKey, duration = 8, resolution = "720p", multiClip = false, numImages = 1, imageSize }) {
|
|
530
|
+
getCreditUsed({ modelKey, duration = 8, resolution = "720p", multiClip = false, numImages = 1, imageSize, creditFloor }) {
|
|
476
531
|
const modelConfig = aiModels_1.aiModelConfigs[modelKey];
|
|
477
532
|
const cost = modelConfig.cost;
|
|
478
533
|
const applyFloor = !multiClip;
|
|
@@ -481,11 +536,11 @@ class GoogleService extends baseAiGenProvider_service_1.BaseAiGenProviderService
|
|
|
481
536
|
if (cost?.perImageSize) {
|
|
482
537
|
const key = imageSize ?? "2K";
|
|
483
538
|
const perImage = cost.perImageSize[key] ?? cost.fixed ?? 0;
|
|
484
|
-
return (0, helpers_1.getCreditsFromCost)(perImage * Math.max(1, numImages), applyFloor);
|
|
539
|
+
return (0, helpers_1.getCreditsFromCost)(perImage * Math.max(1, numImages), applyFloor, creditFloor);
|
|
485
540
|
}
|
|
486
541
|
// Image-gen flat per-image cost (Imagen / Nano Banana).
|
|
487
542
|
if (cost?.fixed !== undefined) {
|
|
488
|
-
return (0, helpers_1.getCreditsFromCost)(cost.fixed * Math.max(1, numImages), applyFloor);
|
|
543
|
+
return (0, helpers_1.getCreditsFromCost)(cost.fixed * Math.max(1, numImages), applyFloor, creditFloor);
|
|
489
544
|
}
|
|
490
545
|
if (cost?.perResolution) {
|
|
491
546
|
const ratePerSecond = cost.perResolution[resolution] ?? cost.perResolution["720p"];
|
|
@@ -10,4 +10,32 @@ import { UserFacingError } from "../../../../utils/errors";
|
|
|
10
10
|
* error instead of raw provider JSON that pages Slack as a platform bug.
|
|
11
11
|
*/
|
|
12
12
|
export declare function classifyGoogleApiError(err: any): UserFacingError | null;
|
|
13
|
+
/** A key-level (billing / auth) failure that warrants failing over to the next
|
|
14
|
+
* key in the pool and circuit-breaking the failed one for `ttlSeconds`. */
|
|
15
|
+
export interface GoogleKeyHealthFailure {
|
|
16
|
+
/** "billing" | "auth" — the class of account problem, for logs + TTL choice. */
|
|
17
|
+
kind: "billing" | "auth";
|
|
18
|
+
/** Short human reason, persisted as the circuit-breaker value (Loki/debug). */
|
|
19
|
+
reason: string;
|
|
20
|
+
/** How long to skip this key before re-probing it. */
|
|
21
|
+
ttlSeconds: number;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Classifies whether a Veo *submit* failure is an ACCOUNT-LEVEL problem with
|
|
25
|
+
* THIS key (billing exhausted / auth revoked) — distinct from a transient
|
|
26
|
+
* per-minute rate limit or a content/validation error.
|
|
27
|
+
*
|
|
28
|
+
* Returns a failure descriptor when the pool should fail over to the next key
|
|
29
|
+
* and circuit-break this one; null otherwise (caller handles normally).
|
|
30
|
+
*
|
|
31
|
+
* IMPORTANT (cost safety): this is only consulted on a THROWN submit — i.e. no
|
|
32
|
+
* Veo operation was created, so the account was NOT billed. Failing over to a
|
|
33
|
+
* second billing account therefore cannot double-charge for the same job.
|
|
34
|
+
*
|
|
35
|
+
* Deliberately does NOT match a bare per-minute 429 / RESOURCE_EXHAUSTED with
|
|
36
|
+
* no billing signal: that's our own RPM cap, not an account outage, and the
|
|
37
|
+
* existing quota router + provider-fallback chain already handle it. Failing
|
|
38
|
+
* over on every RPM blip would needlessly drain the reserve account.
|
|
39
|
+
*/
|
|
40
|
+
export declare function classifyGoogleKeyHealth(err: any): GoogleKeyHealthFailure | null;
|
|
13
41
|
//# sourceMappingURL=googleErrors.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"googleErrors.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/google/googleErrors.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"googleErrors.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/google/googleErrors.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EAGhB,MAAM,0BAA0B,CAAC;AAElC;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,GAAG,GAAG,eAAe,GAAG,IAAI,CA4IvE;AAED;4EAC4E;AAC5E,MAAM,WAAW,sBAAsB;IACrC,gFAAgF;IAChF,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IACzB,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;CACpB;AAUD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,GAAG,GAAG,sBAAsB,GAAG,IAAI,CA4C/E"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.classifyGoogleApiError = classifyGoogleApiError;
|
|
4
|
+
exports.classifyGoogleKeyHealth = classifyGoogleKeyHealth;
|
|
4
5
|
const errors_1 = require("../../../../utils/errors");
|
|
5
6
|
/**
|
|
6
7
|
* Maps `@google/genai` `ApiError` failures (whose message is a JSON string)
|
|
@@ -20,12 +21,25 @@ function classifyGoogleApiError(err) {
|
|
|
20
21
|
const httpCode = inner.code;
|
|
21
22
|
const status = inner.status;
|
|
22
23
|
const msg = inner.message ?? raw;
|
|
24
|
+
// Billing depletion / billing-not-enabled. Veo surfaces this as
|
|
25
|
+
// RESOURCE_EXHAUSTED / 429 but with a billing-specific message ("Your
|
|
26
|
+
// prepayment credits are depleted… manage your project and billing") or as
|
|
27
|
+
// FAILED_PRECONDITION. This is an ACCOUNT problem, NOT a per-minute rate
|
|
28
|
+
// limit — give it the distinct ACCOUNT_QUOTA_EXCEEDED code (which DOES page
|
|
29
|
+
// Slack) and a SAFE message. Never echo the raw provider text: it tells the
|
|
30
|
+
// end user to go manage OUR Google AI Studio billing (the bug we're fixing).
|
|
31
|
+
const billingSignal = /prepayment credits? (?:are|is) depleted|credits? (?:are|is) depleted|\bbilling\b|free tier is not available|check your plan and billing/i.test(msg) || status === "FAILED_PRECONDITION";
|
|
32
|
+
if (billingSignal) {
|
|
33
|
+
return new errors_1.UserFacingError((0, errors_1.friendlyMessageForCode)(errors_1.USER_FACING_ERROR_CODES.ACCOUNT_QUOTA_EXCEEDED), errors_1.USER_FACING_ERROR_CODES.ACCOUNT_QUOTA_EXCEEDED);
|
|
34
|
+
}
|
|
23
35
|
if (status === "RESOURCE_EXHAUSTED" || httpCode === 429) {
|
|
24
|
-
|
|
36
|
+
// Transient per-minute / per-day rate limit. Safe canned message — the raw
|
|
37
|
+
// provider text can still carry billing/console URLs we must not show.
|
|
38
|
+
return new errors_1.UserFacingError((0, errors_1.friendlyMessageForCode)(errors_1.USER_FACING_ERROR_CODES.VIDEO_PROVIDER_RATE_LIMITED), errors_1.USER_FACING_ERROR_CODES.VIDEO_PROVIDER_RATE_LIMITED);
|
|
25
39
|
}
|
|
26
40
|
// gRPC code 14 = UNAVAILABLE; Veo surfaces "high demand" failures with this.
|
|
27
41
|
if (httpCode === 14 || /high demand/i.test(msg)) {
|
|
28
|
-
return new errors_1.UserFacingError(
|
|
42
|
+
return new errors_1.UserFacingError((0, errors_1.friendlyMessageForCode)(errors_1.USER_FACING_ERROR_CODES.VIDEO_PROVIDER_HIGH_DEMAND), errors_1.USER_FACING_ERROR_CODES.VIDEO_PROVIDER_HIGH_DEMAND);
|
|
29
43
|
}
|
|
30
44
|
// INVALID_ARGUMENT 400 — narrow match: only the specific "use case is
|
|
31
45
|
// currently not supported" string, which Veo returns when our request
|
|
@@ -100,3 +114,62 @@ function classifyGoogleApiError(err) {
|
|
|
100
114
|
}
|
|
101
115
|
return null;
|
|
102
116
|
}
|
|
117
|
+
// Re-probe a billing-depleted key every 30 min: long enough to stop hammering a
|
|
118
|
+
// dead account (and to route the whole pipeline onto the healthy key), short
|
|
119
|
+
// enough that a top-up recovers within half an hour. Auth/permission problems
|
|
120
|
+
// (revoked / not-yet-propagated key, disabled API) re-probe faster — they're
|
|
121
|
+
// often a transient config rollout — at 15 min.
|
|
122
|
+
const BILLING_CIRCUIT_TTL_S = 30 * 60;
|
|
123
|
+
const AUTH_CIRCUIT_TTL_S = 15 * 60;
|
|
124
|
+
/**
|
|
125
|
+
* Classifies whether a Veo *submit* failure is an ACCOUNT-LEVEL problem with
|
|
126
|
+
* THIS key (billing exhausted / auth revoked) — distinct from a transient
|
|
127
|
+
* per-minute rate limit or a content/validation error.
|
|
128
|
+
*
|
|
129
|
+
* Returns a failure descriptor when the pool should fail over to the next key
|
|
130
|
+
* and circuit-break this one; null otherwise (caller handles normally).
|
|
131
|
+
*
|
|
132
|
+
* IMPORTANT (cost safety): this is only consulted on a THROWN submit — i.e. no
|
|
133
|
+
* Veo operation was created, so the account was NOT billed. Failing over to a
|
|
134
|
+
* second billing account therefore cannot double-charge for the same job.
|
|
135
|
+
*
|
|
136
|
+
* Deliberately does NOT match a bare per-minute 429 / RESOURCE_EXHAUSTED with
|
|
137
|
+
* no billing signal: that's our own RPM cap, not an account outage, and the
|
|
138
|
+
* existing quota router + provider-fallback chain already handle it. Failing
|
|
139
|
+
* over on every RPM blip would needlessly drain the reserve account.
|
|
140
|
+
*/
|
|
141
|
+
function classifyGoogleKeyHealth(err) {
|
|
142
|
+
const raw = err?.message ?? "";
|
|
143
|
+
let httpCode;
|
|
144
|
+
let status;
|
|
145
|
+
let msg = raw;
|
|
146
|
+
try {
|
|
147
|
+
const parsed = JSON.parse(raw);
|
|
148
|
+
const inner = parsed.error ?? parsed;
|
|
149
|
+
httpCode = inner.code;
|
|
150
|
+
status = inner.status;
|
|
151
|
+
msg = inner.message ?? raw;
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
// Non-JSON message — match against the raw string below.
|
|
155
|
+
}
|
|
156
|
+
// Billing depletion. Veo surfaces this as RESOURCE_EXHAUSTED / 429 but with a
|
|
157
|
+
// billing-specific message ("Your prepayment credits are depleted… manage
|
|
158
|
+
// your project and billing"), OR as FAILED_PRECONDITION (free-tier/billing
|
|
159
|
+
// not enabled). Match the billing signal, NOT a bare 429.
|
|
160
|
+
if (/prepayment credits? (?:are|is) depleted|credits? (?:are|is) depleted|billing|free tier is not available|check your plan and billing|FAILED_PRECONDITION/i.test(msg) ||
|
|
161
|
+
status === "FAILED_PRECONDITION") {
|
|
162
|
+
return { kind: "billing", reason: `billing: ${msg.slice(0, 120)}`, ttlSeconds: BILLING_CIRCUIT_TTL_S };
|
|
163
|
+
}
|
|
164
|
+
// Auth / permission: key revoked, not yet propagated, API disabled, or the
|
|
165
|
+
// billing account suspended. These make every call on this key fail until
|
|
166
|
+
// fixed, so circuit-break + fail over rather than burn the whole job on it.
|
|
167
|
+
if (httpCode === 401 ||
|
|
168
|
+
httpCode === 403 ||
|
|
169
|
+
status === "PERMISSION_DENIED" ||
|
|
170
|
+
status === "UNAUTHENTICATED" ||
|
|
171
|
+
/API[_ ]key not valid|API_KEY_INVALID|permission denied|SERVICE_DISABLED|has not been used in project|consumer .* (?:suspended|disabled)/i.test(msg)) {
|
|
172
|
+
return { kind: "auth", reason: `auth: ${msg.slice(0, 120)}`, ttlSeconds: AUTH_CIRCUIT_TTL_S };
|
|
173
|
+
}
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
@@ -28,6 +28,13 @@ declare class GoogleKeyPool {
|
|
|
28
28
|
private readonly entries;
|
|
29
29
|
constructor();
|
|
30
30
|
get size(): number;
|
|
31
|
+
/** All key ids in priority order (for failover bookkeeping by the caller). */
|
|
32
|
+
get keyIds(): string[];
|
|
33
|
+
private depletedKey;
|
|
34
|
+
/** Circuit-break a key for `ttlSeconds` after an account-level submit failure. */
|
|
35
|
+
markDepleted(id: string, ttlSeconds: number, reason: string): Promise<void>;
|
|
36
|
+
/** Ids currently circuit-broken. Best-effort; empty on Redis trouble (fail-open). */
|
|
37
|
+
private depletedIds;
|
|
31
38
|
/** The highest-priority key's client (index 0). */
|
|
32
39
|
get primaryClient(): GoogleGenAI;
|
|
33
40
|
/** Resolve a tagged task's key id back to its client (undefined if unknown). */
|
|
@@ -41,7 +48,7 @@ declare class GoogleKeyPool {
|
|
|
41
48
|
* if Redis is down or all are at cap (the aggregate model-level gate should
|
|
42
49
|
* have prevented the latter).
|
|
43
50
|
*/
|
|
44
|
-
pickForSubmit(modelId: string, baselineMin: number, baselineDay: number): Promise<KeyEntry>;
|
|
51
|
+
pickForSubmit(modelId: string, baselineMin: number, baselineDay: number, skip?: Set<string>): Promise<KeyEntry>;
|
|
45
52
|
/** Best-effort increment of a key's per-day + per-minute routing counters. */
|
|
46
53
|
private consume;
|
|
47
54
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"googleKeyPool.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/google/googleKeyPool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAG5C,OAAO,EACL,iBAAiB,EAKlB,MAAM,iBAAiB,CAAC;AAEzB;;;;;;;;;;;;;;;;GAgBG;AAEH,UAAU,QAAQ;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC;IACpB,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;CACd;AAcD,cAAM,aAAa;IACjB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;;IA4BrC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,mDAAmD;IACnD,IAAI,aAAa,IAAI,WAAW,CAE/B;IAED,gFAAgF;IAChF,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS;IAK3D;;;;;;;;OAQG;IACG,aAAa,CACjB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"googleKeyPool.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/google/googleKeyPool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAG5C,OAAO,EACL,iBAAiB,EAKlB,MAAM,iBAAiB,CAAC;AAEzB;;;;;;;;;;;;;;;;GAgBG;AAEH,UAAU,QAAQ;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC;IACpB,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;CACd;AAcD,cAAM,aAAa;IACjB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;;IA4BrC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,8EAA8E;IAC9E,IAAI,MAAM,IAAI,MAAM,EAAE,CAErB;IAUD,OAAO,CAAC,WAAW;IAInB,kFAAkF;IAC5E,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBjF,qFAAqF;YACvE,WAAW;IAiBzB,mDAAmD;IACnD,IAAI,aAAa,IAAI,WAAW,CAE/B;IAED,gFAAgF;IAChF,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS;IAK3D;;;;;;;;OAQG;IACG,aAAa,CACjB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,IAAI,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GACjB,OAAO,CAAC,QAAQ,CAAC;IA2DpB,8EAA8E;YAChE,OAAO;CAUtB;AAID,sCAAsC;AACtC,wBAAgB,gBAAgB,IAAI,aAAa,CAGhD;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC;AAC7B,YAAY,EAAE,aAAa,EAAE,CAAC"}
|
|
@@ -44,6 +44,59 @@ class GoogleKeyPool {
|
|
|
44
44
|
get size() {
|
|
45
45
|
return this.entries.length;
|
|
46
46
|
}
|
|
47
|
+
/** All key ids in priority order (for failover bookkeeping by the caller). */
|
|
48
|
+
get keyIds() {
|
|
49
|
+
return this.entries.map((e) => e.id);
|
|
50
|
+
}
|
|
51
|
+
// ── Circuit breaker ────────────────────────────────────────────────────────
|
|
52
|
+
// A key whose BILLING is depleted (or whose auth is revoked) keeps returning
|
|
53
|
+
// the same error on every submit, but our quota counters (which only track
|
|
54
|
+
// OUR request rate) have no way to know that — so routing would keep sending
|
|
55
|
+
// jobs into the dead key. When generateVideo's submit fails with an account-
|
|
56
|
+
// level error it calls markDepleted(); pickForSubmit then skips the key until
|
|
57
|
+
// the TTL lapses and we re-probe it. The flag lives in Redis so every worker
|
|
58
|
+
// shares one view (and it auto-clears if Redis is wiped — fail-open).
|
|
59
|
+
depletedKey(id) {
|
|
60
|
+
return `gkpool:${id}:depleted`;
|
|
61
|
+
}
|
|
62
|
+
/** Circuit-break a key for `ttlSeconds` after an account-level submit failure. */
|
|
63
|
+
async markDepleted(id, ttlSeconds, reason) {
|
|
64
|
+
const client = redis_service_1.redis.getClient();
|
|
65
|
+
if (!client)
|
|
66
|
+
return; // no Redis → fail-open (caller still fails over in-loop)
|
|
67
|
+
try {
|
|
68
|
+
await client.set(this.depletedKey(id), reason, "EX", Math.max(60, ttlSeconds));
|
|
69
|
+
logger_1.logger.warn("googleKeyPool: key circuit-broken (skipping until re-probe)", {
|
|
70
|
+
keyId: id,
|
|
71
|
+
ttlSeconds,
|
|
72
|
+
reason,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
logger_1.logger.warn("googleKeyPool: markDepleted failed", {
|
|
77
|
+
keyId: id,
|
|
78
|
+
err: err instanceof Error ? err.message : String(err),
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/** Ids currently circuit-broken. Best-effort; empty on Redis trouble (fail-open). */
|
|
83
|
+
async depletedIds() {
|
|
84
|
+
const out = new Set();
|
|
85
|
+
const client = redis_service_1.redis.getClient();
|
|
86
|
+
if (!client || this.entries.length <= 1)
|
|
87
|
+
return out;
|
|
88
|
+
try {
|
|
89
|
+
const flags = await Promise.all(this.entries.map((e) => client.get(this.depletedKey(e.id))));
|
|
90
|
+
this.entries.forEach((e, i) => {
|
|
91
|
+
if (flags[i])
|
|
92
|
+
out.add(e.id);
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
// fail-open: treat all as healthy
|
|
97
|
+
}
|
|
98
|
+
return out;
|
|
99
|
+
}
|
|
47
100
|
/** The highest-priority key's client (index 0). */
|
|
48
101
|
get primaryClient() {
|
|
49
102
|
return this.entries[0].client;
|
|
@@ -63,14 +116,25 @@ class GoogleKeyPool {
|
|
|
63
116
|
* if Redis is down or all are at cap (the aggregate model-level gate should
|
|
64
117
|
* have prevented the latter).
|
|
65
118
|
*/
|
|
66
|
-
async pickForSubmit(modelId, baselineMin, baselineDay) {
|
|
119
|
+
async pickForSubmit(modelId, baselineMin, baselineDay, skip) {
|
|
67
120
|
if (this.entries.length === 1)
|
|
68
121
|
return this.entries[0];
|
|
69
122
|
const client = redis_service_1.redis.getClient();
|
|
70
|
-
if (!client)
|
|
71
|
-
|
|
123
|
+
if (!client) {
|
|
124
|
+
// No Redis routing — honor the caller's failover skip set if possible.
|
|
125
|
+
return this.entries.find((e) => !skip?.has(e.id)) ?? this.entries[0];
|
|
126
|
+
}
|
|
127
|
+
// Exclude keys the caller already tried this job (failover) AND keys that
|
|
128
|
+
// are circuit-broken (depleted billing / revoked auth). If that leaves
|
|
129
|
+
// nothing, fall back to the full list so we always return *some* key.
|
|
130
|
+
const depleted = await this.depletedIds();
|
|
131
|
+
const excluded = new Set([...(skip ?? []), ...depleted]);
|
|
132
|
+
const pool = this.entries.filter((e) => !excluded.has(e.id));
|
|
133
|
+
const candidates = pool.length ? pool : this.entries;
|
|
134
|
+
if (candidates.length === 1)
|
|
135
|
+
return candidates[0];
|
|
72
136
|
const date = utcDateKey();
|
|
73
|
-
for (const entry of
|
|
137
|
+
for (const entry of candidates) {
|
|
74
138
|
try {
|
|
75
139
|
const perKeyMinLimit = (0, googleApiKeys_1.scaleLimitForTier)(baselineMin, entry.tier, "rpm");
|
|
76
140
|
const perKeyDayLimit = (0, googleApiKeys_1.scaleLimitForTier)(baselineDay, entry.tier, "rpd");
|
|
@@ -96,14 +160,18 @@ class GoogleKeyPool {
|
|
|
96
160
|
});
|
|
97
161
|
}
|
|
98
162
|
}
|
|
99
|
-
// All at cap / errored — use the
|
|
100
|
-
//
|
|
101
|
-
|
|
163
|
+
// All at cap / errored — use the highest-priority *candidate* (already
|
|
164
|
+
// excludes failover-tried + circuit-broken keys) and let Google's own 429 +
|
|
165
|
+
// the provider-fallback chain handle it.
|
|
166
|
+
const fallback = candidates[0];
|
|
167
|
+
logger_1.logger.warn("googleKeyPool: no key with headroom — using best candidate", {
|
|
102
168
|
modelId,
|
|
103
169
|
poolSize: this.entries.length,
|
|
170
|
+
candidates: candidates.length,
|
|
171
|
+
keyId: fallback.id,
|
|
104
172
|
});
|
|
105
|
-
await this.consume(
|
|
106
|
-
return
|
|
173
|
+
await this.consume(fallback, modelId, date).catch(() => undefined);
|
|
174
|
+
return fallback;
|
|
107
175
|
}
|
|
108
176
|
/** Best-effort increment of a key's per-day + per-minute routing counters. */
|
|
109
177
|
async consume(entry, modelId, date) {
|