venice-video-harness 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/LICENSE +21 -0
- package/README.md +598 -0
- package/dist/assembly/manifest-builder.d.ts +8 -0
- package/dist/assembly/manifest-builder.d.ts.map +1 -0
- package/dist/assembly/manifest-builder.js +196 -0
- package/dist/assembly/manifest-builder.js.map +1 -0
- package/dist/assembly/remotion-scaffold.d.ts +14 -0
- package/dist/assembly/remotion-scaffold.d.ts.map +1 -0
- package/dist/assembly/remotion-scaffold.js +292 -0
- package/dist/assembly/remotion-scaffold.js.map +1 -0
- package/dist/assembly/types.d.ts +67 -0
- package/dist/assembly/types.d.ts.map +1 -0
- package/dist/assembly/types.js +5 -0
- package/dist/assembly/types.js.map +1 -0
- package/dist/characters/describer.d.ts +32 -0
- package/dist/characters/describer.d.ts.map +1 -0
- package/dist/characters/describer.js +309 -0
- package/dist/characters/describer.js.map +1 -0
- package/dist/characters/extractor.d.ts +27 -0
- package/dist/characters/extractor.d.ts.map +1 -0
- package/dist/characters/extractor.js +274 -0
- package/dist/characters/extractor.js.map +1 -0
- package/dist/characters/reference-manager.d.ts +95 -0
- package/dist/characters/reference-manager.d.ts.map +1 -0
- package/dist/characters/reference-manager.js +346 -0
- package/dist/characters/reference-manager.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +310 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +22 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +66 -0
- package/dist/config.js.map +1 -0
- package/dist/editing/aligner.d.ts +44 -0
- package/dist/editing/aligner.d.ts.map +1 -0
- package/dist/editing/aligner.js +201 -0
- package/dist/editing/aligner.js.map +1 -0
- package/dist/editing/edl.d.ts +45 -0
- package/dist/editing/edl.d.ts.map +1 -0
- package/dist/editing/edl.js +133 -0
- package/dist/editing/edl.js.map +1 -0
- package/dist/editing/overlays.d.ts +97 -0
- package/dist/editing/overlays.d.ts.map +1 -0
- package/dist/editing/overlays.js +41 -0
- package/dist/editing/overlays.js.map +1 -0
- package/dist/editing/packer.d.ts +52 -0
- package/dist/editing/packer.d.ts.map +1 -0
- package/dist/editing/packer.js +170 -0
- package/dist/editing/packer.js.map +1 -0
- package/dist/editing/providers/whisper-cpp.d.ts +32 -0
- package/dist/editing/providers/whisper-cpp.d.ts.map +1 -0
- package/dist/editing/providers/whisper-cpp.js +239 -0
- package/dist/editing/providers/whisper-cpp.js.map +1 -0
- package/dist/editing/render.d.ts +36 -0
- package/dist/editing/render.d.ts.map +1 -0
- package/dist/editing/render.js +227 -0
- package/dist/editing/render.js.map +1 -0
- package/dist/editing/self-eval.d.ts +48 -0
- package/dist/editing/self-eval.d.ts.map +1 -0
- package/dist/editing/self-eval.js +293 -0
- package/dist/editing/self-eval.js.map +1 -0
- package/dist/editing/silence.d.ts +76 -0
- package/dist/editing/silence.d.ts.map +1 -0
- package/dist/editing/silence.js +189 -0
- package/dist/editing/silence.js.map +1 -0
- package/dist/editing/types.d.ts +178 -0
- package/dist/editing/types.d.ts.map +1 -0
- package/dist/editing/types.js +13 -0
- package/dist/editing/types.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/interactive.d.ts +12 -0
- package/dist/interactive.d.ts.map +1 -0
- package/dist/interactive.js +78 -0
- package/dist/interactive.js.map +1 -0
- package/dist/mini-drama/assembler.d.ts +61 -0
- package/dist/mini-drama/assembler.d.ts.map +1 -0
- package/dist/mini-drama/assembler.js +466 -0
- package/dist/mini-drama/assembler.js.map +1 -0
- package/dist/mini-drama/audio-mix.d.ts +65 -0
- package/dist/mini-drama/audio-mix.d.ts.map +1 -0
- package/dist/mini-drama/audio-mix.js +118 -0
- package/dist/mini-drama/audio-mix.js.map +1 -0
- package/dist/mini-drama/cli.d.ts +3 -0
- package/dist/mini-drama/cli.d.ts.map +1 -0
- package/dist/mini-drama/cli.js +2869 -0
- package/dist/mini-drama/cli.js.map +1 -0
- package/dist/mini-drama/fcpxml-export.d.ts +2 -0
- package/dist/mini-drama/fcpxml-export.d.ts.map +1 -0
- package/dist/mini-drama/fcpxml-export.js +5 -0
- package/dist/mini-drama/fcpxml-export.js.map +1 -0
- package/dist/mini-drama/generation-planner.d.ts +24 -0
- package/dist/mini-drama/generation-planner.d.ts.map +1 -0
- package/dist/mini-drama/generation-planner.js +304 -0
- package/dist/mini-drama/generation-planner.js.map +1 -0
- package/dist/mini-drama/location-generator.d.ts +23 -0
- package/dist/mini-drama/location-generator.d.ts.map +1 -0
- package/dist/mini-drama/location-generator.js +144 -0
- package/dist/mini-drama/location-generator.js.map +1 -0
- package/dist/mini-drama/music-cues.d.ts +96 -0
- package/dist/mini-drama/music-cues.d.ts.map +1 -0
- package/dist/mini-drama/music-cues.js +332 -0
- package/dist/mini-drama/music-cues.js.map +1 -0
- package/dist/mini-drama/panel-fixer.d.ts +12 -0
- package/dist/mini-drama/panel-fixer.d.ts.map +1 -0
- package/dist/mini-drama/panel-fixer.js +319 -0
- package/dist/mini-drama/panel-fixer.js.map +1 -0
- package/dist/mini-drama/prompt-builder.d.ts +149 -0
- package/dist/mini-drama/prompt-builder.d.ts.map +1 -0
- package/dist/mini-drama/prompt-builder.js +722 -0
- package/dist/mini-drama/prompt-builder.js.map +1 -0
- package/dist/mini-drama/reference-slots.d.ts +31 -0
- package/dist/mini-drama/reference-slots.d.ts.map +1 -0
- package/dist/mini-drama/reference-slots.js +161 -0
- package/dist/mini-drama/reference-slots.js.map +1 -0
- package/dist/mini-drama/shot-paths.d.ts +77 -0
- package/dist/mini-drama/shot-paths.d.ts.map +1 -0
- package/dist/mini-drama/shot-paths.js +101 -0
- package/dist/mini-drama/shot-paths.js.map +1 -0
- package/dist/mini-drama/storyboard-reference-generator.d.ts +47 -0
- package/dist/mini-drama/storyboard-reference-generator.d.ts.map +1 -0
- package/dist/mini-drama/storyboard-reference-generator.js +293 -0
- package/dist/mini-drama/storyboard-reference-generator.js.map +1 -0
- package/dist/mini-drama/subtitle-generator.d.ts +12 -0
- package/dist/mini-drama/subtitle-generator.d.ts.map +1 -0
- package/dist/mini-drama/subtitle-generator.js +139 -0
- package/dist/mini-drama/subtitle-generator.js.map +1 -0
- package/dist/mini-drama/timeline-export/davinci-fcpxml.d.ts +3 -0
- package/dist/mini-drama/timeline-export/davinci-fcpxml.d.ts.map +1 -0
- package/dist/mini-drama/timeline-export/davinci-fcpxml.js +150 -0
- package/dist/mini-drama/timeline-export/davinci-fcpxml.js.map +1 -0
- package/dist/mini-drama/timeline-export/fcpxml.d.ts +11 -0
- package/dist/mini-drama/timeline-export/fcpxml.d.ts.map +1 -0
- package/dist/mini-drama/timeline-export/fcpxml.js +140 -0
- package/dist/mini-drama/timeline-export/fcpxml.js.map +1 -0
- package/dist/mini-drama/timeline-export/index.d.ts +13 -0
- package/dist/mini-drama/timeline-export/index.d.ts.map +1 -0
- package/dist/mini-drama/timeline-export/index.js +26 -0
- package/dist/mini-drama/timeline-export/index.js.map +1 -0
- package/dist/mini-drama/timeline-export/premiere-xmeml.d.ts +3 -0
- package/dist/mini-drama/timeline-export/premiere-xmeml.d.ts.map +1 -0
- package/dist/mini-drama/timeline-export/premiere-xmeml.js +211 -0
- package/dist/mini-drama/timeline-export/premiere-xmeml.js.map +1 -0
- package/dist/mini-drama/timeline-export/probe.d.ts +53 -0
- package/dist/mini-drama/timeline-export/probe.d.ts.map +1 -0
- package/dist/mini-drama/timeline-export/probe.js +89 -0
- package/dist/mini-drama/timeline-export/probe.js.map +1 -0
- package/dist/mini-drama/timeline-export/types.d.ts +57 -0
- package/dist/mini-drama/timeline-export/types.d.ts.map +1 -0
- package/dist/mini-drama/timeline-export/types.js +14 -0
- package/dist/mini-drama/timeline-export/types.js.map +1 -0
- package/dist/mini-drama/video-generator.d.ts +22 -0
- package/dist/mini-drama/video-generator.d.ts.map +1 -0
- package/dist/mini-drama/video-generator.js +1305 -0
- package/dist/mini-drama/video-generator.js.map +1 -0
- package/dist/mini-drama/voice-reference.d.ts +42 -0
- package/dist/mini-drama/voice-reference.d.ts.map +1 -0
- package/dist/mini-drama/voice-reference.js +170 -0
- package/dist/mini-drama/voice-reference.js.map +1 -0
- package/dist/output/html-renderer.d.ts +2 -0
- package/dist/output/html-renderer.d.ts.map +1 -0
- package/dist/output/html-renderer.js +37 -0
- package/dist/output/html-renderer.js.map +1 -0
- package/dist/parsers/fountain-parser.d.ts +34 -0
- package/dist/parsers/fountain-parser.d.ts.map +1 -0
- package/dist/parsers/fountain-parser.js +71 -0
- package/dist/parsers/fountain-parser.js.map +1 -0
- package/dist/parsers/pdf-parser.d.ts +24 -0
- package/dist/parsers/pdf-parser.d.ts.map +1 -0
- package/dist/parsers/pdf-parser.js +237 -0
- package/dist/parsers/pdf-parser.js.map +1 -0
- package/dist/parsers/scene-extractor.d.ts +41 -0
- package/dist/parsers/scene-extractor.d.ts.map +1 -0
- package/dist/parsers/scene-extractor.js +400 -0
- package/dist/parsers/scene-extractor.js.map +1 -0
- package/dist/series/manager.d.ts +64 -0
- package/dist/series/manager.d.ts.map +1 -0
- package/dist/series/manager.js +282 -0
- package/dist/series/manager.js.map +1 -0
- package/dist/series/types.d.ts +641 -0
- package/dist/series/types.d.ts.map +1 -0
- package/dist/series/types.js +304 -0
- package/dist/series/types.js.map +1 -0
- package/dist/storyboard/assembler.d.ts +134 -0
- package/dist/storyboard/assembler.d.ts.map +1 -0
- package/dist/storyboard/assembler.js +319 -0
- package/dist/storyboard/assembler.js.map +1 -0
- package/dist/storyboard/prompt-builder.d.ts +113 -0
- package/dist/storyboard/prompt-builder.d.ts.map +1 -0
- package/dist/storyboard/prompt-builder.js +578 -0
- package/dist/storyboard/prompt-builder.js.map +1 -0
- package/dist/storyboard/shot-planner.d.ts +34 -0
- package/dist/storyboard/shot-planner.d.ts.map +1 -0
- package/dist/storyboard/shot-planner.js +446 -0
- package/dist/storyboard/shot-planner.js.map +1 -0
- package/dist/user-config.d.ts +15 -0
- package/dist/user-config.d.ts.map +1 -0
- package/dist/user-config.js +103 -0
- package/dist/user-config.js.map +1 -0
- package/dist/venice/audio-preflight.d.ts +80 -0
- package/dist/venice/audio-preflight.d.ts.map +1 -0
- package/dist/venice/audio-preflight.js +142 -0
- package/dist/venice/audio-preflight.js.map +1 -0
- package/dist/venice/audio.d.ts +88 -0
- package/dist/venice/audio.d.ts.map +1 -0
- package/dist/venice/audio.js +234 -0
- package/dist/venice/audio.js.map +1 -0
- package/dist/venice/client.d.ts +86 -0
- package/dist/venice/client.d.ts.map +1 -0
- package/dist/venice/client.js +375 -0
- package/dist/venice/client.js.map +1 -0
- package/dist/venice/edit.d.ts +57 -0
- package/dist/venice/edit.d.ts.map +1 -0
- package/dist/venice/edit.js +100 -0
- package/dist/venice/edit.js.map +1 -0
- package/dist/venice/generate.d.ts +47 -0
- package/dist/venice/generate.d.ts.map +1 -0
- package/dist/venice/generate.js +150 -0
- package/dist/venice/generate.js.map +1 -0
- package/dist/venice/image-bytes.d.ts +26 -0
- package/dist/venice/image-bytes.d.ts.map +1 -0
- package/dist/venice/image-bytes.js +101 -0
- package/dist/venice/image-bytes.js.map +1 -0
- package/dist/venice/models.d.ts +143 -0
- package/dist/venice/models.d.ts.map +1 -0
- package/dist/venice/models.js +984 -0
- package/dist/venice/models.js.map +1 -0
- package/dist/venice/multi-edit.d.ts +40 -0
- package/dist/venice/multi-edit.d.ts.map +1 -0
- package/dist/venice/multi-edit.js +97 -0
- package/dist/venice/multi-edit.js.map +1 -0
- package/dist/venice/provenance.d.ts +66 -0
- package/dist/venice/provenance.d.ts.map +1 -0
- package/dist/venice/provenance.js +151 -0
- package/dist/venice/provenance.js.map +1 -0
- package/dist/venice/recipe.d.ts +69 -0
- package/dist/venice/recipe.d.ts.map +1 -0
- package/dist/venice/recipe.js +103 -0
- package/dist/venice/recipe.js.map +1 -0
- package/dist/venice/rejection.d.ts +63 -0
- package/dist/venice/rejection.d.ts.map +1 -0
- package/dist/venice/rejection.js +111 -0
- package/dist/venice/rejection.js.map +1 -0
- package/dist/venice/seedance-preflight.d.ts +38 -0
- package/dist/venice/seedance-preflight.d.ts.map +1 -0
- package/dist/venice/seedance-preflight.js +29 -0
- package/dist/venice/seedance-preflight.js.map +1 -0
- package/dist/venice/types.d.ts +212 -0
- package/dist/venice/types.d.ts.map +1 -0
- package/dist/venice/types.js +9 -0
- package/dist/venice/types.js.map +1 -0
- package/dist/venice/video.d.ts +77 -0
- package/dist/venice/video.d.ts.map +1 -0
- package/dist/venice/video.js +163 -0
- package/dist/venice/video.js.map +1 -0
- package/dist/venice/voices.d.ts +40 -0
- package/dist/venice/voices.d.ts.map +1 -0
- package/dist/venice/voices.js +194 -0
- package/dist/venice/voices.js.map +1 -0
- package/package.json +79 -0
|
@@ -0,0 +1,722 @@
|
|
|
1
|
+
import { VIDEO_NO_MUSIC_SUFFIX, FEMALE_BASE_TRAITS, MALE_BASE_TRAITS, KLING_MULTISHOT_MODEL, KLING_R2V_MODEL, DAYTIME_ENVIRONMENTS, MODELS_SUPPORTING_ELEMENTS, MODELS_SUPPORTING_REFERENCE_IMAGES, MODELS_SUPPORTING_SCENE_IMAGES, MODELS_SUPPORTING_REFERENCE_AUDIO, MODELS_USING_IMAGE_TAGS, DEFAULT_CHARACTER_CONSISTENCY_MODEL, getMaxReferenceImages, } from '../series/types.js';
|
|
2
|
+
import { parseShotDuration } from './generation-planner.js';
|
|
3
|
+
import { getMaxPositivePromptChars } from '../venice/models.js';
|
|
4
|
+
import { getLocation } from '../series/manager.js';
|
|
5
|
+
import { buildReferenceSlotPlan } from './reference-slots.js';
|
|
6
|
+
// The audio-suppression chunk keeps the video model from baking music,
|
|
7
|
+
// score, sound effects, or foley into the dialogue track. The harness adds
|
|
8
|
+
// music (musicCues / generate_music) and ambient/SFX (generate_ambient /
|
|
9
|
+
// mix_audio) in post — bakes-in fight the assembler's mix and can't be
|
|
10
|
+
// removed once they're in the output. Keep this in NEGATIVE_PROMPT for
|
|
11
|
+
// every shot type; the workshop system prompt also asks the script LLM to
|
|
12
|
+
// repeat it per-shot, but this is the always-on belt-and-braces.
|
|
13
|
+
const AUDIO_SUPPRESSION_NEGATIVE = 'background music, soundtrack, score, musical score, sound effects, sfx, foley, ' +
|
|
14
|
+
'orchestral hits, sound design, audio drops';
|
|
15
|
+
const NEGATIVE_PROMPT = 'comic panels, multiple panels, panel layout, panel borders, panel grid, speech bubbles, text bubbles, ' +
|
|
16
|
+
'manga panels, comic strip, storyboard grid, split screen, multiple frames, ' +
|
|
17
|
+
'deformed, blurry, bad anatomy, bad hands, extra fingers, mutation, ' +
|
|
18
|
+
'poorly drawn face, watermark, text, signature, low quality, ugly, ' +
|
|
19
|
+
'umbrella, holding umbrella, ' +
|
|
20
|
+
AUDIO_SUPPRESSION_NEGATIVE;
|
|
21
|
+
const NO_PEOPLE_NEGATIVE = NEGATIVE_PROMPT + ', people, person, human, figure, silhouette, crowd, pedestrian, bystander';
|
|
22
|
+
const CAMERA_TERMS = {
|
|
23
|
+
'static': 'locked-off static shot',
|
|
24
|
+
'slow dolly forward': 'slow dolly shot pushing forward',
|
|
25
|
+
'slow dolly back': 'slow dolly shot pulling back',
|
|
26
|
+
'pan left': 'slow pan left',
|
|
27
|
+
'pan right': 'slow pan right',
|
|
28
|
+
'tilt up': 'tilt up',
|
|
29
|
+
'tilt down': 'tilt down',
|
|
30
|
+
'tracking': 'tracking shot following the subject',
|
|
31
|
+
'crane up': 'crane shot rising upward',
|
|
32
|
+
'handheld': 'handheld shot with subtle movement',
|
|
33
|
+
'zoom in': 'slow zoom in',
|
|
34
|
+
'zoom out': 'slow zoom out',
|
|
35
|
+
};
|
|
36
|
+
function getCharacterPromptText(char) {
|
|
37
|
+
const baseTraits = char.baseTraits ?? (char.gender === 'female' ? FEMALE_BASE_TRAITS : MALE_BASE_TRAITS);
|
|
38
|
+
return `${char.name} (${baseTraits}): ${char.fullDescription}`;
|
|
39
|
+
}
|
|
40
|
+
function buildAestheticString(aesthetic) {
|
|
41
|
+
return [aesthetic.style, aesthetic.palette, aesthetic.lighting, aesthetic.lensCharacteristics, `shot on ${aesthetic.filmStock}`]
|
|
42
|
+
.filter(Boolean)
|
|
43
|
+
.join(', ');
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Determines if a shot should use daytime aesthetics. Uses the explicit
|
|
47
|
+
* `environment` field when set; falls back to panelDescription heuristics
|
|
48
|
+
* for backwards compatibility with scripts that don't have it yet.
|
|
49
|
+
*/
|
|
50
|
+
function isDaytimeShot(shot) {
|
|
51
|
+
if (shot.environment) {
|
|
52
|
+
return DAYTIME_ENVIRONMENTS.has(shot.environment);
|
|
53
|
+
}
|
|
54
|
+
const sceneText = (shot.panelDescription ?? shot.description).toUpperCase();
|
|
55
|
+
return sceneText.includes('NO RAIN') || sceneText.includes('BRIGHT INDOOR') || sceneText.includes('DAYTIME');
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Strips rain, dark-sky, and wet-surface terms from the aesthetic string
|
|
59
|
+
* so that daytime scenes aren't contaminated by the series' default
|
|
60
|
+
* nighttime cyberpunk aesthetic.
|
|
61
|
+
*/
|
|
62
|
+
function stripDarkAesthetic(aestheticStr) {
|
|
63
|
+
return aestheticStr
|
|
64
|
+
.replace(/,?\s*rain rendered as[^,]*(?:,|$)/gi, ', ')
|
|
65
|
+
.replace(/,?\s*neon reflections on wet surfaces/gi, '')
|
|
66
|
+
.replace(/,?\s*volumetric light rays through rain/gi, ', soft volumetric light')
|
|
67
|
+
.replace(/,?\s*dark charcoal backgrounds\s*\([^)]*\)/gi, ', warm bright interior backgrounds')
|
|
68
|
+
.replace(/\s{2,}/g, ' ')
|
|
69
|
+
.replace(/,\s*,/g, ',')
|
|
70
|
+
.trim();
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Selects the video model for a shot. The core principle is simple:
|
|
74
|
+
*
|
|
75
|
+
* - **R2V by default** for all non-establishing shots (consistency first)
|
|
76
|
+
* - **Atmosphere model only** for truly empty establishing/mood shots
|
|
77
|
+
*
|
|
78
|
+
* R2V models accept `elements` and `reference_image_urls`, which are the
|
|
79
|
+
* only reliable way to maintain character identity across shots. Since the
|
|
80
|
+
* action model now defaults to R2V, almost all shots benefit from reference
|
|
81
|
+
* anchoring. The atmosphere model is reserved for truly empty
|
|
82
|
+
* establishing/insert shots with no characters on screen.
|
|
83
|
+
*
|
|
84
|
+
* When the resolved model supports elements or reference images, those
|
|
85
|
+
* capabilities are auto-enabled.
|
|
86
|
+
*/
|
|
87
|
+
export function resolveVideoModel(shot, series, _previousShot) {
|
|
88
|
+
const baseModel = shot.videoModel === 'action'
|
|
89
|
+
? series.videoDefaults.actionModel
|
|
90
|
+
: series.videoDefaults.atmosphereModel;
|
|
91
|
+
const consistencyModel = series.videoDefaults.characterConsistencyModel ?? DEFAULT_CHARACTER_CONSISTENCY_MODEL;
|
|
92
|
+
const hasCharacters = shot.characters.length > 0;
|
|
93
|
+
if (!hasCharacters) {
|
|
94
|
+
// With Enhanced R2V as the default for all lanes, even empty
|
|
95
|
+
// establishing/atmosphere shots can anchor to location references
|
|
96
|
+
// (@ImageN env tags) when the base model supports them.
|
|
97
|
+
return {
|
|
98
|
+
modelId: baseModel,
|
|
99
|
+
upgraded: false,
|
|
100
|
+
reason: 'no characters — atmosphere model (location refs when supported)',
|
|
101
|
+
autoUseElements: false,
|
|
102
|
+
autoUseReferenceImages: MODELS_SUPPORTING_REFERENCE_IMAGES.has(baseModel),
|
|
103
|
+
useImageTags: MODELS_USING_IMAGE_TAGS.has(baseModel),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
// dialogue shots whose speaker is a non-narrator visible
|
|
107
|
+
// character with low/medium motion route to Wan 2.7 i2v for lip-sync.
|
|
108
|
+
// High-motion dialogue stays on R2V because Wan 2.7 prioritizes motion
|
|
109
|
+
// over reference adherence (hair color shifts, shirt pattern simplifies,
|
|
110
|
+
// eyes change) when forced to handle big movement.
|
|
111
|
+
const lipSyncModel = series.videoDefaults.lipSyncModel;
|
|
112
|
+
if (lipSyncModel && shotWantsLipSync(shot) && shot.characters.length <= 1) {
|
|
113
|
+
return {
|
|
114
|
+
modelId: lipSyncModel,
|
|
115
|
+
upgraded: true,
|
|
116
|
+
reason: 'single-speaker dialogue, low/medium motion — Wan 2.7 lip-sync',
|
|
117
|
+
autoUseElements: false,
|
|
118
|
+
autoUseReferenceImages: false,
|
|
119
|
+
useImageTags: false,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
// Kling fallback only when the character count alone would overflow the
|
|
123
|
+
// flat-reference budget (leaving no room for location/storyboard refs).
|
|
124
|
+
// With the 9-image budget on Seedance 2.0 R2V / Enhanced, scenes with up
|
|
125
|
+
// to ~6 characters stay in-family; the old 3+ threshold dated from the
|
|
126
|
+
// 4-image era.
|
|
127
|
+
const refBudget = getMaxReferenceImages(consistencyModel);
|
|
128
|
+
const needsElementsFallback = shot.characters.length > Math.max(2, refBudget - 3)
|
|
129
|
+
&& MODELS_USING_IMAGE_TAGS.has(consistencyModel)
|
|
130
|
+
&& !MODELS_SUPPORTING_ELEMENTS.has(consistencyModel);
|
|
131
|
+
if (needsElementsFallback) {
|
|
132
|
+
return {
|
|
133
|
+
modelId: KLING_R2V_MODEL,
|
|
134
|
+
upgraded: true,
|
|
135
|
+
reason: `${shot.characters.length} characters overflow the ${refBudget}-reference budget — falling back to Kling O3 R2V for structured elements`,
|
|
136
|
+
autoUseElements: true,
|
|
137
|
+
autoUseReferenceImages: true,
|
|
138
|
+
useImageTags: false,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
return {
|
|
142
|
+
modelId: consistencyModel,
|
|
143
|
+
upgraded: consistencyModel !== baseModel,
|
|
144
|
+
reason: 'characters present — R2V for identity anchoring',
|
|
145
|
+
autoUseElements: MODELS_SUPPORTING_ELEMENTS.has(consistencyModel),
|
|
146
|
+
autoUseReferenceImages: MODELS_SUPPORTING_REFERENCE_IMAGES.has(consistencyModel),
|
|
147
|
+
useImageTags: MODELS_USING_IMAGE_TAGS.has(consistencyModel),
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* does this shot want lip-sync routing? A dialogue shot whose
|
|
152
|
+
* speaker is not the narrator and whose face is (or might be) visible,
|
|
153
|
+
* with motion not classified as 'high'.
|
|
154
|
+
*/
|
|
155
|
+
function shotWantsLipSync(shot) {
|
|
156
|
+
if (!shot.dialogue)
|
|
157
|
+
return false;
|
|
158
|
+
const speaker = shot.dialogue.character.toUpperCase();
|
|
159
|
+
if (speaker === 'NARRATOR' || speaker === 'V.O.' || speaker === 'VO')
|
|
160
|
+
return false;
|
|
161
|
+
if (shot.motion === 'high')
|
|
162
|
+
return false;
|
|
163
|
+
if (shot.faceVisible === false)
|
|
164
|
+
return false;
|
|
165
|
+
return true;
|
|
166
|
+
}
|
|
167
|
+
export function buildImagePrompt(shot, series) {
|
|
168
|
+
if (!series.aesthetic) {
|
|
169
|
+
throw new Error('Series aesthetic must be set before generating images.');
|
|
170
|
+
}
|
|
171
|
+
const isDaytime = isDaytimeShot(shot);
|
|
172
|
+
let aestheticStr = buildAestheticString(series.aesthetic);
|
|
173
|
+
if (isDaytime) {
|
|
174
|
+
aestheticStr = stripDarkAesthetic(aestheticStr);
|
|
175
|
+
}
|
|
176
|
+
const parts = [];
|
|
177
|
+
parts.push(`STYLE: ${aestheticStr}.`);
|
|
178
|
+
parts.push('Single cinematic frame, one continuous image, NOT a comic panel layout, NO panel borders, NO speech bubbles, NO text overlays.');
|
|
179
|
+
const isEmptyScene = shot.characters.length === 0;
|
|
180
|
+
parts.push('Characters are engaged in the scene, NOT looking at the camera.');
|
|
181
|
+
if (!isEmptyScene && shot.characters.length === 1) {
|
|
182
|
+
parts.push('This is NOT a portrait or headshot. The environment, props, and action are equally important as the character. Show the full scene composition with widescreen cinematic framing.');
|
|
183
|
+
}
|
|
184
|
+
parts.push(`Camera: ${shot.cameraMovement}.`);
|
|
185
|
+
parts.push(shot.panelDescription ?? shot.description);
|
|
186
|
+
// Silhouette characters appear in the panel but don't trigger R2V
|
|
187
|
+
if (shot.silhouetteCharacters && shot.silhouetteCharacters.length > 0) {
|
|
188
|
+
for (const charName of shot.silhouetteCharacters) {
|
|
189
|
+
const char = series.characters.find(c => c.name.toUpperCase() === charName.toUpperCase());
|
|
190
|
+
if (char) {
|
|
191
|
+
parts.push(`A distant silhouetted figure (${char.name}) is visible — seen from behind or at a distance, no face detail needed, identifiable by wardrobe: ${char.wardrobe}.`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
if (isEmptyScene && (!shot.silhouetteCharacters || shot.silhouetteCharacters.length === 0)) {
|
|
196
|
+
parts.push('Empty environment, no people present, no human figures, uninhabited scene.');
|
|
197
|
+
}
|
|
198
|
+
else if (isEmptyScene) {
|
|
199
|
+
// Has silhouette characters but no main characters — don't add "no people" directive
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
for (const charName of shot.characters) {
|
|
203
|
+
const char = series.characters.find(c => c.name.toUpperCase() === charName.toUpperCase());
|
|
204
|
+
if (char) {
|
|
205
|
+
const baseTraits = char.baseTraits ?? (char.gender === 'female' ? FEMALE_BASE_TRAITS : MALE_BASE_TRAITS);
|
|
206
|
+
const wardrobe = shot.episodeWardrobe?.[charName.toUpperCase()] ?? char.wardrobe;
|
|
207
|
+
parts.push(`${char.name} (${baseTraits}): ${char.description}, wearing ${wardrobe}.`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
parts.push(`STYLE REMINDER: ${aestheticStr}.`);
|
|
212
|
+
const seed = series.aestheticSeed ?? undefined;
|
|
213
|
+
const hasSilhouettes = shot.silhouetteCharacters && shot.silhouetteCharacters.length > 0;
|
|
214
|
+
let negativePrompt = (isEmptyScene && !hasSilhouettes) ? NO_PEOPLE_NEGATIVE : NEGATIVE_PROMPT;
|
|
215
|
+
if (isDaytime) {
|
|
216
|
+
negativePrompt += ', rain, rain streaks, wet surfaces, wet pavement, dark sky, storm, night sky, outdoor rain, neon reflections on wet ground';
|
|
217
|
+
}
|
|
218
|
+
return {
|
|
219
|
+
prompt: parts.join(' ').trim(),
|
|
220
|
+
negativePrompt,
|
|
221
|
+
seed,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
function getCharacterVideoTag(char) {
|
|
225
|
+
const key = char.gender === 'female'
|
|
226
|
+
? `${char.name}, ${char.age}, ${char.wardrobe}`
|
|
227
|
+
: `${char.name}, ${char.age}, ${char.wardrobe}`;
|
|
228
|
+
return key;
|
|
229
|
+
}
|
|
230
|
+
function buildCharacterAnchorText(characters) {
|
|
231
|
+
if (characters.length === 0)
|
|
232
|
+
return '';
|
|
233
|
+
const anchors = characters.map(char => {
|
|
234
|
+
const baseTraits = char.baseTraits ?? (char.gender === 'female' ? FEMALE_BASE_TRAITS : MALE_BASE_TRAITS);
|
|
235
|
+
return `${char.name}: ${baseTraits}, ${char.fullDescription}, wearing ${char.wardrobe}`;
|
|
236
|
+
});
|
|
237
|
+
return `Core subjects: ${anchors.join('; ')}.`;
|
|
238
|
+
}
|
|
239
|
+
function buildCompactAestheticString(aesthetic) {
|
|
240
|
+
return [
|
|
241
|
+
aesthetic.style,
|
|
242
|
+
aesthetic.palette,
|
|
243
|
+
aesthetic.lighting,
|
|
244
|
+
]
|
|
245
|
+
.filter(Boolean)
|
|
246
|
+
.join(', ');
|
|
247
|
+
}
|
|
248
|
+
function summarizeCharacterForMultiShot(char, wardrobeOverride, elementSlot) {
|
|
249
|
+
const wardrobe = wardrobeOverride ?? char.wardrobe;
|
|
250
|
+
const shortWardrobe = wardrobe.split(',').slice(0, 2).join(',').trim();
|
|
251
|
+
const baseTraits = char.baseTraits ?? (char.gender === 'female' ? FEMALE_BASE_TRAITS : MALE_BASE_TRAITS);
|
|
252
|
+
const label = elementSlot ? `@Element${elementSlot.elementIndex} (${char.name})` : char.name;
|
|
253
|
+
return `[${label}: ${baseTraits}, ${char.age}, ${char.description.split(',').slice(0, 3).join(',').trim()}, wearing ${shortWardrobe}]`;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Build a video-generation prompt for a shot.
|
|
257
|
+
*
|
|
258
|
+
* Directing principle (see .claude/agents/prompt-engineer.md and the README
|
|
259
|
+
* "Directing layer"): this assembles the prose that DIRECTS the shot -- one
|
|
260
|
+
* intention expressed through camera, light, blocking, performance, and sound
|
|
261
|
+
* -- not a pile of "cinematic" adjectives. It intentionally leans on the
|
|
262
|
+
* shot's authored `description`/delivery (which the workshop system prompt
|
|
263
|
+
* directs) rather than decorating it here. Identity is locked downstream by
|
|
264
|
+
* R2V references, so this does not inject exhaustive character descriptions.
|
|
265
|
+
*/
|
|
266
|
+
export function buildVideoPrompt(shot, series, previousShot, episodeAudioMix) {
|
|
267
|
+
if (!series.aesthetic) {
|
|
268
|
+
throw new Error('Series aesthetic must be set before generating videos.');
|
|
269
|
+
}
|
|
270
|
+
const resolution = resolveVideoModel(shot, series, previousShot);
|
|
271
|
+
const modelId = resolution.modelId;
|
|
272
|
+
const useElements = resolution.autoUseElements
|
|
273
|
+
|| (shot.useElements && MODELS_SUPPORTING_ELEMENTS.has(modelId));
|
|
274
|
+
const useRefs = resolution.autoUseReferenceImages
|
|
275
|
+
|| (shot.useReferenceImages && MODELS_SUPPORTING_REFERENCE_IMAGES.has(modelId));
|
|
276
|
+
const useImageTags = resolution.useImageTags;
|
|
277
|
+
const resolvedCharacters = shot.characters
|
|
278
|
+
.map(name => series.characters.find(c => c.name.toUpperCase() === name.toUpperCase()))
|
|
279
|
+
.filter((c) => Boolean(c));
|
|
280
|
+
let characterElements;
|
|
281
|
+
let referenceSlots;
|
|
282
|
+
if (useImageTags) {
|
|
283
|
+
// Central slot allocator: character primaries first, then storyboard
|
|
284
|
+
// blocking plate (protected), location angles, second character angles —
|
|
285
|
+
// budgeted per model (9 on Seedance R2V). The @ImageN indices here are
|
|
286
|
+
// authoritative; the video generator pushes reference_image_urls in
|
|
287
|
+
// exactly this order.
|
|
288
|
+
const plan = buildReferenceSlotPlan(series, shot, modelId, {
|
|
289
|
+
characterNames: resolvedCharacters.map(c => c.name),
|
|
290
|
+
});
|
|
291
|
+
referenceSlots = plan.slots;
|
|
292
|
+
for (const note of plan.dropped) {
|
|
293
|
+
console.warn(` ⚠ Reference budget: dropped ${note}`);
|
|
294
|
+
}
|
|
295
|
+
if (plan.characterSlotByName.size > 0) {
|
|
296
|
+
characterElements = resolvedCharacters
|
|
297
|
+
.filter(char => plan.characterSlotByName.has(char.name.toUpperCase()))
|
|
298
|
+
.map(char => ({
|
|
299
|
+
characterName: char.name,
|
|
300
|
+
elementIndex: plan.characterSlotByName.get(char.name.toUpperCase()),
|
|
301
|
+
}));
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
else if (useElements && resolvedCharacters.length > 0) {
|
|
305
|
+
characterElements = resolvedCharacters.slice(0, 2).map((char, index) => ({
|
|
306
|
+
characterName: char.name,
|
|
307
|
+
elementIndex: index + 1,
|
|
308
|
+
}));
|
|
309
|
+
}
|
|
310
|
+
const tagPrefix = useImageTags ? '@Image' : '@Element';
|
|
311
|
+
const parts = [];
|
|
312
|
+
const cameraTerm = CAMERA_TERMS[shot.cameraMovement.toLowerCase()] ?? shot.cameraMovement;
|
|
313
|
+
parts.push(`${cameraTerm}.`);
|
|
314
|
+
// Up-front identity declarations for @Image-tag models ("@Image1 is Bob").
|
|
315
|
+
// Restating the invariant identity per shot (rule 37) plus the explicit
|
|
316
|
+
// tag→name binding keeps multi-reference prompts unambiguous for the model.
|
|
317
|
+
if (useImageTags && characterElements) {
|
|
318
|
+
for (const slot of characterElements) {
|
|
319
|
+
const char = resolvedCharacters.find(c => c.name.toUpperCase() === slot.characterName.toUpperCase());
|
|
320
|
+
if (!char)
|
|
321
|
+
continue;
|
|
322
|
+
const wardrobe = shot.episodeWardrobe?.[char.name.toUpperCase()] ?? char.wardrobe;
|
|
323
|
+
parts.push(`@Image${slot.elementIndex} is ${char.name} — wearing ${wardrobe}.`);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
if ((useElements || useImageTags) && characterElements) {
|
|
327
|
+
let desc = shot.description;
|
|
328
|
+
for (const slot of characterElements) {
|
|
329
|
+
const re = new RegExp(`\\b${slot.characterName}\\b`, 'gi');
|
|
330
|
+
desc = desc.replace(re, `${tagPrefix}${slot.elementIndex}`);
|
|
331
|
+
}
|
|
332
|
+
parts.push(desc);
|
|
333
|
+
}
|
|
334
|
+
else {
|
|
335
|
+
parts.push(shot.description);
|
|
336
|
+
}
|
|
337
|
+
// NARRATOR / V.O. lines are voice-over — there is no on-camera speaker, so
|
|
338
|
+
// the line must never reach the video prompt. Including it is what makes
|
|
339
|
+
// Seedance synthesize its own competing English narrator. The line still
|
|
340
|
+
// lives in script.json for the assembler's TTS pass.
|
|
341
|
+
const dialogueSpeaker = shot.dialogue?.character?.toUpperCase();
|
|
342
|
+
const isVoiceOverLine = dialogueSpeaker === 'NARRATOR' || dialogueSpeaker === 'V.O.' || dialogueSpeaker === 'VO';
|
|
343
|
+
// Voice-reference slots: when the model can take reference audio, voice-refs
|
|
344
|
+
// aren't disabled, and the speaking character has a voice-donor clip, bind
|
|
345
|
+
// it in-prompt as @AudioN. The audioIndex here MUST match the push order of
|
|
346
|
+
// reference_audio_urls in the video generator (see resolveVoiceReferences).
|
|
347
|
+
const voiceRefEnabled = series.videoDefaults.voiceReferenceForDialogue !== false
|
|
348
|
+
&& MODELS_SUPPORTING_REFERENCE_AUDIO.has(modelId);
|
|
349
|
+
let voiceReferenceSlots;
|
|
350
|
+
if (voiceRefEnabled && shot.dialogue && !isVoiceOverLine) {
|
|
351
|
+
// Multi-speaker: the dialogue speaker gets @Audio1; every OTHER on-screen
|
|
352
|
+
// character with a voice-donor clip gets the next slot (Venice budget:
|
|
353
|
+
// ≤3 clips, ≤15s aggregate — enforced downstream in renderVideoFile).
|
|
354
|
+
// This keeps each character's voice right even when the model improvises
|
|
355
|
+
// reactions/off-lines for non-speaking characters.
|
|
356
|
+
const slots = [];
|
|
357
|
+
const speakerUpper = shot.dialogue.character.toUpperCase();
|
|
358
|
+
const speakingChar = series.characters.find(c => c.name.toUpperCase() === speakerUpper);
|
|
359
|
+
if (speakingChar?.voiceReferencePath) {
|
|
360
|
+
slots.push({ characterName: speakingChar.name, audioIndex: slots.length + 1 });
|
|
361
|
+
}
|
|
362
|
+
for (const charName of shot.characters) {
|
|
363
|
+
if (charName.toUpperCase() === speakerUpper)
|
|
364
|
+
continue;
|
|
365
|
+
if (slots.length >= 3)
|
|
366
|
+
break;
|
|
367
|
+
const char = series.characters.find(c => c.name.toUpperCase() === charName.toUpperCase());
|
|
368
|
+
if (char?.voiceReferencePath) {
|
|
369
|
+
slots.push({ characterName: char.name, audioIndex: slots.length + 1 });
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
if (slots.length > 0)
|
|
373
|
+
voiceReferenceSlots = slots;
|
|
374
|
+
}
|
|
375
|
+
if (shot.dialogue && !isVoiceOverLine) {
|
|
376
|
+
const speakingChar = series.characters.find(c => c.name.toUpperCase() === shot.dialogue.character.toUpperCase());
|
|
377
|
+
const voiceDesc = speakingChar?.voiceDescription ?? '';
|
|
378
|
+
const delivery = shot.dialogue.delivery || '';
|
|
379
|
+
const charRef = (useElements || useImageTags) && characterElements
|
|
380
|
+
? (characterElements.find(s => s.characterName.toUpperCase() === shot.dialogue.character.toUpperCase())
|
|
381
|
+
? `${tagPrefix}${characterElements.find(s => s.characterName.toUpperCase() === shot.dialogue.character.toUpperCase()).elementIndex}`
|
|
382
|
+
: shot.dialogue.character)
|
|
383
|
+
: shot.dialogue.character;
|
|
384
|
+
const voiceParts = [voiceDesc, delivery].filter(Boolean).join(', ');
|
|
385
|
+
parts.push(`[${charRef}, ${voiceParts}]: "${shot.dialogue.line}"`);
|
|
386
|
+
// Bind the voice-donor clip. @AudioN carries voice identity ONLY; the
|
|
387
|
+
// model should still render clean studio dialogue for the line above.
|
|
388
|
+
const slot = voiceReferenceSlots?.find(s => s.characterName.toUpperCase() === shot.dialogue.character.toUpperCase());
|
|
389
|
+
if (slot) {
|
|
390
|
+
parts.push(`Use @Audio${slot.audioIndex} only for voice identity — timbre, accent, pacing; ` +
|
|
391
|
+
`regenerate clean studio dialogue, do not copy any noise from the reference.`);
|
|
392
|
+
}
|
|
393
|
+
// Bind the remaining voice-donor clips to their characters so any
|
|
394
|
+
// improvised lines/reactions from non-speaking characters use the
|
|
395
|
+
// right voice too.
|
|
396
|
+
for (const other of voiceReferenceSlots ?? []) {
|
|
397
|
+
if (slot && other.audioIndex === slot.audioIndex)
|
|
398
|
+
continue;
|
|
399
|
+
const otherCharSlot = characterElements?.find(s => s.characterName.toUpperCase() === other.characterName.toUpperCase());
|
|
400
|
+
const otherRef = otherCharSlot ? `${tagPrefix}${otherCharSlot.elementIndex}` : other.characterName;
|
|
401
|
+
parts.push(`@Audio${other.audioIndex} is ${otherRef}'s voice — use it only if ${otherRef} speaks.`);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
if (shot.sfx) {
|
|
405
|
+
parts.push(`Sound of ${shot.sfx}.`);
|
|
406
|
+
}
|
|
407
|
+
// Scene image refs use @Image tags — offset indices when image tags are
|
|
408
|
+
// already used for character refs so tags don't collide.
|
|
409
|
+
if (shot.sceneImagePaths && shot.sceneImagePaths.length > 0 && MODELS_SUPPORTING_SCENE_IMAGES.has(modelId)) {
|
|
410
|
+
const sceneOffset = useImageTags ? (characterElements?.length ?? 0) : 0;
|
|
411
|
+
const refs = shot.sceneImagePaths.slice(0, 4).map((_, i) => `@Image${sceneOffset + i + 1}`);
|
|
412
|
+
parts.push(`Scene style references: ${refs.join(', ')}.`);
|
|
413
|
+
}
|
|
414
|
+
// Location environment description. When the shot is tagged with a location,
|
|
415
|
+
// inject its locked description + lighting so consecutive shots in the same
|
|
416
|
+
// place stay consistent (anti-pattern 7).
|
|
417
|
+
let locationEnvSlot;
|
|
418
|
+
if (shot.location) {
|
|
419
|
+
const loc = getLocation(series, shot.location);
|
|
420
|
+
if (loc) {
|
|
421
|
+
const envNote = [loc.description, loc.lightingNotes ? `Lighting: ${loc.lightingNotes}.` : '']
|
|
422
|
+
.filter(Boolean).join(' ');
|
|
423
|
+
if (envNote)
|
|
424
|
+
parts.push(`Location: ${envNote}`);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
// Role clauses for every non-primary-character reference slot (storyboard
|
|
428
|
+
// blocking plates, location angles, second character angles). The clause
|
|
429
|
+
// text lives on the slot itself (reference-slots.ts) so the prompt and the
|
|
430
|
+
// reference_image_urls array can never disagree about what @ImageN means.
|
|
431
|
+
if (useImageTags && referenceSlots) {
|
|
432
|
+
for (const slot of referenceSlots) {
|
|
433
|
+
if (slot.kind === 'character-primary')
|
|
434
|
+
continue; // named via description substitution
|
|
435
|
+
parts.push(`@Image${slot.imageIndex} ${slot.roleClause}.`);
|
|
436
|
+
}
|
|
437
|
+
const firstLocationSlot = referenceSlots.find(s => s.kind === 'location');
|
|
438
|
+
if (firstLocationSlot) {
|
|
439
|
+
locationEnvSlot = { slug: firstLocationSlot.label, imageIndex: firstLocationSlot.imageIndex };
|
|
440
|
+
}
|
|
441
|
+
const sbSlot = referenceSlots.find(s => s.kind === 'storyboard');
|
|
442
|
+
if (sbSlot) {
|
|
443
|
+
parts.push('Every reference must stay consistent across space and time: characters keep their ' +
|
|
444
|
+
'appearance, the location keeps its geography, and the blocking follows ' +
|
|
445
|
+
`@Image${sbSlot.imageIndex} even as the camera angle changes.`);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
let aestheticStr = buildAestheticString(series.aesthetic);
|
|
449
|
+
if (isDaytimeShot(shot)) {
|
|
450
|
+
aestheticStr = stripDarkAesthetic(aestheticStr);
|
|
451
|
+
parts.push('Bright daytime scene, natural light, no rain.');
|
|
452
|
+
}
|
|
453
|
+
parts.push(aestheticStr + '.');
|
|
454
|
+
parts.push(VIDEO_NO_MUSIC_SUFFIX);
|
|
455
|
+
if (isVoiceOverLine) {
|
|
456
|
+
// The VO line was withheld from the prompt above; also tell the model
|
|
457
|
+
// explicitly that this shot carries no speech, so its audio track is
|
|
458
|
+
// ambient + SFX only (the Venice TTS narrator is mixed in by the
|
|
459
|
+
// assembler).
|
|
460
|
+
parts.push('No narration, no voice-over, no spoken words in this shot.');
|
|
461
|
+
}
|
|
462
|
+
const videoPrompt = parts.join(' ');
|
|
463
|
+
// Decide whether to ask the video model to synthesize audio. Default is
|
|
464
|
+
// `true` — ambient sound and SFX from the model are wanted even on
|
|
465
|
+
// narrator-VO shots (the VO line never reaches the prompt, see above, so
|
|
466
|
+
// there is no competing narrator to suppress). Audio is only disabled when:
|
|
467
|
+
// - the episode's audio mix has opted into suppressModelNarration for
|
|
468
|
+
// every dialogue-bearing shot, or
|
|
469
|
+
// - the shot has nativeAudio === 'mute' (per-shot override).
|
|
470
|
+
// Callers that need the native track regardless can flip nativeAudio: 'keep'.
|
|
471
|
+
const suppressGlobal = episodeAudioMix?.suppressModelNarration === true && Boolean(shot.dialogue);
|
|
472
|
+
const muteByOverride = shot.nativeAudio === 'mute';
|
|
473
|
+
const keepByOverride = shot.nativeAudio === 'keep';
|
|
474
|
+
const audio = keepByOverride
|
|
475
|
+
? true
|
|
476
|
+
: (suppressGlobal || muteByOverride)
|
|
477
|
+
? false
|
|
478
|
+
: true;
|
|
479
|
+
return {
|
|
480
|
+
prompt: videoPrompt,
|
|
481
|
+
model: modelId,
|
|
482
|
+
duration: shot.duration,
|
|
483
|
+
audio,
|
|
484
|
+
characterElements,
|
|
485
|
+
sceneImagePaths: shot.sceneImagePaths,
|
|
486
|
+
voiceReferenceSlots,
|
|
487
|
+
locationEnvSlot,
|
|
488
|
+
referenceSlots,
|
|
489
|
+
referenceImageUrls: useRefs ? [] : undefined,
|
|
490
|
+
modelResolution: resolution,
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
export function buildKlingMultiShotPrompt(shots, unit, series) {
|
|
494
|
+
if (!series.aesthetic) {
|
|
495
|
+
throw new Error('Series aesthetic must be set before generating videos.');
|
|
496
|
+
}
|
|
497
|
+
const uniqueCharNames = Array.from(new Set(shots.flatMap(shot => shot.characters.map(name => name.toUpperCase()))));
|
|
498
|
+
const uniqueCharacters = uniqueCharNames
|
|
499
|
+
.map(name => series.characters.find(char => char.name.toUpperCase() === name))
|
|
500
|
+
.filter((char) => Boolean(char));
|
|
501
|
+
// Build element slots for identity anchoring (Kling O3 Pro supports elements)
|
|
502
|
+
const useElements = MODELS_SUPPORTING_ELEMENTS.has(KLING_MULTISHOT_MODEL);
|
|
503
|
+
let characterElements;
|
|
504
|
+
if (useElements && uniqueCharacters.length > 0) {
|
|
505
|
+
characterElements = uniqueCharacters.slice(0, 2).map((char, index) => ({
|
|
506
|
+
characterName: char.name,
|
|
507
|
+
elementIndex: index + 1,
|
|
508
|
+
}));
|
|
509
|
+
}
|
|
510
|
+
const useRefs = MODELS_SUPPORTING_REFERENCE_IMAGES.has(KLING_MULTISHOT_MODEL);
|
|
511
|
+
const wardrobeByChar = new Map();
|
|
512
|
+
for (const shot of shots) {
|
|
513
|
+
if (!shot.episodeWardrobe)
|
|
514
|
+
continue;
|
|
515
|
+
for (const [charName, wardrobe] of Object.entries(shot.episodeWardrobe)) {
|
|
516
|
+
if (!wardrobeByChar.has(charName.toUpperCase())) {
|
|
517
|
+
wardrobeByChar.set(charName.toUpperCase(), wardrobe);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
// --- Kling 3.0 native multi-shot prompt structure ---
|
|
522
|
+
// Per https://blog.fal.ai/kling-3-0-prompting-guide/:
|
|
523
|
+
// 1. Define core subjects up front with @Element refs and traits
|
|
524
|
+
// 2. State shot count and continuity instruction
|
|
525
|
+
// 3. Label each shot as "Shot N (Xs):" with cinematic direction
|
|
526
|
+
// 4. Use [Character, voice description]: "dialogue" format
|
|
527
|
+
// 5. Use "Immediately," between shots for temporal control
|
|
528
|
+
// 6. Append compact aesthetic and audio instructions
|
|
529
|
+
const parts = [];
|
|
530
|
+
// Subject definition block — Kling 3.0 locks these across all shots
|
|
531
|
+
if (uniqueCharacters.length > 0) {
|
|
532
|
+
for (const char of uniqueCharacters) {
|
|
533
|
+
const slot = characterElements?.find(s => s.characterName.toUpperCase() === char.name.toUpperCase());
|
|
534
|
+
const wardrobe = wardrobeByChar.get(char.name.toUpperCase()) ?? char.wardrobe;
|
|
535
|
+
const baseTraits = char.baseTraits ?? (char.gender === 'female' ? FEMALE_BASE_TRAITS : MALE_BASE_TRAITS);
|
|
536
|
+
const label = slot ? `@Element${slot.elementIndex}` : char.name;
|
|
537
|
+
parts.push(`${label} is ${char.name}: ${char.age}, ${baseTraits}, ${char.description}. Wearing ${wardrobe}.`);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
parts.push(`\n${shots.length}-shot continuous sequence. Lock face, wardrobe, and environment across all shots.\n`);
|
|
541
|
+
// Per-shot blocks with Kling 3.0 dialogue format
|
|
542
|
+
for (let index = 0; index < shots.length; index++) {
|
|
543
|
+
const shot = shots[index];
|
|
544
|
+
const cameraTerm = CAMERA_TERMS[shot.cameraMovement.toLowerCase()] ?? shot.cameraMovement;
|
|
545
|
+
const shotParts = [];
|
|
546
|
+
shotParts.push(`Shot ${index + 1} (${parseShotDuration(shot.duration)}s): ${cameraTerm}.`);
|
|
547
|
+
let desc = shot.description;
|
|
548
|
+
if (useElements && characterElements) {
|
|
549
|
+
for (const slot of characterElements) {
|
|
550
|
+
const re = new RegExp(`\\b${slot.characterName}\\b`, 'gi');
|
|
551
|
+
desc = desc.replace(re, `@Element${slot.elementIndex}`);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
shotParts.push(desc);
|
|
555
|
+
// Kling 3.0 dialogue format: [Character, voice description]: "line"
|
|
556
|
+
if (shot.dialogue) {
|
|
557
|
+
const speakingChar = series.characters.find(c => c.name.toUpperCase() === shot.dialogue.character.toUpperCase());
|
|
558
|
+
const voiceDesc = speakingChar?.voiceDescription ?? '';
|
|
559
|
+
const delivery = shot.dialogue.delivery || '';
|
|
560
|
+
const charRef = useElements && characterElements
|
|
561
|
+
? (characterElements.find(s => s.characterName.toUpperCase() === shot.dialogue.character.toUpperCase())
|
|
562
|
+
? `@Element${characterElements.find(s => s.characterName.toUpperCase() === shot.dialogue.character.toUpperCase()).elementIndex}`
|
|
563
|
+
: shot.dialogue.character)
|
|
564
|
+
: shot.dialogue.character;
|
|
565
|
+
const voiceParts = [voiceDesc, delivery].filter(Boolean).join(', ');
|
|
566
|
+
shotParts.push(`[${charRef}, ${voiceParts}]: "${shot.dialogue.line}"`);
|
|
567
|
+
}
|
|
568
|
+
if (shot.sfx) {
|
|
569
|
+
shotParts.push(`Sound: ${shot.sfx}.`);
|
|
570
|
+
}
|
|
571
|
+
parts.push(shotParts.join('\n'));
|
|
572
|
+
// Temporal separator between shots
|
|
573
|
+
if (index < shots.length - 1) {
|
|
574
|
+
parts.push('\nImmediately, cut to:\n');
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
const anyDaytime = shots.some(s => isDaytimeShot(s));
|
|
578
|
+
let compactAesthetic = buildCompactAestheticString(series.aesthetic);
|
|
579
|
+
if (anyDaytime) {
|
|
580
|
+
compactAesthetic = stripDarkAesthetic(compactAesthetic);
|
|
581
|
+
parts.push('Bright daytime scene, natural light, no rain.');
|
|
582
|
+
}
|
|
583
|
+
parts.push(`Visual style: ${compactAesthetic}.`);
|
|
584
|
+
parts.push(VIDEO_NO_MUSIC_SUFFIX);
|
|
585
|
+
let prompt = parts.join(' ').trim();
|
|
586
|
+
const VENICE_PROMPT_LIMIT = 2500;
|
|
587
|
+
if (prompt.length > VENICE_PROMPT_LIMIT) {
|
|
588
|
+
console.warn(` Multi-shot prompt is ${prompt.length} chars (limit: ${VENICE_PROMPT_LIMIT}). Truncating aesthetic to fit.`);
|
|
589
|
+
const overBy = prompt.length - VENICE_PROMPT_LIMIT + 20;
|
|
590
|
+
const aestheticPart = buildCompactAestheticString(series.aesthetic);
|
|
591
|
+
const truncatedAesthetic = aestheticPart.slice(0, Math.max(40, aestheticPart.length - overBy));
|
|
592
|
+
parts[parts.length - 2] = `Visual style: ${truncatedAesthetic}.`;
|
|
593
|
+
prompt = parts.join(' ').trim();
|
|
594
|
+
if (prompt.length > VENICE_PROMPT_LIMIT) {
|
|
595
|
+
prompt = prompt.slice(0, VENICE_PROMPT_LIMIT);
|
|
596
|
+
console.warn(` Prompt still over limit after truncation. Hard-cut to ${VENICE_PROMPT_LIMIT} chars.`);
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
return {
|
|
600
|
+
prompt,
|
|
601
|
+
model: KLING_MULTISHOT_MODEL,
|
|
602
|
+
duration: unit.duration,
|
|
603
|
+
audio: true,
|
|
604
|
+
characterElements,
|
|
605
|
+
referenceImageUrls: useRefs ? [] : undefined,
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* Build a character reference prompt within a per-model length cap.
|
|
610
|
+
*
|
|
611
|
+
* Returns just the positive prompt (string) for backwards compatibility.
|
|
612
|
+
* For the structured form that includes the recommended negative-prompt
|
|
613
|
+
* additions, use `buildCharacterReferencePromptParts`.
|
|
614
|
+
*/
|
|
615
|
+
export function buildCharacterReferencePrompt(char, aesthetic, angle, options) {
|
|
616
|
+
return buildCharacterReferencePromptParts(char, aesthetic, angle, options).positive;
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Structured character-reference prompt that splits style and
|
|
620
|
+
* "no realism" cues into the negative prompt, keeping the positive prompt
|
|
621
|
+
* under the per-model cap.
|
|
622
|
+
*
|
|
623
|
+
* Discovered: above ~1800-2200 chars on seedream-v5-lite Venice silently
|
|
624
|
+
* rejects the request (panel returns < 30KB; see ). Moving STYLE
|
|
625
|
+
* REMINDER content to negative_prompt drops 60-80 chars and stops the
|
|
626
|
+
* silent rejections in production (Glass panel re-generation, v3 -> v4).
|
|
627
|
+
*
|
|
628
|
+
* The positive prompt keeps the most-important style cue + character
|
|
629
|
+
* anchor inline; everything else moves to negative_prompt.
|
|
630
|
+
*/
|
|
631
|
+
export function buildCharacterReferencePromptParts(char, aesthetic, angle, options) {
|
|
632
|
+
const baseTraits = char.baseTraits ?? (char.gender === 'female' ? FEMALE_BASE_TRAITS : MALE_BASE_TRAITS);
|
|
633
|
+
const cap = options?.maxChars
|
|
634
|
+
?? getMaxPositivePromptChars(options?.model ?? 'seedream-v5-lite');
|
|
635
|
+
const anglePrompts = {
|
|
636
|
+
'front': 'front portrait, looking at camera, centered, studio lighting, neutral background',
|
|
637
|
+
'three-quarter': 'three-quarter view, 45 degree angle, studio lighting, neutral background',
|
|
638
|
+
'profile': 'side profile, 90 degree angle, studio lighting, neutral background',
|
|
639
|
+
'full-body': 'full body, head to toe, standing pose, studio lighting, neutral background',
|
|
640
|
+
};
|
|
641
|
+
// Style is a single front-loaded cue; the rest (palette, lensCharacteristics,
|
|
642
|
+
// film stock) move to negativeAdditions as anti-realism guards.
|
|
643
|
+
const styleCue = aesthetic.style;
|
|
644
|
+
// Build prompt parts in priority order. We greedily append until the cap
|
|
645
|
+
// is hit; the tail is dropped and the consumer can choose to push the
|
|
646
|
+
// dropped pieces into the negative prompt.
|
|
647
|
+
const parts = [
|
|
648
|
+
`STYLE: ${styleCue}.`,
|
|
649
|
+
`${anglePrompts[angle]}.`,
|
|
650
|
+
`${char.fullDescription}.`,
|
|
651
|
+
`${baseTraits}.`,
|
|
652
|
+
`${char.wardrobe}.`,
|
|
653
|
+
];
|
|
654
|
+
let positive = '';
|
|
655
|
+
for (const p of parts) {
|
|
656
|
+
const candidate = positive ? `${positive} ${p}` : p;
|
|
657
|
+
if (candidate.length > cap)
|
|
658
|
+
break;
|
|
659
|
+
positive = candidate;
|
|
660
|
+
}
|
|
661
|
+
// If the very first part already exceeds the cap, hard-truncate.
|
|
662
|
+
if (!positive) {
|
|
663
|
+
positive = parts[0].slice(0, cap);
|
|
664
|
+
}
|
|
665
|
+
// Style-reminder content + photorealism guards belong on the negative side.
|
|
666
|
+
// They steer the model away from the wrong rendering family without eating
|
|
667
|
+
// positive-prompt budget.
|
|
668
|
+
//
|
|
669
|
+
// Strategy selects whether to emit anti-photoreal guards. For photoreal
|
|
670
|
+
// aesthetics ("documentary", "photograph", "cinematic photography") these
|
|
671
|
+
// guards fight the positives — the legislator-as-bird and founder-as-bird
|
|
672
|
+
// regressions in the PNW field-guide episode were caused exactly by this.
|
|
673
|
+
const strategy = options?.negativePromptStrategy ?? 'auto';
|
|
674
|
+
const wantsAntiPhotoreal = strategy === 'stylized'
|
|
675
|
+
? true
|
|
676
|
+
: strategy === 'photoreal' || strategy === 'none'
|
|
677
|
+
? false
|
|
678
|
+
: !isPhotorealAesthetic(aesthetic);
|
|
679
|
+
if (strategy === 'none') {
|
|
680
|
+
return { positive, negativeAdditions: [] };
|
|
681
|
+
}
|
|
682
|
+
const antiPhotorealParts = wantsAntiPhotoreal
|
|
683
|
+
? ['photorealistic', 'photograph', 'photo', '3D render', 'Pixar']
|
|
684
|
+
: [];
|
|
685
|
+
const negativeAdditions = [
|
|
686
|
+
aesthetic.filmStock ? `not ${aesthetic.filmStock}` : null,
|
|
687
|
+
aesthetic.palette && wantsAntiPhotoreal ? `not ${aesthetic.palette}` : null,
|
|
688
|
+
...antiPhotorealParts,
|
|
689
|
+
'no text',
|
|
690
|
+
'no labels',
|
|
691
|
+
'no annotations',
|
|
692
|
+
'no inset panels',
|
|
693
|
+
'no detail callouts',
|
|
694
|
+
'no multi-view layout',
|
|
695
|
+
].filter((s) => Boolean(s));
|
|
696
|
+
return { positive, negativeAdditions };
|
|
697
|
+
}
|
|
698
|
+
/**
|
|
699
|
+
* Detect a photoreal series aesthetic from natural-language fields so
|
|
700
|
+
* `negativePromptStrategy: 'auto'` can do the right thing without making
|
|
701
|
+
* the user spell it out. We check `style`, `lensCharacteristics`,
|
|
702
|
+
* `filmStock`, and a `notes` field (when present) for any of:
|
|
703
|
+
* photoreal, photograph, photo, documentary, live action, naturalist,
|
|
704
|
+
* cinematic photography
|
|
705
|
+
*
|
|
706
|
+
* This is a precision-over-recall heuristic — anything close to photoreal
|
|
707
|
+
* suppresses the anti-photoreal guards. Operators who want the legacy
|
|
708
|
+
* behaviour back can flip `negativePromptStrategy: 'stylized'`.
|
|
709
|
+
*/
|
|
710
|
+
function isPhotorealAesthetic(aesthetic) {
|
|
711
|
+
const blob = [
|
|
712
|
+
aesthetic.style,
|
|
713
|
+
aesthetic.lensCharacteristics,
|
|
714
|
+
aesthetic.filmStock,
|
|
715
|
+
aesthetic.notes,
|
|
716
|
+
]
|
|
717
|
+
.filter((s) => typeof s === 'string')
|
|
718
|
+
.join(' ')
|
|
719
|
+
.toLowerCase();
|
|
720
|
+
return /\b(photoreal|photograph|photo|documentary|live[- ]action|naturalist|cinematic photography|nature[- ]documentary)\b/.test(blob);
|
|
721
|
+
}
|
|
722
|
+
//# sourceMappingURL=prompt-builder.js.map
|