caudate-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api/__init__.py +5 -0
- api/anthropic_compat.py +1518 -0
- api/artifact_viewer.py +366 -0
- api/caudate_middleware.py +618 -0
- api/forge_bootstrapper_routes.py +377 -0
- api/forge_routes.py +630 -0
- api/forge_system_routes.py +294 -0
- api/openai_compat.py +1993 -0
- api/server.py +667 -0
- api/storyboard_page.py +677 -0
- caudate_cli-0.1.0.dist-info/METADATA +354 -0
- caudate_cli-0.1.0.dist-info/RECORD +153 -0
- caudate_cli-0.1.0.dist-info/WHEEL +5 -0
- caudate_cli-0.1.0.dist-info/entry_points.txt +2 -0
- caudate_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- caudate_cli-0.1.0.dist-info/top_level.txt +14 -0
- cognos_mcp/__init__.py +4 -0
- cognos_mcp/bridge.py +41 -0
- cognos_mcp/client.py +70 -0
- cognos_mcp/config.py +49 -0
- cognos_mcp/server.py +66 -0
- config.py +82 -0
- core/__init__.py +0 -0
- core/agent.py +468 -0
- core/agentic_loop.py +731 -0
- core/anthropic_auth.py +91 -0
- core/background.py +113 -0
- core/banner.py +134 -0
- core/bootstrap.py +292 -0
- core/citations.py +131 -0
- core/compaction.py +109 -0
- core/constitution.py +198 -0
- core/diff_viewer.py +87 -0
- core/export.py +85 -0
- core/file_refs.py +119 -0
- core/files.py +199 -0
- core/hooks.py +209 -0
- core/image.py +599 -0
- core/input.py +91 -0
- core/loop.py +238 -0
- core/memory_md.py +147 -0
- core/notifications.py +99 -0
- core/ownership.py +181 -0
- core/paste.py +81 -0
- core/permissions.py +210 -0
- core/plan_mode.py +215 -0
- core/sandbox_prompt.py +185 -0
- core/scheduler.py +195 -0
- core/schemas.py +202 -0
- core/session.py +90 -0
- core/settings.py +132 -0
- core/skills.py +398 -0
- core/slash_commands.py +977 -0
- core/statusline.py +61 -0
- core/subagent.py +300 -0
- core/thinking.py +50 -0
- core/updater.py +122 -0
- core/usage.py +109 -0
- core/worktree.py +93 -0
- execution/__init__.py +0 -0
- execution/executor.py +329 -0
- execution/plugins.py +108 -0
- execution/tools/__init__.py +0 -0
- execution/tools/agent_tool.py +107 -0
- execution/tools/agentic_tool.py +297 -0
- execution/tools/artifact_tool.py +191 -0
- execution/tools/ask_user_question_tool.py +137 -0
- execution/tools/base.py +81 -0
- execution/tools/calculator_tool.py +137 -0
- execution/tools/cognos_card_tool.py +124 -0
- execution/tools/cron_tool.py +215 -0
- execution/tools/datetime_tool.py +215 -0
- execution/tools/describe_image_tool.py +161 -0
- execution/tools/draw_tool.py +164 -0
- execution/tools/edit_image_tool.py +262 -0
- execution/tools/edit_tool.py +245 -0
- execution/tools/file_tool.py +90 -0
- execution/tools/find_anywhere_tool.py +255 -0
- execution/tools/forge_feature_tools.py +377 -0
- execution/tools/glob_tool.py +59 -0
- execution/tools/grep_tool.py +89 -0
- execution/tools/http_request_tool.py +224 -0
- execution/tools/load_skill_tool.py +104 -0
- execution/tools/longcat_avatar_tool.py +384 -0
- execution/tools/mcp_tool.py +100 -0
- execution/tools/notebook_tool.py +279 -0
- execution/tools/openapi_tool.py +440 -0
- execution/tools/plan_mode_tool.py +95 -0
- execution/tools/push_notification_tool.py +157 -0
- execution/tools/python_tool.py +61 -0
- execution/tools/respond_tool.py +40 -0
- execution/tools/sandbox_tool.py +378 -0
- execution/tools/search_tool.py +153 -0
- execution/tools/semantic_search_tool.py +106 -0
- execution/tools/shell_tool.py +283 -0
- execution/tools/speak_tool.py +134 -0
- execution/tools/storyboard_tool.py +727 -0
- execution/tools/system_info_tool.py +212 -0
- execution/tools/task_tool.py +323 -0
- execution/tools/think_tool.py +49 -0
- execution/tools/transcribe_audio_tool.py +86 -0
- execution/tools/update_memory_tool.py +92 -0
- execution/tools/web_fetch_tool.py +82 -0
- execution/tools/worktree_tool.py +174 -0
- llm/__init__.py +0 -0
- llm/fallback.py +116 -0
- llm/models.py +320 -0
- llm/provider.py +1356 -0
- llm/router.py +373 -0
- main.py +1889 -0
- memory/__init__.py +0 -0
- memory/episodic.py +99 -0
- memory/procedural.py +145 -0
- memory/semantic.py +71 -0
- memory/working.py +64 -0
- nn/__init__.py +43 -0
- nn/auto_evolve.py +245 -0
- nn/caudate.py +136 -0
- nn/config.py +141 -0
- nn/consolidator.py +81 -0
- nn/data.py +1635 -0
- nn/encoder.py +258 -0
- nn/forge_advisor.py +303 -0
- nn/format.py +235 -0
- nn/heads.py +432 -0
- nn/observer.py +994 -0
- nn/policy.py +214 -0
- nn/runtime.py +343 -0
- nn/scorer.py +175 -0
- nn/trainer.py +515 -0
- nn/vision.py +352 -0
- personality/__init__.py +23 -0
- personality/engine.py +129 -0
- personality/identity.py +144 -0
- personality/inner_voice.py +100 -0
- personality/mood.py +205 -0
- planning/__init__.py +0 -0
- planning/dev_server.py +221 -0
- planning/forge_models.py +718 -0
- planning/orchestrator.py +1363 -0
- planning/planner.py +451 -0
- planning/task_graph.py +61 -0
- reflection/__init__.py +0 -0
- reflection/meta_learner.py +156 -0
- reflection/reflector.py +127 -0
- ui/__init__.py +5 -0
- ui/display.py +88 -0
- voice/__init__.py +0 -0
- voice/conversation.py +125 -0
- voice/listener.py +111 -0
- voice/speaker.py +59 -0
- voice/stt.py +126 -0
- voice/tts.py +214 -0
|
@@ -0,0 +1,727 @@
|
|
|
1
|
+
"""Storyboard — turn a story description into a sequence of images.
|
|
2
|
+
|
|
3
|
+
Pipeline (per request):
|
|
4
|
+
1. Ask the LLM to break the story into N panel-sized beats.
|
|
5
|
+
2. Generate panel 1 with FLUX-schnell text→image (Draw backend).
|
|
6
|
+
3. For panels 2..N, use FLUX.1-Kontext with the previous panel as
|
|
7
|
+
the reference image — Kontext preserves character identity and
|
|
8
|
+
setting across panels, which is the whole point of a storyboard.
|
|
9
|
+
4. Persist every panel to the FileStore.
|
|
10
|
+
5. Return a markdown gallery the chat UI renders inline (one image
|
|
11
|
+
per beat, captioned with the beat description).
|
|
12
|
+
|
|
13
|
+
Time budget on a 24 GB 3090 with `enable_sequential_cpu_offload`:
|
|
14
|
+
- Panel 1 (Draw, 4 steps): ~10–30 s
|
|
15
|
+
- Panel N>1 (Kontext, 28 steps): ~6–10 min each
|
|
16
|
+
So a 6-panel story ≈ 35–50 min. Use `panels` carefully.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import logging
|
|
23
|
+
import re
|
|
24
|
+
import tempfile
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
from core.image import (
|
|
29
|
+
DiffusersImageEdit,
|
|
30
|
+
DiffusersImageGen,
|
|
31
|
+
ImageEdit,
|
|
32
|
+
ImageGen,
|
|
33
|
+
)
|
|
34
|
+
from core.schemas import ToolResult
|
|
35
|
+
from execution.tools.base import BaseTool
|
|
36
|
+
|
|
37
|
+
logger = logging.getLogger(__name__)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_BREAKDOWN_SYSTEM = """You output a JSON object and nothing else.
|
|
41
|
+
Do NOT include reasoning, analysis, commentary, planning steps, or
|
|
42
|
+
prose of any kind. No code fences, no markdown, no preamble — only
|
|
43
|
+
the raw JSON object. The first character of your response must be
|
|
44
|
+
`{` and the last must be `}`.
|
|
45
|
+
|
|
46
|
+
You are a story-to-storyboard planner. Given a script or scene
|
|
47
|
+
description, produce a JSON object that captures the structure
|
|
48
|
+
needed to render a visually consistent storyboard.
|
|
49
|
+
|
|
50
|
+
Use these EXACT keys (do not invent your own; do not use synonyms
|
|
51
|
+
like `description`, `panel_number`, `dialogue`, etc.):
|
|
52
|
+
|
|
53
|
+
{
|
|
54
|
+
"art_style": "<global art-style description applied to every panel.
|
|
55
|
+
Be precise: medium (2D digital, watercolor, oil),
|
|
56
|
+
line quality, colour palette, lighting, era. ~30 words.>",
|
|
57
|
+
"characters": [
|
|
58
|
+
{
|
|
59
|
+
"name": "<short proper name used to refer to this character>",
|
|
60
|
+
"visual_description": "<concrete physical traits: hair colour
|
|
61
|
+
and style, eye colour, age range, height,
|
|
62
|
+
specific clothing items, distinguishing
|
|
63
|
+
accessories. ~30 words. Generic enough that
|
|
64
|
+
a diffusion model can render it.>"
|
|
65
|
+
}
|
|
66
|
+
],
|
|
67
|
+
"panels": [
|
|
68
|
+
{
|
|
69
|
+
"panel_text": "<the relevant slice of the original script>",
|
|
70
|
+
"scene_action": "<what is happening visually in this panel:
|
|
71
|
+
location, action, framing, time of day. ~25 words.>",
|
|
72
|
+
"character_names_in_scene": ["<name1>", "<name2>"]
|
|
73
|
+
}
|
|
74
|
+
]
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
Rules:
|
|
78
|
+
- Use exactly N panels (the user will specify N).
|
|
79
|
+
- Reuse the same character names across every panel they appear in.
|
|
80
|
+
Do NOT invent new names mid-storyboard for the same person.
|
|
81
|
+
- Every name in a panel's `character_names_in_scene` MUST also appear
|
|
82
|
+
in the top-level `characters` list.
|
|
83
|
+
- Keep `scene_action` purely visual (no internal monologue, no
|
|
84
|
+
inaudible thoughts, no spoken dialogue lines).
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _strip_code_fence(s: str) -> str:
|
|
89
|
+
s = s.strip()
|
|
90
|
+
m = re.match(r"^```(?:json)?\s*(.*?)\s*```$", s, re.DOTALL)
|
|
91
|
+
return m.group(1) if m else s
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class StoryboardTool(BaseTool):
|
|
95
|
+
mutates = True
|
|
96
|
+
name = "Storyboard"
|
|
97
|
+
description = (
|
|
98
|
+
"Turn a story description into a SEQUENCE of images (a "
|
|
99
|
+
"storyboard / comic strip / picture-book spread).\n\n"
|
|
100
|
+
"Pipeline:\n"
|
|
101
|
+
" 1. The LLM extracts a structured plan from the script:\n"
|
|
102
|
+
" - global art_style description\n"
|
|
103
|
+
" - cast list (each character with consistent visual "
|
|
104
|
+
" descriptors: hair, clothing, distinguishing traits)\n"
|
|
105
|
+
" - per-panel scene_action + which characters appear\n"
|
|
106
|
+
" 2. Panel 1 is rendered with FLUX-schnell text-to-image,\n"
|
|
107
|
+
" prompt = art_style + scene_action + each character's\n"
|
|
108
|
+
" visual description.\n"
|
|
109
|
+
" 3. Panels 2..N use FLUX.1-Kontext img2img referencing the\n"
|
|
110
|
+
" previous panel, so faces/outfits/world stay consistent.\n"
|
|
111
|
+
"Returns a markdown gallery the chat UI renders inline.\n\n"
|
|
112
|
+
"Use this when the user asks for a comic, picture book, "
|
|
113
|
+
"storyboard, manga page, multiple images of the story, or to "
|
|
114
|
+
"'visualize this script'.\n\n"
|
|
115
|
+
"Time budget on a 24 GB GPU: ~30 s breakdown + ~30 s for panel "
|
|
116
|
+
"1 + ~7 min per subsequent panel. Default is 6 panels — pick "
|
|
117
|
+
"fewer for speed.\n\n"
|
|
118
|
+
"NOTE: panels 2..N use Kontext (non-commercial license)."
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
def __init__(
|
|
122
|
+
self,
|
|
123
|
+
file_store: Any | None = None,
|
|
124
|
+
llm: Any | None = None,
|
|
125
|
+
image_gen: ImageGen | None = None,
|
|
126
|
+
image_edit: ImageEdit | None = None,
|
|
127
|
+
):
|
|
128
|
+
self._file_store = file_store
|
|
129
|
+
self._llm = llm
|
|
130
|
+
self._image_gen = image_gen
|
|
131
|
+
self._image_edit = image_edit
|
|
132
|
+
# Dedicated LLM for the structured-JSON breakdown step.
|
|
133
|
+
# Lazy-loaded on first use from settings.json::system1, which
|
|
134
|
+
# is typically a model that's good at strict JSON output (e.g.
|
|
135
|
+
# GLM-5.1) — distinct from the agent's primary LLM (system2)
|
|
136
|
+
# used for chat reasoning.
|
|
137
|
+
self._breakdown_llm: Any | None = None
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def input_schema(self) -> dict:
|
|
141
|
+
return {
|
|
142
|
+
"type": "object",
|
|
143
|
+
"properties": {
|
|
144
|
+
"story": {
|
|
145
|
+
"type": "string",
|
|
146
|
+
"description": (
|
|
147
|
+
"The story or scene description. Can be a few "
|
|
148
|
+
"lines or several paragraphs. Include character "
|
|
149
|
+
"names + visual descriptors so panels stay "
|
|
150
|
+
"consistent."
|
|
151
|
+
),
|
|
152
|
+
},
|
|
153
|
+
"panels": {
|
|
154
|
+
"type": "integer",
|
|
155
|
+
"description": "Number of panels (default 6, max 12).",
|
|
156
|
+
"default": 6,
|
|
157
|
+
},
|
|
158
|
+
"style": {
|
|
159
|
+
"type": "string",
|
|
160
|
+
"description": (
|
|
161
|
+
"Visual style hint, e.g. 'comic book panels with "
|
|
162
|
+
"halftone dots', 'watercolor children's book', "
|
|
163
|
+
"'noir film stills', 'anime'. Applied to every "
|
|
164
|
+
"panel for stylistic consistency."
|
|
165
|
+
),
|
|
166
|
+
},
|
|
167
|
+
"size": {
|
|
168
|
+
"type": "string",
|
|
169
|
+
"description": "Pixel size per panel, e.g. '1024x1024'. Default 1024x1024.",
|
|
170
|
+
"default": "1024x1024",
|
|
171
|
+
},
|
|
172
|
+
"seed": {
|
|
173
|
+
"type": "integer",
|
|
174
|
+
"description": "Seed for panel 1 only (Kontext panels follow the chain).",
|
|
175
|
+
},
|
|
176
|
+
},
|
|
177
|
+
"required": ["story"],
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
def set_file_store(self, fs: Any) -> None:
|
|
181
|
+
self._file_store = fs
|
|
182
|
+
|
|
183
|
+
def set_llm(self, llm: Any) -> None:
|
|
184
|
+
self._llm = llm
|
|
185
|
+
|
|
186
|
+
def set_image_gen(self, gen: ImageGen) -> None:
|
|
187
|
+
self._image_gen = gen
|
|
188
|
+
|
|
189
|
+
def set_image_edit(self, edit: ImageEdit) -> None:
|
|
190
|
+
self._image_edit = edit
|
|
191
|
+
|
|
192
|
+
async def _breakdown_story(
|
|
193
|
+
self, story: str, panels: int, style: str,
|
|
194
|
+
) -> dict[str, Any]:
|
|
195
|
+
"""Call the LLM to extract a structured storyboard plan.
|
|
196
|
+
|
|
197
|
+
Returns:
|
|
198
|
+
{
|
|
199
|
+
"art_style": str,
|
|
200
|
+
"characters": [{"name": str, "visual_description": str}, ...],
|
|
201
|
+
"panels": [
|
|
202
|
+
{"panel_text": str, "scene_action": str,
|
|
203
|
+
"character_names_in_scene": [str, ...]}
|
|
204
|
+
]
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
Falls back to a heuristic breakdown if the LLM is unavailable
|
|
208
|
+
or returns malformed JSON — never raises.
|
|
209
|
+
"""
|
|
210
|
+
# Heuristic fallback used both when no LLM is wired and when
|
|
211
|
+
# JSON parsing fails. Splits on sentences and packs each into
|
|
212
|
+
# a panel with no character extraction.
|
|
213
|
+
def _fallback() -> dict[str, Any]:
|
|
214
|
+
sentences = re.split(r"(?<=[.!?])\s+", story.strip())
|
|
215
|
+
sentences = [s for s in sentences if s][:panels] or [story]
|
|
216
|
+
return {
|
|
217
|
+
"art_style": (style or "warm storybook illustration").strip(),
|
|
218
|
+
"characters": [],
|
|
219
|
+
"panels": [
|
|
220
|
+
{
|
|
221
|
+
"panel_text": s,
|
|
222
|
+
"scene_action": s,
|
|
223
|
+
"character_names_in_scene": [],
|
|
224
|
+
}
|
|
225
|
+
for s in sentences
|
|
226
|
+
],
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
# Resolve the breakdown LLM. Prefer the dedicated `system1`
|
|
230
|
+
# model from settings (typically GLM-5.1, strong at strict
|
|
231
|
+
# JSON). Fall back to the agent's main LLM if system1 isn't
|
|
232
|
+
# configured or instantiation fails.
|
|
233
|
+
breakdown_llm = self._breakdown_llm
|
|
234
|
+
if breakdown_llm is None:
|
|
235
|
+
try:
|
|
236
|
+
from core.settings import Settings
|
|
237
|
+
from llm.provider import LLMProvider
|
|
238
|
+
cfg = Settings.load()
|
|
239
|
+
model_id = cfg.get("system1")
|
|
240
|
+
if model_id:
|
|
241
|
+
breakdown_llm = LLMProvider(model=model_id)
|
|
242
|
+
self._breakdown_llm = breakdown_llm
|
|
243
|
+
logger.info(
|
|
244
|
+
f"Storyboard breakdown will use system1 model: {model_id}"
|
|
245
|
+
)
|
|
246
|
+
except Exception as e:
|
|
247
|
+
logger.warning(
|
|
248
|
+
f"could not init system1 LLM for breakdown ({e}); "
|
|
249
|
+
f"falling back to primary LLM"
|
|
250
|
+
)
|
|
251
|
+
if breakdown_llm is None:
|
|
252
|
+
breakdown_llm = self._llm
|
|
253
|
+
if breakdown_llm is None:
|
|
254
|
+
return _fallback()
|
|
255
|
+
|
|
256
|
+
user = (
|
|
257
|
+
f"Script:\n{story}\n\nProduce a structured storyboard with "
|
|
258
|
+
f"exactly {panels} panels. Visual style hint from the user: "
|
|
259
|
+
f"{style or '(none — choose something appropriate)'}.\n"
|
|
260
|
+
f"Remember: output a single JSON object as specified."
|
|
261
|
+
)
|
|
262
|
+
# Hard-bound the LLM breakdown so a slow / runaway thinking
|
|
263
|
+
# model can't make the SSE stream stall past the browser's
|
|
264
|
+
# fetch timeout. 90s is generous for 6 panels of structured
|
|
265
|
+
# JSON; if it's not back by then, the user is better served
|
|
266
|
+
# by the heuristic fallback than by a hung connection.
|
|
267
|
+
try:
|
|
268
|
+
import asyncio as _asyncio
|
|
269
|
+
resp = await _asyncio.wait_for(
|
|
270
|
+
breakdown_llm.chat(
|
|
271
|
+
messages=[
|
|
272
|
+
{"role": "system", "content": _BREAKDOWN_SYSTEM},
|
|
273
|
+
{"role": "user", "content": user},
|
|
274
|
+
],
|
|
275
|
+
# Reasoning models (GLM-5.1, Kimi) consume tokens
|
|
276
|
+
# in their `thinking` block. 5000 leaves headroom
|
|
277
|
+
# for both the chain-of-thought and the JSON.
|
|
278
|
+
max_tokens=5000,
|
|
279
|
+
temperature=0.5,
|
|
280
|
+
),
|
|
281
|
+
timeout=90.0,
|
|
282
|
+
)
|
|
283
|
+
except _asyncio.TimeoutError:
|
|
284
|
+
logger.warning(
|
|
285
|
+
"breakdown LLM call timed out at 90s; using fallback"
|
|
286
|
+
)
|
|
287
|
+
return _fallback()
|
|
288
|
+
except Exception as e:
|
|
289
|
+
logger.warning(f"breakdown LLM call failed ({e}); using fallback")
|
|
290
|
+
return _fallback()
|
|
291
|
+
|
|
292
|
+
# Some thinking-models return content="" with the JSON tucked
|
|
293
|
+
# inside the `thinking` block. Search both before giving up.
|
|
294
|
+
content_raw = (resp.content or "").strip()
|
|
295
|
+
if not content_raw:
|
|
296
|
+
content_raw = (getattr(resp, "thinking", "") or "").strip()
|
|
297
|
+
raw = _strip_code_fence(content_raw)
|
|
298
|
+
try:
|
|
299
|
+
plan = json.loads(raw)
|
|
300
|
+
except Exception as e:
|
|
301
|
+
# Try to recover the JSON object embedded in surrounding prose
|
|
302
|
+
m = re.search(r"\{[\s\S]+\}", raw)
|
|
303
|
+
if m:
|
|
304
|
+
try:
|
|
305
|
+
plan = json.loads(m.group(0))
|
|
306
|
+
except Exception:
|
|
307
|
+
logger.warning(
|
|
308
|
+
f"breakdown JSON un-parseable ({e}); using fallback"
|
|
309
|
+
)
|
|
310
|
+
return _fallback()
|
|
311
|
+
else:
|
|
312
|
+
logger.warning(
|
|
313
|
+
f"breakdown returned no JSON ({e}); using fallback"
|
|
314
|
+
)
|
|
315
|
+
return _fallback()
|
|
316
|
+
|
|
317
|
+
if not isinstance(plan, dict) or "panels" not in plan:
|
|
318
|
+
logger.warning("breakdown JSON missing `panels`; using fallback")
|
|
319
|
+
return _fallback()
|
|
320
|
+
|
|
321
|
+
# Normalise + clamp to N panels
|
|
322
|
+
plan["art_style"] = str(
|
|
323
|
+
plan.get("art_style") or style or "warm storybook illustration"
|
|
324
|
+
).strip()
|
|
325
|
+
chars = plan.get("characters") or []
|
|
326
|
+
plan["characters"] = [
|
|
327
|
+
{
|
|
328
|
+
"name": str(c.get("name", "")).strip(),
|
|
329
|
+
"visual_description": str(
|
|
330
|
+
c.get("visual_description", "")
|
|
331
|
+
).strip(),
|
|
332
|
+
}
|
|
333
|
+
for c in chars
|
|
334
|
+
if isinstance(c, dict) and c.get("name")
|
|
335
|
+
]
|
|
336
|
+
normd_panels = []
|
|
337
|
+
for p in (plan.get("panels") or [])[:panels]:
|
|
338
|
+
if not isinstance(p, dict):
|
|
339
|
+
continue
|
|
340
|
+
normd_panels.append({
|
|
341
|
+
"panel_text": str(p.get("panel_text", "")).strip(),
|
|
342
|
+
"scene_action": str(p.get("scene_action", "")).strip(),
|
|
343
|
+
"character_names_in_scene": [
|
|
344
|
+
str(n).strip()
|
|
345
|
+
for n in (p.get("character_names_in_scene") or [])
|
|
346
|
+
if isinstance(n, str) and str(n).strip()
|
|
347
|
+
],
|
|
348
|
+
})
|
|
349
|
+
if not normd_panels:
|
|
350
|
+
return _fallback()
|
|
351
|
+
plan["panels"] = normd_panels
|
|
352
|
+
return plan
|
|
353
|
+
|
|
354
|
+
@staticmethod
|
|
355
|
+
def _panel_prompt(
|
|
356
|
+
plan: dict[str, Any],
|
|
357
|
+
panel: dict[str, Any],
|
|
358
|
+
*,
|
|
359
|
+
include_full_cast: bool = False,
|
|
360
|
+
) -> str:
|
|
361
|
+
"""Build the full prompt for one panel.
|
|
362
|
+
|
|
363
|
+
Always includes:
|
|
364
|
+
- global `art_style`
|
|
365
|
+
- panel `scene_action`
|
|
366
|
+
- visual descriptors of characters listed `in_scene`
|
|
367
|
+
|
|
368
|
+
When `include_full_cast=True` (used for Kontext panels 2..N),
|
|
369
|
+
also appends a "World cast (for visual continuity)" block with
|
|
370
|
+
every other named character's descriptor — strong consistency
|
|
371
|
+
anchor across the storyboard. Skipped for panel 1 (text→image
|
|
372
|
+
mode) because FLUX-schnell would risk rendering off-scene
|
|
373
|
+
characters that the prompt is reminding it about.
|
|
374
|
+
"""
|
|
375
|
+
parts: list[str] = []
|
|
376
|
+
if plan.get("art_style"):
|
|
377
|
+
parts.append(plan["art_style"])
|
|
378
|
+
if panel.get("scene_action"):
|
|
379
|
+
parts.append(panel["scene_action"])
|
|
380
|
+
|
|
381
|
+
cast = {
|
|
382
|
+
c["name"].lower(): c["visual_description"]
|
|
383
|
+
for c in plan.get("characters") or []
|
|
384
|
+
if c.get("name")
|
|
385
|
+
}
|
|
386
|
+
in_scene = [n for n in panel.get("character_names_in_scene") or []]
|
|
387
|
+
in_scene_lower = {n.lower() for n in in_scene}
|
|
388
|
+
|
|
389
|
+
for n in in_scene:
|
|
390
|
+
desc = cast.get(n.lower())
|
|
391
|
+
parts.append(f"{n}: {desc}" if desc else n)
|
|
392
|
+
|
|
393
|
+
if include_full_cast:
|
|
394
|
+
offscene = [
|
|
395
|
+
c for c in plan.get("characters") or []
|
|
396
|
+
if c.get("name") and c["name"].lower() not in in_scene_lower
|
|
397
|
+
]
|
|
398
|
+
if offscene:
|
|
399
|
+
world = " | ".join(
|
|
400
|
+
f"{c['name']}: {c['visual_description']}"
|
|
401
|
+
for c in offscene
|
|
402
|
+
)
|
|
403
|
+
parts.append(
|
|
404
|
+
"World cast (visual continuity reference, only if "
|
|
405
|
+
f"present in this panel): {world}"
|
|
406
|
+
)
|
|
407
|
+
return ". ".join(p.strip(". ") for p in parts if p) + "."
|
|
408
|
+
|
|
409
|
+
def _release_image_gen(self) -> None:
|
|
410
|
+
"""Free the FLUX-schnell pipeline + GPU memory.
|
|
411
|
+
|
|
412
|
+
Called after panel 1 in the storyboard pipeline so the
|
|
413
|
+
FluxKontextPipeline can be loaded without offload-hook
|
|
414
|
+
collisions. Best-effort: any exception is swallowed because
|
|
415
|
+
we don't want pipeline cleanup to abort the storyboard.
|
|
416
|
+
"""
|
|
417
|
+
gen = self._image_gen
|
|
418
|
+
if gen is None:
|
|
419
|
+
return
|
|
420
|
+
try:
|
|
421
|
+
# DiffusersImageGen owns a `_pipe` attribute. Drop it so
|
|
422
|
+
# accelerate's hooks deregister and CUDA tensors get
|
|
423
|
+
# collected. Force-empty the cache too.
|
|
424
|
+
inner = getattr(gen, "_pipe", None)
|
|
425
|
+
if inner is not None:
|
|
426
|
+
# Move components to CPU first to make sure offloaded
|
|
427
|
+
# weights actually leave the GPU.
|
|
428
|
+
try:
|
|
429
|
+
inner.to("cpu")
|
|
430
|
+
except Exception:
|
|
431
|
+
pass
|
|
432
|
+
setattr(gen, "_pipe", None)
|
|
433
|
+
self._image_gen = None
|
|
434
|
+
import gc, torch
|
|
435
|
+
gc.collect()
|
|
436
|
+
if torch.cuda.is_available():
|
|
437
|
+
torch.cuda.empty_cache()
|
|
438
|
+
logger.info("Storyboard: released FLUX-schnell pipeline before Kontext")
|
|
439
|
+
except Exception as e:
|
|
440
|
+
logger.warning(f"Storyboard: image_gen release failed: {e}")
|
|
441
|
+
|
|
442
|
+
def _ensure_backends(self) -> None:
|
|
443
|
+
if self._image_gen is None:
|
|
444
|
+
# Default: same FLUX-schnell weights Draw uses
|
|
445
|
+
self._image_gen = DiffusersImageGen(
|
|
446
|
+
model="black-forest-labs/FLUX.1-schnell",
|
|
447
|
+
)
|
|
448
|
+
if self._image_edit is None:
|
|
449
|
+
self._image_edit = DiffusersImageEdit()
|
|
450
|
+
|
|
451
|
+
async def execute_streaming(self, **kwargs: Any):
|
|
452
|
+
"""Async generator that yields progress events as panels render.
|
|
453
|
+
|
|
454
|
+
Yields dicts of shape:
|
|
455
|
+
{"type": "breakdown", "plan": {...}}
|
|
456
|
+
{"type": "panel", "index": int, "file_id": str,
|
|
457
|
+
"scene_action": str, "panel_text": str,
|
|
458
|
+
"character_names_in_scene": [str, ...]}
|
|
459
|
+
{"type": "done", "panel_count": int, "panel_file_ids": [str, ...]}
|
|
460
|
+
{"type": "error", "message": str}
|
|
461
|
+
|
|
462
|
+
Used by the /storyboard/generate SSE endpoint. The non-streaming
|
|
463
|
+
`execute()` consumes this same generator under the hood.
|
|
464
|
+
"""
|
|
465
|
+
story = (kwargs.get("story") or "").strip()
|
|
466
|
+
if not story:
|
|
467
|
+
yield {"type": "error", "message": "`story` is required"}
|
|
468
|
+
return
|
|
469
|
+
if self._file_store is None:
|
|
470
|
+
yield {
|
|
471
|
+
"type": "error",
|
|
472
|
+
"message": "Storyboard is not wired to a FileStore.",
|
|
473
|
+
}
|
|
474
|
+
return
|
|
475
|
+
|
|
476
|
+
panels = max(1, min(12, int(kwargs.get("panels") or 6)))
|
|
477
|
+
style = (kwargs.get("style") or "").strip()
|
|
478
|
+
size = kwargs.get("size") or "1024x1024"
|
|
479
|
+
seed = kwargs.get("seed")
|
|
480
|
+
self._ensure_backends()
|
|
481
|
+
|
|
482
|
+
# Surface a "started" event before the slow LLM breakdown call
|
|
483
|
+
# so the SSE connection isn't idle for ~30-90 s — the React
|
|
484
|
+
# client (and proxies) treat long idle streams as broken.
|
|
485
|
+
yield {
|
|
486
|
+
"type": "started",
|
|
487
|
+
"message": (
|
|
488
|
+
f"Analyzing script and extracting cast for "
|
|
489
|
+
f"{panels} panels..."
|
|
490
|
+
),
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
# 1) Breakdown
|
|
494
|
+
plan = await self._breakdown_story(story, panels, style)
|
|
495
|
+
plan_panels = plan.get("panels") or []
|
|
496
|
+
if not plan_panels:
|
|
497
|
+
yield {"type": "error", "message": "story produced no panels"}
|
|
498
|
+
return
|
|
499
|
+
yield {"type": "breakdown", "plan": plan}
|
|
500
|
+
|
|
501
|
+
# 2) Panel 1 — Draw (text→image; only in-scene characters
|
|
502
|
+
# so FLUX-schnell doesn't hallucinate off-scene cast).
|
|
503
|
+
gen_kwargs: dict[str, Any] = {"size": size}
|
|
504
|
+
if seed is not None:
|
|
505
|
+
gen_kwargs["seed"] = int(seed)
|
|
506
|
+
prompt1 = self._panel_prompt(plan, plan_panels[0])
|
|
507
|
+
try:
|
|
508
|
+
png1 = await self._image_gen.generate(prompt1, **gen_kwargs)
|
|
509
|
+
except Exception as e:
|
|
510
|
+
logger.exception("Storyboard panel 1 failed")
|
|
511
|
+
yield {"type": "error", "message": f"panel 1 failed: {e}"}
|
|
512
|
+
return
|
|
513
|
+
rec1 = self._save_panel(
|
|
514
|
+
png1, 1, plan_panels[0].get("scene_action", ""),
|
|
515
|
+
)
|
|
516
|
+
yield {
|
|
517
|
+
"type": "panel",
|
|
518
|
+
"index": 1,
|
|
519
|
+
"file_id": rec1.id,
|
|
520
|
+
**{k: plan_panels[0].get(k, "") for k in (
|
|
521
|
+
"scene_action", "panel_text", "character_names_in_scene",
|
|
522
|
+
)},
|
|
523
|
+
}
|
|
524
|
+
# Fix 1 (anti-drift): every subsequent panel keys off PANEL 1
|
|
525
|
+
# — never the immediately previous panel — so the canonical
|
|
526
|
+
# look is at most 1 hop away from any panel.
|
|
527
|
+
anchor_bytes = png1
|
|
528
|
+
ids: list[str] = [rec1.id]
|
|
529
|
+
|
|
530
|
+
# Free the FLUX-schnell pipeline before Kontext loads. See
|
|
531
|
+
# `execute()` above for the full rationale — accelerate
|
|
532
|
+
# offload hooks collide otherwise.
|
|
533
|
+
self._release_image_gen()
|
|
534
|
+
|
|
535
|
+
# 3) Panels 2..N — Kontext, anchored on panel 1
|
|
536
|
+
for i, p in enumerate(plan_panels[1:], start=2):
|
|
537
|
+
panel_prompt = self._panel_prompt(
|
|
538
|
+
plan, p, include_full_cast=True,
|
|
539
|
+
)
|
|
540
|
+
instruction = (
|
|
541
|
+
f"Same characters (faces / outfits), same art style, "
|
|
542
|
+
f"same lighting and palette as the source image. "
|
|
543
|
+
f"New scene: {panel_prompt}"
|
|
544
|
+
)
|
|
545
|
+
try:
|
|
546
|
+
png_n = await self._image_edit.edit(
|
|
547
|
+
image_bytes=anchor_bytes,
|
|
548
|
+
prompt=instruction,
|
|
549
|
+
mode="edit",
|
|
550
|
+
size=size,
|
|
551
|
+
)
|
|
552
|
+
except Exception as e:
|
|
553
|
+
logger.exception(f"Storyboard panel {i} failed")
|
|
554
|
+
yield {
|
|
555
|
+
"type": "error",
|
|
556
|
+
"message": f"panel {i} failed: {e}",
|
|
557
|
+
}
|
|
558
|
+
yield {
|
|
559
|
+
"type": "done",
|
|
560
|
+
"panel_count": len(ids),
|
|
561
|
+
"panel_file_ids": ids,
|
|
562
|
+
}
|
|
563
|
+
return
|
|
564
|
+
rec_n = self._save_panel(png_n, i, p.get("scene_action", ""))
|
|
565
|
+
ids.append(rec_n.id)
|
|
566
|
+
yield {
|
|
567
|
+
"type": "panel",
|
|
568
|
+
"index": i,
|
|
569
|
+
"file_id": rec_n.id,
|
|
570
|
+
**{k: p.get(k, "") for k in (
|
|
571
|
+
"scene_action", "panel_text", "character_names_in_scene",
|
|
572
|
+
)},
|
|
573
|
+
}
|
|
574
|
+
# Note: deliberately NOT updating anchor_bytes — keeps
|
|
575
|
+
# panel 1 as the persistent style/character anchor.
|
|
576
|
+
|
|
577
|
+
yield {
|
|
578
|
+
"type": "done",
|
|
579
|
+
"panel_count": len(ids),
|
|
580
|
+
"panel_file_ids": ids,
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
async def execute(self, **kwargs: Any) -> ToolResult:
|
|
584
|
+
story = (kwargs.get("story") or "").strip()
|
|
585
|
+
if not story:
|
|
586
|
+
return ToolResult(
|
|
587
|
+
tool_name=self.name, status="error",
|
|
588
|
+
error="`story` is required",
|
|
589
|
+
)
|
|
590
|
+
if self._file_store is None:
|
|
591
|
+
return ToolResult(
|
|
592
|
+
tool_name=self.name, status="error",
|
|
593
|
+
error="Storyboard is not wired to a FileStore.",
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
panels = max(1, min(12, int(kwargs.get("panels") or 6)))
|
|
597
|
+
style = (kwargs.get("style") or "").strip()
|
|
598
|
+
size = kwargs.get("size") or "1024x1024"
|
|
599
|
+
seed = kwargs.get("seed")
|
|
600
|
+
|
|
601
|
+
self._ensure_backends()
|
|
602
|
+
|
|
603
|
+
# 1) Structured breakdown — art style, cast, panels
|
|
604
|
+
plan = await self._breakdown_story(story, panels, style)
|
|
605
|
+
plan_panels = plan.get("panels") or []
|
|
606
|
+
if not plan_panels:
|
|
607
|
+
return ToolResult(
|
|
608
|
+
tool_name=self.name, status="error",
|
|
609
|
+
error="story produced no panels",
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
# 2) Panel 1 — Draw with full structured prompt
|
|
613
|
+
gen_kwargs: dict[str, Any] = {"size": size}
|
|
614
|
+
if seed is not None:
|
|
615
|
+
gen_kwargs["seed"] = int(seed)
|
|
616
|
+
prompt1 = self._panel_prompt(plan, plan_panels[0])
|
|
617
|
+
try:
|
|
618
|
+
png1 = await self._image_gen.generate(prompt1, **gen_kwargs)
|
|
619
|
+
except Exception as e:
|
|
620
|
+
logger.exception("Storyboard panel 1 failed")
|
|
621
|
+
return ToolResult(
|
|
622
|
+
tool_name=self.name, status="error",
|
|
623
|
+
error=f"panel 1 generation failed: {e}",
|
|
624
|
+
)
|
|
625
|
+
# Fix 1: pin the anchor to panel 1 so subsequent panels can't
|
|
626
|
+
# drift more than 1 hop from the canonical look.
|
|
627
|
+
anchor_bytes = png1
|
|
628
|
+
records: list[tuple[dict[str, Any], Any]] = [(
|
|
629
|
+
plan_panels[0],
|
|
630
|
+
self._save_panel(png1, 1, plan_panels[0].get("scene_action", "")),
|
|
631
|
+
)]
|
|
632
|
+
|
|
633
|
+
# Fix: free the FLUX-schnell pipeline before loading Kontext.
|
|
634
|
+
# When both pipelines are resident with `enable_sequential_
|
|
635
|
+
# cpu_offload`, accelerate's offload hooks can collide and
|
|
636
|
+
# one pipeline's scheduler reads the other's sigma array,
|
|
637
|
+
# causing "index N out of bounds for size N" mid-denoise on
|
|
638
|
+
# panel 2. We won't need Draw again in this storyboard, so
|
|
639
|
+
# release it.
|
|
640
|
+
self._release_image_gen()
|
|
641
|
+
|
|
642
|
+
# 3) Panels 2..N — Kontext keyed on PANEL 1 (not previous)
|
|
643
|
+
for i, p in enumerate(plan_panels[1:], start=2):
|
|
644
|
+
panel_prompt = self._panel_prompt(plan, p, include_full_cast=True)
|
|
645
|
+
instruction = (
|
|
646
|
+
f"Same characters (faces / outfits), same art style, "
|
|
647
|
+
f"same lighting and palette as the source image. "
|
|
648
|
+
f"New scene: {panel_prompt}"
|
|
649
|
+
)
|
|
650
|
+
try:
|
|
651
|
+
png_n = await self._image_edit.edit(
|
|
652
|
+
image_bytes=anchor_bytes,
|
|
653
|
+
prompt=instruction,
|
|
654
|
+
mode="edit",
|
|
655
|
+
size=size,
|
|
656
|
+
)
|
|
657
|
+
except Exception as e:
|
|
658
|
+
logger.exception(f"Storyboard panel {i} failed")
|
|
659
|
+
break
|
|
660
|
+
records.append((
|
|
661
|
+
p,
|
|
662
|
+
self._save_panel(png_n, i, p.get("scene_action", "")),
|
|
663
|
+
))
|
|
664
|
+
# anchor_bytes stays = panel 1 (no chain accumulation)
|
|
665
|
+
|
|
666
|
+
# 4) Render gallery markdown
|
|
667
|
+
gallery: list[str] = [
|
|
668
|
+
f"**Storyboard — {len(records)} panel(s)**",
|
|
669
|
+
f"*Style:* {plan.get('art_style', '')}",
|
|
670
|
+
]
|
|
671
|
+
if plan.get("characters"):
|
|
672
|
+
gallery.append("*Cast:* " + ", ".join(
|
|
673
|
+
f"**{c['name']}** ({c['visual_description'][:60]}…)"
|
|
674
|
+
for c in plan["characters"]
|
|
675
|
+
))
|
|
676
|
+
gallery.append("")
|
|
677
|
+
ids: list[str] = []
|
|
678
|
+
for i, (p, rec) in enumerate(records, start=1):
|
|
679
|
+
url = f"http://127.0.0.1:8000/files/{rec.id}/content"
|
|
680
|
+
viewer = f"http://127.0.0.1:8000/artifact/{rec.id}"
|
|
681
|
+
beat = p.get("scene_action") or p.get("panel_text") or ""
|
|
682
|
+
short = beat[:160] + ("…" if len(beat) > 160 else "")
|
|
683
|
+
gallery.append(f"**Panel {i}.** {short}")
|
|
684
|
+
gallery.append(f"")
|
|
685
|
+
gallery.append(f"[📄 viewer]({viewer})")
|
|
686
|
+
gallery.append("")
|
|
687
|
+
ids.append(rec.id)
|
|
688
|
+
|
|
689
|
+
return ToolResult(
|
|
690
|
+
tool_name=self.name, status="success",
|
|
691
|
+
output="\n".join(gallery),
|
|
692
|
+
metadata={
|
|
693
|
+
"panel_count": len(records),
|
|
694
|
+
"panel_file_ids": ids,
|
|
695
|
+
"art_style": plan.get("art_style"),
|
|
696
|
+
"characters": plan.get("characters"),
|
|
697
|
+
"panels": [
|
|
698
|
+
{
|
|
699
|
+
"panel_text": p.get("panel_text", ""),
|
|
700
|
+
"scene_action": p.get("scene_action", ""),
|
|
701
|
+
"character_names_in_scene": p.get(
|
|
702
|
+
"character_names_in_scene", []
|
|
703
|
+
),
|
|
704
|
+
"file_id": rec.id,
|
|
705
|
+
}
|
|
706
|
+
for p, rec in records
|
|
707
|
+
],
|
|
708
|
+
},
|
|
709
|
+
)
|
|
710
|
+
|
|
711
|
+
def _save_panel(self, png: bytes, idx: int, beat: str) -> Any:
|
|
712
|
+
"""Save panel bytes to FileStore and return the FileRecord."""
|
|
713
|
+
with tempfile.NamedTemporaryFile(
|
|
714
|
+
suffix=".png", delete=False,
|
|
715
|
+
) as tmp:
|
|
716
|
+
tmp.write(png)
|
|
717
|
+
tmp_path = Path(tmp.name)
|
|
718
|
+
try:
|
|
719
|
+
slug = "".join(
|
|
720
|
+
c for c in beat[:30]
|
|
721
|
+
if c.isalnum() or c in " _-"
|
|
722
|
+
).strip().replace(" ", "_") or "panel"
|
|
723
|
+
return self._file_store.upload(
|
|
724
|
+
tmp_path, filename=f"panel_{idx:02d}_{slug}.png",
|
|
725
|
+
)
|
|
726
|
+
finally:
|
|
727
|
+
tmp_path.unlink(missing_ok=True)
|