caudate-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. api/__init__.py +5 -0
  2. api/anthropic_compat.py +1518 -0
  3. api/artifact_viewer.py +366 -0
  4. api/caudate_middleware.py +618 -0
  5. api/forge_bootstrapper_routes.py +377 -0
  6. api/forge_routes.py +630 -0
  7. api/forge_system_routes.py +294 -0
  8. api/openai_compat.py +1993 -0
  9. api/server.py +667 -0
  10. api/storyboard_page.py +677 -0
  11. caudate_cli-0.1.0.dist-info/METADATA +354 -0
  12. caudate_cli-0.1.0.dist-info/RECORD +153 -0
  13. caudate_cli-0.1.0.dist-info/WHEEL +5 -0
  14. caudate_cli-0.1.0.dist-info/entry_points.txt +2 -0
  15. caudate_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
  16. caudate_cli-0.1.0.dist-info/top_level.txt +14 -0
  17. cognos_mcp/__init__.py +4 -0
  18. cognos_mcp/bridge.py +41 -0
  19. cognos_mcp/client.py +70 -0
  20. cognos_mcp/config.py +49 -0
  21. cognos_mcp/server.py +66 -0
  22. config.py +82 -0
  23. core/__init__.py +0 -0
  24. core/agent.py +468 -0
  25. core/agentic_loop.py +731 -0
  26. core/anthropic_auth.py +91 -0
  27. core/background.py +113 -0
  28. core/banner.py +134 -0
  29. core/bootstrap.py +292 -0
  30. core/citations.py +131 -0
  31. core/compaction.py +109 -0
  32. core/constitution.py +198 -0
  33. core/diff_viewer.py +87 -0
  34. core/export.py +85 -0
  35. core/file_refs.py +119 -0
  36. core/files.py +199 -0
  37. core/hooks.py +209 -0
  38. core/image.py +599 -0
  39. core/input.py +91 -0
  40. core/loop.py +238 -0
  41. core/memory_md.py +147 -0
  42. core/notifications.py +99 -0
  43. core/ownership.py +181 -0
  44. core/paste.py +81 -0
  45. core/permissions.py +210 -0
  46. core/plan_mode.py +215 -0
  47. core/sandbox_prompt.py +185 -0
  48. core/scheduler.py +195 -0
  49. core/schemas.py +202 -0
  50. core/session.py +90 -0
  51. core/settings.py +132 -0
  52. core/skills.py +398 -0
  53. core/slash_commands.py +977 -0
  54. core/statusline.py +61 -0
  55. core/subagent.py +300 -0
  56. core/thinking.py +50 -0
  57. core/updater.py +122 -0
  58. core/usage.py +109 -0
  59. core/worktree.py +93 -0
  60. execution/__init__.py +0 -0
  61. execution/executor.py +329 -0
  62. execution/plugins.py +108 -0
  63. execution/tools/__init__.py +0 -0
  64. execution/tools/agent_tool.py +107 -0
  65. execution/tools/agentic_tool.py +297 -0
  66. execution/tools/artifact_tool.py +191 -0
  67. execution/tools/ask_user_question_tool.py +137 -0
  68. execution/tools/base.py +81 -0
  69. execution/tools/calculator_tool.py +137 -0
  70. execution/tools/cognos_card_tool.py +124 -0
  71. execution/tools/cron_tool.py +215 -0
  72. execution/tools/datetime_tool.py +215 -0
  73. execution/tools/describe_image_tool.py +161 -0
  74. execution/tools/draw_tool.py +164 -0
  75. execution/tools/edit_image_tool.py +262 -0
  76. execution/tools/edit_tool.py +245 -0
  77. execution/tools/file_tool.py +90 -0
  78. execution/tools/find_anywhere_tool.py +255 -0
  79. execution/tools/forge_feature_tools.py +377 -0
  80. execution/tools/glob_tool.py +59 -0
  81. execution/tools/grep_tool.py +89 -0
  82. execution/tools/http_request_tool.py +224 -0
  83. execution/tools/load_skill_tool.py +104 -0
  84. execution/tools/longcat_avatar_tool.py +384 -0
  85. execution/tools/mcp_tool.py +100 -0
  86. execution/tools/notebook_tool.py +279 -0
  87. execution/tools/openapi_tool.py +440 -0
  88. execution/tools/plan_mode_tool.py +95 -0
  89. execution/tools/push_notification_tool.py +157 -0
  90. execution/tools/python_tool.py +61 -0
  91. execution/tools/respond_tool.py +40 -0
  92. execution/tools/sandbox_tool.py +378 -0
  93. execution/tools/search_tool.py +153 -0
  94. execution/tools/semantic_search_tool.py +106 -0
  95. execution/tools/shell_tool.py +283 -0
  96. execution/tools/speak_tool.py +134 -0
  97. execution/tools/storyboard_tool.py +727 -0
  98. execution/tools/system_info_tool.py +212 -0
  99. execution/tools/task_tool.py +323 -0
  100. execution/tools/think_tool.py +49 -0
  101. execution/tools/transcribe_audio_tool.py +86 -0
  102. execution/tools/update_memory_tool.py +92 -0
  103. execution/tools/web_fetch_tool.py +82 -0
  104. execution/tools/worktree_tool.py +174 -0
  105. llm/__init__.py +0 -0
  106. llm/fallback.py +116 -0
  107. llm/models.py +320 -0
  108. llm/provider.py +1356 -0
  109. llm/router.py +373 -0
  110. main.py +1889 -0
  111. memory/__init__.py +0 -0
  112. memory/episodic.py +99 -0
  113. memory/procedural.py +145 -0
  114. memory/semantic.py +71 -0
  115. memory/working.py +64 -0
  116. nn/__init__.py +43 -0
  117. nn/auto_evolve.py +245 -0
  118. nn/caudate.py +136 -0
  119. nn/config.py +141 -0
  120. nn/consolidator.py +81 -0
  121. nn/data.py +1635 -0
  122. nn/encoder.py +258 -0
  123. nn/forge_advisor.py +303 -0
  124. nn/format.py +235 -0
  125. nn/heads.py +432 -0
  126. nn/observer.py +994 -0
  127. nn/policy.py +214 -0
  128. nn/runtime.py +343 -0
  129. nn/scorer.py +175 -0
  130. nn/trainer.py +515 -0
  131. nn/vision.py +352 -0
  132. personality/__init__.py +23 -0
  133. personality/engine.py +129 -0
  134. personality/identity.py +144 -0
  135. personality/inner_voice.py +100 -0
  136. personality/mood.py +205 -0
  137. planning/__init__.py +0 -0
  138. planning/dev_server.py +221 -0
  139. planning/forge_models.py +718 -0
  140. planning/orchestrator.py +1363 -0
  141. planning/planner.py +451 -0
  142. planning/task_graph.py +61 -0
  143. reflection/__init__.py +0 -0
  144. reflection/meta_learner.py +156 -0
  145. reflection/reflector.py +127 -0
  146. ui/__init__.py +5 -0
  147. ui/display.py +88 -0
  148. voice/__init__.py +0 -0
  149. voice/conversation.py +125 -0
  150. voice/listener.py +111 -0
  151. voice/speaker.py +59 -0
  152. voice/stt.py +126 -0
  153. voice/tts.py +214 -0
@@ -0,0 +1,727 @@
1
+ """Storyboard — turn a story description into a sequence of images.
2
+
3
+ Pipeline (per request):
4
+ 1. Ask the LLM to break the story into N panel-sized beats.
5
+ 2. Generate panel 1 with FLUX-schnell text→image (Draw backend).
6
+ 3. For panels 2..N, use FLUX.1-Kontext with the previous panel as
7
+ the reference image — Kontext preserves character identity and
8
+ setting across panels, which is the whole point of a storyboard.
9
+ 4. Persist every panel to the FileStore.
10
+ 5. Return a markdown gallery the chat UI renders inline (one image
11
+ per beat, captioned with the beat description).
12
+
13
+ Time budget on a 24 GB 3090 with `enable_sequential_cpu_offload`:
14
+ - Panel 1 (Draw, 4 steps): ~10–30 s
15
+ - Panel N>1 (Kontext, 28 steps): ~6–10 min each
16
+ So a 6-panel story ≈ 35–50 min. Use `panels` carefully.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import logging
23
+ import re
24
+ import tempfile
25
+ from pathlib import Path
26
+ from typing import Any
27
+
28
+ from core.image import (
29
+ DiffusersImageEdit,
30
+ DiffusersImageGen,
31
+ ImageEdit,
32
+ ImageGen,
33
+ )
34
+ from core.schemas import ToolResult
35
+ from execution.tools.base import BaseTool
36
+
37
+ logger = logging.getLogger(__name__)
38
+
39
+
40
+ _BREAKDOWN_SYSTEM = """You output a JSON object and nothing else.
41
+ Do NOT include reasoning, analysis, commentary, planning steps, or
42
+ prose of any kind. No code fences, no markdown, no preamble — only
43
+ the raw JSON object. The first character of your response must be
44
+ `{` and the last must be `}`.
45
+
46
+ You are a story-to-storyboard planner. Given a script or scene
47
+ description, produce a JSON object that captures the structure
48
+ needed to render a visually consistent storyboard.
49
+
50
+ Use these EXACT keys (do not invent your own; do not use synonyms
51
+ like `description`, `panel_number`, `dialogue`, etc.):
52
+
53
+ {
54
+ "art_style": "<global art-style description applied to every panel.
55
+ Be precise: medium (2D digital, watercolor, oil),
56
+ line quality, colour palette, lighting, era. ~30 words.>",
57
+ "characters": [
58
+ {
59
+ "name": "<short proper name used to refer to this character>",
60
+ "visual_description": "<concrete physical traits: hair colour
61
+ and style, eye colour, age range, height,
62
+ specific clothing items, distinguishing
63
+ accessories. ~30 words. Generic enough that
64
+ a diffusion model can render it.>"
65
+ }
66
+ ],
67
+ "panels": [
68
+ {
69
+ "panel_text": "<the relevant slice of the original script>",
70
+ "scene_action": "<what is happening visually in this panel:
71
+ location, action, framing, time of day. ~25 words.>",
72
+ "character_names_in_scene": ["<name1>", "<name2>"]
73
+ }
74
+ ]
75
+ }
76
+
77
+ Rules:
78
+ - Use exactly N panels (the user will specify N).
79
+ - Reuse the same character names across every panel they appear in.
80
+ Do NOT invent new names mid-storyboard for the same person.
81
+ - Every name in a panel's `character_names_in_scene` MUST also appear
82
+ in the top-level `characters` list.
83
+ - Keep `scene_action` purely visual (no internal monologue, no
84
+ inaudible thoughts, no spoken dialogue lines).
85
+ """
86
+
87
+
88
+ def _strip_code_fence(s: str) -> str:
89
+ s = s.strip()
90
+ m = re.match(r"^```(?:json)?\s*(.*?)\s*```$", s, re.DOTALL)
91
+ return m.group(1) if m else s
92
+
93
+
94
+ class StoryboardTool(BaseTool):
95
+ mutates = True
96
+ name = "Storyboard"
97
+ description = (
98
+ "Turn a story description into a SEQUENCE of images (a "
99
+ "storyboard / comic strip / picture-book spread).\n\n"
100
+ "Pipeline:\n"
101
+ " 1. The LLM extracts a structured plan from the script:\n"
102
+ " - global art_style description\n"
103
+ " - cast list (each character with consistent visual "
104
+ " descriptors: hair, clothing, distinguishing traits)\n"
105
+ " - per-panel scene_action + which characters appear\n"
106
+ " 2. Panel 1 is rendered with FLUX-schnell text-to-image,\n"
107
+ " prompt = art_style + scene_action + each character's\n"
108
+ " visual description.\n"
109
+ " 3. Panels 2..N use FLUX.1-Kontext img2img referencing the\n"
110
+ " previous panel, so faces/outfits/world stay consistent.\n"
111
+ "Returns a markdown gallery the chat UI renders inline.\n\n"
112
+ "Use this when the user asks for a comic, picture book, "
113
+ "storyboard, manga page, multiple images of the story, or to "
114
+ "'visualize this script'.\n\n"
115
+ "Time budget on a 24 GB GPU: ~30 s breakdown + ~30 s for panel "
116
+ "1 + ~7 min per subsequent panel. Default is 6 panels — pick "
117
+ "fewer for speed.\n\n"
118
+ "NOTE: panels 2..N use Kontext (non-commercial license)."
119
+ )
120
+
121
+ def __init__(
122
+ self,
123
+ file_store: Any | None = None,
124
+ llm: Any | None = None,
125
+ image_gen: ImageGen | None = None,
126
+ image_edit: ImageEdit | None = None,
127
+ ):
128
+ self._file_store = file_store
129
+ self._llm = llm
130
+ self._image_gen = image_gen
131
+ self._image_edit = image_edit
132
+ # Dedicated LLM for the structured-JSON breakdown step.
133
+ # Lazy-loaded on first use from settings.json::system1, which
134
+ # is typically a model that's good at strict JSON output (e.g.
135
+ # GLM-5.1) — distinct from the agent's primary LLM (system2)
136
+ # used for chat reasoning.
137
+ self._breakdown_llm: Any | None = None
138
+
139
+ @property
140
+ def input_schema(self) -> dict:
141
+ return {
142
+ "type": "object",
143
+ "properties": {
144
+ "story": {
145
+ "type": "string",
146
+ "description": (
147
+ "The story or scene description. Can be a few "
148
+ "lines or several paragraphs. Include character "
149
+ "names + visual descriptors so panels stay "
150
+ "consistent."
151
+ ),
152
+ },
153
+ "panels": {
154
+ "type": "integer",
155
+ "description": "Number of panels (default 6, max 12).",
156
+ "default": 6,
157
+ },
158
+ "style": {
159
+ "type": "string",
160
+ "description": (
161
+ "Visual style hint, e.g. 'comic book panels with "
162
+ "halftone dots', 'watercolor children's book', "
163
+ "'noir film stills', 'anime'. Applied to every "
164
+ "panel for stylistic consistency."
165
+ ),
166
+ },
167
+ "size": {
168
+ "type": "string",
169
+ "description": "Pixel size per panel, e.g. '1024x1024'. Default 1024x1024.",
170
+ "default": "1024x1024",
171
+ },
172
+ "seed": {
173
+ "type": "integer",
174
+ "description": "Seed for panel 1 only (Kontext panels follow the chain).",
175
+ },
176
+ },
177
+ "required": ["story"],
178
+ }
179
+
180
+ def set_file_store(self, fs: Any) -> None:
181
+ self._file_store = fs
182
+
183
+ def set_llm(self, llm: Any) -> None:
184
+ self._llm = llm
185
+
186
+ def set_image_gen(self, gen: ImageGen) -> None:
187
+ self._image_gen = gen
188
+
189
+ def set_image_edit(self, edit: ImageEdit) -> None:
190
+ self._image_edit = edit
191
+
192
+ async def _breakdown_story(
193
+ self, story: str, panels: int, style: str,
194
+ ) -> dict[str, Any]:
195
+ """Call the LLM to extract a structured storyboard plan.
196
+
197
+ Returns:
198
+ {
199
+ "art_style": str,
200
+ "characters": [{"name": str, "visual_description": str}, ...],
201
+ "panels": [
202
+ {"panel_text": str, "scene_action": str,
203
+ "character_names_in_scene": [str, ...]}
204
+ ]
205
+ }
206
+
207
+ Falls back to a heuristic breakdown if the LLM is unavailable
208
+ or returns malformed JSON — never raises.
209
+ """
210
+ # Heuristic fallback used both when no LLM is wired and when
211
+ # JSON parsing fails. Splits on sentences and packs each into
212
+ # a panel with no character extraction.
213
+ def _fallback() -> dict[str, Any]:
214
+ sentences = re.split(r"(?<=[.!?])\s+", story.strip())
215
+ sentences = [s for s in sentences if s][:panels] or [story]
216
+ return {
217
+ "art_style": (style or "warm storybook illustration").strip(),
218
+ "characters": [],
219
+ "panels": [
220
+ {
221
+ "panel_text": s,
222
+ "scene_action": s,
223
+ "character_names_in_scene": [],
224
+ }
225
+ for s in sentences
226
+ ],
227
+ }
228
+
229
+ # Resolve the breakdown LLM. Prefer the dedicated `system1`
230
+ # model from settings (typically GLM-5.1, strong at strict
231
+ # JSON). Fall back to the agent's main LLM if system1 isn't
232
+ # configured or instantiation fails.
233
+ breakdown_llm = self._breakdown_llm
234
+ if breakdown_llm is None:
235
+ try:
236
+ from core.settings import Settings
237
+ from llm.provider import LLMProvider
238
+ cfg = Settings.load()
239
+ model_id = cfg.get("system1")
240
+ if model_id:
241
+ breakdown_llm = LLMProvider(model=model_id)
242
+ self._breakdown_llm = breakdown_llm
243
+ logger.info(
244
+ f"Storyboard breakdown will use system1 model: {model_id}"
245
+ )
246
+ except Exception as e:
247
+ logger.warning(
248
+ f"could not init system1 LLM for breakdown ({e}); "
249
+ f"falling back to primary LLM"
250
+ )
251
+ if breakdown_llm is None:
252
+ breakdown_llm = self._llm
253
+ if breakdown_llm is None:
254
+ return _fallback()
255
+
256
+ user = (
257
+ f"Script:\n{story}\n\nProduce a structured storyboard with "
258
+ f"exactly {panels} panels. Visual style hint from the user: "
259
+ f"{style or '(none — choose something appropriate)'}.\n"
260
+ f"Remember: output a single JSON object as specified."
261
+ )
262
+ # Hard-bound the LLM breakdown so a slow / runaway thinking
263
+ # model can't make the SSE stream stall past the browser's
264
+ # fetch timeout. 90s is generous for 6 panels of structured
265
+ # JSON; if it's not back by then, the user is better served
266
+ # by the heuristic fallback than by a hung connection.
267
+ try:
268
+ import asyncio as _asyncio
269
+ resp = await _asyncio.wait_for(
270
+ breakdown_llm.chat(
271
+ messages=[
272
+ {"role": "system", "content": _BREAKDOWN_SYSTEM},
273
+ {"role": "user", "content": user},
274
+ ],
275
+ # Reasoning models (GLM-5.1, Kimi) consume tokens
276
+ # in their `thinking` block. 5000 leaves headroom
277
+ # for both the chain-of-thought and the JSON.
278
+ max_tokens=5000,
279
+ temperature=0.5,
280
+ ),
281
+ timeout=90.0,
282
+ )
283
+ except _asyncio.TimeoutError:
284
+ logger.warning(
285
+ "breakdown LLM call timed out at 90s; using fallback"
286
+ )
287
+ return _fallback()
288
+ except Exception as e:
289
+ logger.warning(f"breakdown LLM call failed ({e}); using fallback")
290
+ return _fallback()
291
+
292
+ # Some thinking-models return content="" with the JSON tucked
293
+ # inside the `thinking` block. Search both before giving up.
294
+ content_raw = (resp.content or "").strip()
295
+ if not content_raw:
296
+ content_raw = (getattr(resp, "thinking", "") or "").strip()
297
+ raw = _strip_code_fence(content_raw)
298
+ try:
299
+ plan = json.loads(raw)
300
+ except Exception as e:
301
+ # Try to recover the JSON object embedded in surrounding prose
302
+ m = re.search(r"\{[\s\S]+\}", raw)
303
+ if m:
304
+ try:
305
+ plan = json.loads(m.group(0))
306
+ except Exception:
307
+ logger.warning(
308
+ f"breakdown JSON un-parseable ({e}); using fallback"
309
+ )
310
+ return _fallback()
311
+ else:
312
+ logger.warning(
313
+ f"breakdown returned no JSON ({e}); using fallback"
314
+ )
315
+ return _fallback()
316
+
317
+ if not isinstance(plan, dict) or "panels" not in plan:
318
+ logger.warning("breakdown JSON missing `panels`; using fallback")
319
+ return _fallback()
320
+
321
+ # Normalise + clamp to N panels
322
+ plan["art_style"] = str(
323
+ plan.get("art_style") or style or "warm storybook illustration"
324
+ ).strip()
325
+ chars = plan.get("characters") or []
326
+ plan["characters"] = [
327
+ {
328
+ "name": str(c.get("name", "")).strip(),
329
+ "visual_description": str(
330
+ c.get("visual_description", "")
331
+ ).strip(),
332
+ }
333
+ for c in chars
334
+ if isinstance(c, dict) and c.get("name")
335
+ ]
336
+ normd_panels = []
337
+ for p in (plan.get("panels") or [])[:panels]:
338
+ if not isinstance(p, dict):
339
+ continue
340
+ normd_panels.append({
341
+ "panel_text": str(p.get("panel_text", "")).strip(),
342
+ "scene_action": str(p.get("scene_action", "")).strip(),
343
+ "character_names_in_scene": [
344
+ str(n).strip()
345
+ for n in (p.get("character_names_in_scene") or [])
346
+ if isinstance(n, str) and str(n).strip()
347
+ ],
348
+ })
349
+ if not normd_panels:
350
+ return _fallback()
351
+ plan["panels"] = normd_panels
352
+ return plan
353
+
354
+ @staticmethod
355
+ def _panel_prompt(
356
+ plan: dict[str, Any],
357
+ panel: dict[str, Any],
358
+ *,
359
+ include_full_cast: bool = False,
360
+ ) -> str:
361
+ """Build the full prompt for one panel.
362
+
363
+ Always includes:
364
+ - global `art_style`
365
+ - panel `scene_action`
366
+ - visual descriptors of characters listed `in_scene`
367
+
368
+ When `include_full_cast=True` (used for Kontext panels 2..N),
369
+ also appends a "World cast (for visual continuity)" block with
370
+ every other named character's descriptor — strong consistency
371
+ anchor across the storyboard. Skipped for panel 1 (text→image
372
+ mode) because FLUX-schnell would risk rendering off-scene
373
+ characters that the prompt is reminding it about.
374
+ """
375
+ parts: list[str] = []
376
+ if plan.get("art_style"):
377
+ parts.append(plan["art_style"])
378
+ if panel.get("scene_action"):
379
+ parts.append(panel["scene_action"])
380
+
381
+ cast = {
382
+ c["name"].lower(): c["visual_description"]
383
+ for c in plan.get("characters") or []
384
+ if c.get("name")
385
+ }
386
+ in_scene = [n for n in panel.get("character_names_in_scene") or []]
387
+ in_scene_lower = {n.lower() for n in in_scene}
388
+
389
+ for n in in_scene:
390
+ desc = cast.get(n.lower())
391
+ parts.append(f"{n}: {desc}" if desc else n)
392
+
393
+ if include_full_cast:
394
+ offscene = [
395
+ c for c in plan.get("characters") or []
396
+ if c.get("name") and c["name"].lower() not in in_scene_lower
397
+ ]
398
+ if offscene:
399
+ world = " | ".join(
400
+ f"{c['name']}: {c['visual_description']}"
401
+ for c in offscene
402
+ )
403
+ parts.append(
404
+ "World cast (visual continuity reference, only if "
405
+ f"present in this panel): {world}"
406
+ )
407
+ return ". ".join(p.strip(". ") for p in parts if p) + "."
408
+
409
+ def _release_image_gen(self) -> None:
410
+ """Free the FLUX-schnell pipeline + GPU memory.
411
+
412
+ Called after panel 1 in the storyboard pipeline so the
413
+ FluxKontextPipeline can be loaded without offload-hook
414
+ collisions. Best-effort: any exception is swallowed because
415
+ we don't want pipeline cleanup to abort the storyboard.
416
+ """
417
+ gen = self._image_gen
418
+ if gen is None:
419
+ return
420
+ try:
421
+ # DiffusersImageGen owns a `_pipe` attribute. Drop it so
422
+ # accelerate's hooks deregister and CUDA tensors get
423
+ # collected. Force-empty the cache too.
424
+ inner = getattr(gen, "_pipe", None)
425
+ if inner is not None:
426
+ # Move components to CPU first to make sure offloaded
427
+ # weights actually leave the GPU.
428
+ try:
429
+ inner.to("cpu")
430
+ except Exception:
431
+ pass
432
+ setattr(gen, "_pipe", None)
433
+ self._image_gen = None
434
+ import gc, torch
435
+ gc.collect()
436
+ if torch.cuda.is_available():
437
+ torch.cuda.empty_cache()
438
+ logger.info("Storyboard: released FLUX-schnell pipeline before Kontext")
439
+ except Exception as e:
440
+ logger.warning(f"Storyboard: image_gen release failed: {e}")
441
+
442
+ def _ensure_backends(self) -> None:
443
+ if self._image_gen is None:
444
+ # Default: same FLUX-schnell weights Draw uses
445
+ self._image_gen = DiffusersImageGen(
446
+ model="black-forest-labs/FLUX.1-schnell",
447
+ )
448
+ if self._image_edit is None:
449
+ self._image_edit = DiffusersImageEdit()
450
+
451
+ async def execute_streaming(self, **kwargs: Any):
452
+ """Async generator that yields progress events as panels render.
453
+
454
+ Yields dicts of shape:
455
+ {"type": "breakdown", "plan": {...}}
456
+ {"type": "panel", "index": int, "file_id": str,
457
+ "scene_action": str, "panel_text": str,
458
+ "character_names_in_scene": [str, ...]}
459
+ {"type": "done", "panel_count": int, "panel_file_ids": [str, ...]}
460
+ {"type": "error", "message": str}
461
+
462
+ Used by the /storyboard/generate SSE endpoint. The non-streaming
463
+ `execute()` consumes this same generator under the hood.
464
+ """
465
+ story = (kwargs.get("story") or "").strip()
466
+ if not story:
467
+ yield {"type": "error", "message": "`story` is required"}
468
+ return
469
+ if self._file_store is None:
470
+ yield {
471
+ "type": "error",
472
+ "message": "Storyboard is not wired to a FileStore.",
473
+ }
474
+ return
475
+
476
+ panels = max(1, min(12, int(kwargs.get("panels") or 6)))
477
+ style = (kwargs.get("style") or "").strip()
478
+ size = kwargs.get("size") or "1024x1024"
479
+ seed = kwargs.get("seed")
480
+ self._ensure_backends()
481
+
482
+ # Surface a "started" event before the slow LLM breakdown call
483
+ # so the SSE connection isn't idle for ~30-90 s — the React
484
+ # client (and proxies) treat long idle streams as broken.
485
+ yield {
486
+ "type": "started",
487
+ "message": (
488
+ f"Analyzing script and extracting cast for "
489
+ f"{panels} panels..."
490
+ ),
491
+ }
492
+
493
+ # 1) Breakdown
494
+ plan = await self._breakdown_story(story, panels, style)
495
+ plan_panels = plan.get("panels") or []
496
+ if not plan_panels:
497
+ yield {"type": "error", "message": "story produced no panels"}
498
+ return
499
+ yield {"type": "breakdown", "plan": plan}
500
+
501
+ # 2) Panel 1 — Draw (text→image; only in-scene characters
502
+ # so FLUX-schnell doesn't hallucinate off-scene cast).
503
+ gen_kwargs: dict[str, Any] = {"size": size}
504
+ if seed is not None:
505
+ gen_kwargs["seed"] = int(seed)
506
+ prompt1 = self._panel_prompt(plan, plan_panels[0])
507
+ try:
508
+ png1 = await self._image_gen.generate(prompt1, **gen_kwargs)
509
+ except Exception as e:
510
+ logger.exception("Storyboard panel 1 failed")
511
+ yield {"type": "error", "message": f"panel 1 failed: {e}"}
512
+ return
513
+ rec1 = self._save_panel(
514
+ png1, 1, plan_panels[0].get("scene_action", ""),
515
+ )
516
+ yield {
517
+ "type": "panel",
518
+ "index": 1,
519
+ "file_id": rec1.id,
520
+ **{k: plan_panels[0].get(k, "") for k in (
521
+ "scene_action", "panel_text", "character_names_in_scene",
522
+ )},
523
+ }
524
+ # Fix 1 (anti-drift): every subsequent panel keys off PANEL 1
525
+ # — never the immediately previous panel — so the canonical
526
+ # look is at most 1 hop away from any panel.
527
+ anchor_bytes = png1
528
+ ids: list[str] = [rec1.id]
529
+
530
+ # Free the FLUX-schnell pipeline before Kontext loads. See
531
+ # `execute()` above for the full rationale — accelerate
532
+ # offload hooks collide otherwise.
533
+ self._release_image_gen()
534
+
535
+ # 3) Panels 2..N — Kontext, anchored on panel 1
536
+ for i, p in enumerate(plan_panels[1:], start=2):
537
+ panel_prompt = self._panel_prompt(
538
+ plan, p, include_full_cast=True,
539
+ )
540
+ instruction = (
541
+ f"Same characters (faces / outfits), same art style, "
542
+ f"same lighting and palette as the source image. "
543
+ f"New scene: {panel_prompt}"
544
+ )
545
+ try:
546
+ png_n = await self._image_edit.edit(
547
+ image_bytes=anchor_bytes,
548
+ prompt=instruction,
549
+ mode="edit",
550
+ size=size,
551
+ )
552
+ except Exception as e:
553
+ logger.exception(f"Storyboard panel {i} failed")
554
+ yield {
555
+ "type": "error",
556
+ "message": f"panel {i} failed: {e}",
557
+ }
558
+ yield {
559
+ "type": "done",
560
+ "panel_count": len(ids),
561
+ "panel_file_ids": ids,
562
+ }
563
+ return
564
+ rec_n = self._save_panel(png_n, i, p.get("scene_action", ""))
565
+ ids.append(rec_n.id)
566
+ yield {
567
+ "type": "panel",
568
+ "index": i,
569
+ "file_id": rec_n.id,
570
+ **{k: p.get(k, "") for k in (
571
+ "scene_action", "panel_text", "character_names_in_scene",
572
+ )},
573
+ }
574
+ # Note: deliberately NOT updating anchor_bytes — keeps
575
+ # panel 1 as the persistent style/character anchor.
576
+
577
+ yield {
578
+ "type": "done",
579
+ "panel_count": len(ids),
580
+ "panel_file_ids": ids,
581
+ }
582
+
583
+ async def execute(self, **kwargs: Any) -> ToolResult:
584
+ story = (kwargs.get("story") or "").strip()
585
+ if not story:
586
+ return ToolResult(
587
+ tool_name=self.name, status="error",
588
+ error="`story` is required",
589
+ )
590
+ if self._file_store is None:
591
+ return ToolResult(
592
+ tool_name=self.name, status="error",
593
+ error="Storyboard is not wired to a FileStore.",
594
+ )
595
+
596
+ panels = max(1, min(12, int(kwargs.get("panels") or 6)))
597
+ style = (kwargs.get("style") or "").strip()
598
+ size = kwargs.get("size") or "1024x1024"
599
+ seed = kwargs.get("seed")
600
+
601
+ self._ensure_backends()
602
+
603
+ # 1) Structured breakdown — art style, cast, panels
604
+ plan = await self._breakdown_story(story, panels, style)
605
+ plan_panels = plan.get("panels") or []
606
+ if not plan_panels:
607
+ return ToolResult(
608
+ tool_name=self.name, status="error",
609
+ error="story produced no panels",
610
+ )
611
+
612
+ # 2) Panel 1 — Draw with full structured prompt
613
+ gen_kwargs: dict[str, Any] = {"size": size}
614
+ if seed is not None:
615
+ gen_kwargs["seed"] = int(seed)
616
+ prompt1 = self._panel_prompt(plan, plan_panels[0])
617
+ try:
618
+ png1 = await self._image_gen.generate(prompt1, **gen_kwargs)
619
+ except Exception as e:
620
+ logger.exception("Storyboard panel 1 failed")
621
+ return ToolResult(
622
+ tool_name=self.name, status="error",
623
+ error=f"panel 1 generation failed: {e}",
624
+ )
625
+ # Fix 1: pin the anchor to panel 1 so subsequent panels can't
626
+ # drift more than 1 hop from the canonical look.
627
+ anchor_bytes = png1
628
+ records: list[tuple[dict[str, Any], Any]] = [(
629
+ plan_panels[0],
630
+ self._save_panel(png1, 1, plan_panels[0].get("scene_action", "")),
631
+ )]
632
+
633
+ # Fix: free the FLUX-schnell pipeline before loading Kontext.
634
+ # When both pipelines are resident with `enable_sequential_
635
+ # cpu_offload`, accelerate's offload hooks can collide and
636
+ # one pipeline's scheduler reads the other's sigma array,
637
+ # causing "index N out of bounds for size N" mid-denoise on
638
+ # panel 2. We won't need Draw again in this storyboard, so
639
+ # release it.
640
+ self._release_image_gen()
641
+
642
+ # 3) Panels 2..N — Kontext keyed on PANEL 1 (not previous)
643
+ for i, p in enumerate(plan_panels[1:], start=2):
644
+ panel_prompt = self._panel_prompt(plan, p, include_full_cast=True)
645
+ instruction = (
646
+ f"Same characters (faces / outfits), same art style, "
647
+ f"same lighting and palette as the source image. "
648
+ f"New scene: {panel_prompt}"
649
+ )
650
+ try:
651
+ png_n = await self._image_edit.edit(
652
+ image_bytes=anchor_bytes,
653
+ prompt=instruction,
654
+ mode="edit",
655
+ size=size,
656
+ )
657
+ except Exception as e:
658
+ logger.exception(f"Storyboard panel {i} failed")
659
+ break
660
+ records.append((
661
+ p,
662
+ self._save_panel(png_n, i, p.get("scene_action", "")),
663
+ ))
664
+ # anchor_bytes stays = panel 1 (no chain accumulation)
665
+
666
+ # 4) Render gallery markdown
667
+ gallery: list[str] = [
668
+ f"**Storyboard — {len(records)} panel(s)**",
669
+ f"*Style:* {plan.get('art_style', '')}",
670
+ ]
671
+ if plan.get("characters"):
672
+ gallery.append("*Cast:* " + ", ".join(
673
+ f"**{c['name']}** ({c['visual_description'][:60]}…)"
674
+ for c in plan["characters"]
675
+ ))
676
+ gallery.append("")
677
+ ids: list[str] = []
678
+ for i, (p, rec) in enumerate(records, start=1):
679
+ url = f"http://127.0.0.1:8000/files/{rec.id}/content"
680
+ viewer = f"http://127.0.0.1:8000/artifact/{rec.id}"
681
+ beat = p.get("scene_action") or p.get("panel_text") or ""
682
+ short = beat[:160] + ("…" if len(beat) > 160 else "")
683
+ gallery.append(f"**Panel {i}.** {short}")
684
+ gallery.append(f"![{short}]({url})")
685
+ gallery.append(f"[📄 viewer]({viewer})")
686
+ gallery.append("")
687
+ ids.append(rec.id)
688
+
689
+ return ToolResult(
690
+ tool_name=self.name, status="success",
691
+ output="\n".join(gallery),
692
+ metadata={
693
+ "panel_count": len(records),
694
+ "panel_file_ids": ids,
695
+ "art_style": plan.get("art_style"),
696
+ "characters": plan.get("characters"),
697
+ "panels": [
698
+ {
699
+ "panel_text": p.get("panel_text", ""),
700
+ "scene_action": p.get("scene_action", ""),
701
+ "character_names_in_scene": p.get(
702
+ "character_names_in_scene", []
703
+ ),
704
+ "file_id": rec.id,
705
+ }
706
+ for p, rec in records
707
+ ],
708
+ },
709
+ )
710
+
711
+ def _save_panel(self, png: bytes, idx: int, beat: str) -> Any:
712
+ """Save panel bytes to FileStore and return the FileRecord."""
713
+ with tempfile.NamedTemporaryFile(
714
+ suffix=".png", delete=False,
715
+ ) as tmp:
716
+ tmp.write(png)
717
+ tmp_path = Path(tmp.name)
718
+ try:
719
+ slug = "".join(
720
+ c for c in beat[:30]
721
+ if c.isalnum() or c in " _-"
722
+ ).strip().replace(" ", "_") or "panel"
723
+ return self._file_store.upload(
724
+ tmp_path, filename=f"panel_{idx:02d}_{slug}.png",
725
+ )
726
+ finally:
727
+ tmp_path.unlink(missing_ok=True)