caudate-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api/__init__.py +5 -0
- api/anthropic_compat.py +1518 -0
- api/artifact_viewer.py +366 -0
- api/caudate_middleware.py +618 -0
- api/forge_bootstrapper_routes.py +377 -0
- api/forge_routes.py +630 -0
- api/forge_system_routes.py +294 -0
- api/openai_compat.py +1993 -0
- api/server.py +667 -0
- api/storyboard_page.py +677 -0
- caudate_cli-0.1.0.dist-info/METADATA +354 -0
- caudate_cli-0.1.0.dist-info/RECORD +153 -0
- caudate_cli-0.1.0.dist-info/WHEEL +5 -0
- caudate_cli-0.1.0.dist-info/entry_points.txt +2 -0
- caudate_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- caudate_cli-0.1.0.dist-info/top_level.txt +14 -0
- cognos_mcp/__init__.py +4 -0
- cognos_mcp/bridge.py +41 -0
- cognos_mcp/client.py +70 -0
- cognos_mcp/config.py +49 -0
- cognos_mcp/server.py +66 -0
- config.py +82 -0
- core/__init__.py +0 -0
- core/agent.py +468 -0
- core/agentic_loop.py +731 -0
- core/anthropic_auth.py +91 -0
- core/background.py +113 -0
- core/banner.py +134 -0
- core/bootstrap.py +292 -0
- core/citations.py +131 -0
- core/compaction.py +109 -0
- core/constitution.py +198 -0
- core/diff_viewer.py +87 -0
- core/export.py +85 -0
- core/file_refs.py +119 -0
- core/files.py +199 -0
- core/hooks.py +209 -0
- core/image.py +599 -0
- core/input.py +91 -0
- core/loop.py +238 -0
- core/memory_md.py +147 -0
- core/notifications.py +99 -0
- core/ownership.py +181 -0
- core/paste.py +81 -0
- core/permissions.py +210 -0
- core/plan_mode.py +215 -0
- core/sandbox_prompt.py +185 -0
- core/scheduler.py +195 -0
- core/schemas.py +202 -0
- core/session.py +90 -0
- core/settings.py +132 -0
- core/skills.py +398 -0
- core/slash_commands.py +977 -0
- core/statusline.py +61 -0
- core/subagent.py +300 -0
- core/thinking.py +50 -0
- core/updater.py +122 -0
- core/usage.py +109 -0
- core/worktree.py +93 -0
- execution/__init__.py +0 -0
- execution/executor.py +329 -0
- execution/plugins.py +108 -0
- execution/tools/__init__.py +0 -0
- execution/tools/agent_tool.py +107 -0
- execution/tools/agentic_tool.py +297 -0
- execution/tools/artifact_tool.py +191 -0
- execution/tools/ask_user_question_tool.py +137 -0
- execution/tools/base.py +81 -0
- execution/tools/calculator_tool.py +137 -0
- execution/tools/cognos_card_tool.py +124 -0
- execution/tools/cron_tool.py +215 -0
- execution/tools/datetime_tool.py +215 -0
- execution/tools/describe_image_tool.py +161 -0
- execution/tools/draw_tool.py +164 -0
- execution/tools/edit_image_tool.py +262 -0
- execution/tools/edit_tool.py +245 -0
- execution/tools/file_tool.py +90 -0
- execution/tools/find_anywhere_tool.py +255 -0
- execution/tools/forge_feature_tools.py +377 -0
- execution/tools/glob_tool.py +59 -0
- execution/tools/grep_tool.py +89 -0
- execution/tools/http_request_tool.py +224 -0
- execution/tools/load_skill_tool.py +104 -0
- execution/tools/longcat_avatar_tool.py +384 -0
- execution/tools/mcp_tool.py +100 -0
- execution/tools/notebook_tool.py +279 -0
- execution/tools/openapi_tool.py +440 -0
- execution/tools/plan_mode_tool.py +95 -0
- execution/tools/push_notification_tool.py +157 -0
- execution/tools/python_tool.py +61 -0
- execution/tools/respond_tool.py +40 -0
- execution/tools/sandbox_tool.py +378 -0
- execution/tools/search_tool.py +153 -0
- execution/tools/semantic_search_tool.py +106 -0
- execution/tools/shell_tool.py +283 -0
- execution/tools/speak_tool.py +134 -0
- execution/tools/storyboard_tool.py +727 -0
- execution/tools/system_info_tool.py +212 -0
- execution/tools/task_tool.py +323 -0
- execution/tools/think_tool.py +49 -0
- execution/tools/transcribe_audio_tool.py +86 -0
- execution/tools/update_memory_tool.py +92 -0
- execution/tools/web_fetch_tool.py +82 -0
- execution/tools/worktree_tool.py +174 -0
- llm/__init__.py +0 -0
- llm/fallback.py +116 -0
- llm/models.py +320 -0
- llm/provider.py +1356 -0
- llm/router.py +373 -0
- main.py +1889 -0
- memory/__init__.py +0 -0
- memory/episodic.py +99 -0
- memory/procedural.py +145 -0
- memory/semantic.py +71 -0
- memory/working.py +64 -0
- nn/__init__.py +43 -0
- nn/auto_evolve.py +245 -0
- nn/caudate.py +136 -0
- nn/config.py +141 -0
- nn/consolidator.py +81 -0
- nn/data.py +1635 -0
- nn/encoder.py +258 -0
- nn/forge_advisor.py +303 -0
- nn/format.py +235 -0
- nn/heads.py +432 -0
- nn/observer.py +994 -0
- nn/policy.py +214 -0
- nn/runtime.py +343 -0
- nn/scorer.py +175 -0
- nn/trainer.py +515 -0
- nn/vision.py +352 -0
- personality/__init__.py +23 -0
- personality/engine.py +129 -0
- personality/identity.py +144 -0
- personality/inner_voice.py +100 -0
- personality/mood.py +205 -0
- planning/__init__.py +0 -0
- planning/dev_server.py +221 -0
- planning/forge_models.py +718 -0
- planning/orchestrator.py +1363 -0
- planning/planner.py +451 -0
- planning/task_graph.py +61 -0
- reflection/__init__.py +0 -0
- reflection/meta_learner.py +156 -0
- reflection/reflector.py +127 -0
- ui/__init__.py +5 -0
- ui/display.py +88 -0
- voice/__init__.py +0 -0
- voice/conversation.py +125 -0
- voice/listener.py +111 -0
- voice/speaker.py +59 -0
- voice/stt.py +126 -0
- voice/tts.py +214 -0
core/image.py
ADDED
|
@@ -0,0 +1,599 @@
|
|
|
1
|
+
"""Image generation — pluggable backends behind a common interface.
|
|
2
|
+
|
|
3
|
+
Mirrors `voice/stt.py`'s pattern: a small `ImageGen` protocol, three
|
|
4
|
+
concrete backends, and a `make_image_gen(name)` factory. Whichever
|
|
5
|
+
backend is in use, the result is the same: a PNG file written to
|
|
6
|
+
`data/files/` via the FileStore, returning a `FileRecord` the agent
|
|
7
|
+
can reference in subsequent turns.
|
|
8
|
+
|
|
9
|
+
Backends:
|
|
10
|
+
|
|
11
|
+
- **DiffusersImageGen** (default, local) — Hugging Face `diffusers`
|
|
12
|
+
library. Default model is `stabilityai/sdxl-turbo` because it's
|
|
13
|
+
a single-step model that runs on a 6-8GB GPU. Swap to
|
|
14
|
+
`black-forest-labs/FLUX.1-schnell` if you have 24GB+.
|
|
15
|
+
|
|
16
|
+
- **ComfyUIImageGen** — HTTP proxy to a running ComfyUI server.
|
|
17
|
+
Uses ComfyUI's `/prompt` API. Good if you already run ComfyUI
|
|
18
|
+
with custom workflows.
|
|
19
|
+
|
|
20
|
+
- **LiteLLMImageGen** — cloud APIs via `litellm.aimage_generation`.
|
|
21
|
+
Works with `openai/gpt-image-1`, `dall-e-3`, Stability, etc.
|
|
22
|
+
Requires the relevant API key.
|
|
23
|
+
|
|
24
|
+
Backends are lazy-loaded — importing this module is cheap; the heavy
|
|
25
|
+
torch/diffusers deps only load when you actually call `.generate()`.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import asyncio
|
|
31
|
+
import base64
|
|
32
|
+
import io
|
|
33
|
+
import json
|
|
34
|
+
import logging
|
|
35
|
+
import tempfile
|
|
36
|
+
import urllib.parse
|
|
37
|
+
import urllib.request
|
|
38
|
+
import uuid
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
from typing import Any, Protocol
|
|
41
|
+
|
|
42
|
+
logger = logging.getLogger(__name__)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ImageGen(Protocol):
|
|
46
|
+
"""Common shape for any image generator."""
|
|
47
|
+
|
|
48
|
+
async def generate(
|
|
49
|
+
self,
|
|
50
|
+
prompt: str,
|
|
51
|
+
size: str = "1024x1024",
|
|
52
|
+
negative_prompt: str = "",
|
|
53
|
+
seed: int | None = None,
|
|
54
|
+
steps: int | None = None,
|
|
55
|
+
guidance_scale: float | None = None,
|
|
56
|
+
) -> bytes:
|
|
57
|
+
"""Return PNG bytes for the prompt. Raises on failure."""
|
|
58
|
+
...
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _parse_size(size: str) -> tuple[int, int]:
|
|
62
|
+
try:
|
|
63
|
+
w, h = size.lower().split("x")
|
|
64
|
+
return int(w), int(h)
|
|
65
|
+
except Exception:
|
|
66
|
+
return 1024, 1024
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# --------------------------------------------------------------------
|
|
70
|
+
# Diffusers (local)
|
|
71
|
+
# --------------------------------------------------------------------
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class DiffusersImageGen:
|
|
75
|
+
"""Hugging Face diffusers backend.
|
|
76
|
+
|
|
77
|
+
Lazy-loads the pipeline on first call. Picks a sensible scheduler
|
|
78
|
+
for the chosen model. Defaults to fp16 on CUDA, fp32 on CPU.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
def __init__(
|
|
82
|
+
self,
|
|
83
|
+
model: str = "stabilityai/sdxl-turbo",
|
|
84
|
+
device: str | None = None,
|
|
85
|
+
dtype: str | None = None,
|
|
86
|
+
):
|
|
87
|
+
self.model = model
|
|
88
|
+
self._device = device
|
|
89
|
+
self._dtype = dtype
|
|
90
|
+
self._pipe = None
|
|
91
|
+
|
|
92
|
+
def _ensure_loaded(self) -> None:
|
|
93
|
+
if self._pipe is not None:
|
|
94
|
+
return
|
|
95
|
+
# Skip transformers' opportunistic TF import — TF is broken/unused
|
|
96
|
+
# here and its import chain corrupts the FluxPipeline class load.
|
|
97
|
+
import os
|
|
98
|
+
os.environ.setdefault("USE_TF", "0")
|
|
99
|
+
os.environ.setdefault("TRANSFORMERS_NO_TF", "1")
|
|
100
|
+
try:
|
|
101
|
+
import torch
|
|
102
|
+
from diffusers import AutoPipelineForText2Image
|
|
103
|
+
except ImportError as e:
|
|
104
|
+
raise RuntimeError(
|
|
105
|
+
"Diffusers backend requested but `diffusers` / `torch` "
|
|
106
|
+
"are not installed. Run: pip install diffusers transformers torch accelerate"
|
|
107
|
+
) from e
|
|
108
|
+
|
|
109
|
+
device = self._device or ("cuda" if torch.cuda.is_available() else "cpu")
|
|
110
|
+
is_flux = "flux" in self.model.lower()
|
|
111
|
+
# FLUX is bf16-native — fp16 produces black/garbled outputs.
|
|
112
|
+
# Other SDXL-class models default to fp16 on CUDA.
|
|
113
|
+
if self._dtype:
|
|
114
|
+
dtype_name = self._dtype
|
|
115
|
+
elif is_flux and device == "cuda":
|
|
116
|
+
dtype_name = "bfloat16"
|
|
117
|
+
elif device == "cuda":
|
|
118
|
+
dtype_name = "float16"
|
|
119
|
+
else:
|
|
120
|
+
dtype_name = "float32"
|
|
121
|
+
torch_dtype = getattr(torch, dtype_name)
|
|
122
|
+
|
|
123
|
+
logger.info(f"Loading diffusers pipeline {self.model!r} on {device} ({dtype_name})...")
|
|
124
|
+
# FLUX has no `variant="fp16"` weights on HF — only bf16 / fp32.
|
|
125
|
+
pipe_kwargs: dict[str, Any] = {"torch_dtype": torch_dtype}
|
|
126
|
+
if dtype_name == "float16" and not is_flux:
|
|
127
|
+
pipe_kwargs["variant"] = "fp16"
|
|
128
|
+
pipe = AutoPipelineForText2Image.from_pretrained(self.model, **pipe_kwargs)
|
|
129
|
+
|
|
130
|
+
# FLUX-schnell at bf16 nominally needs ~22 GB; in practice
|
|
131
|
+
# `enable_model_cpu_offload()` (component-level offload) still
|
|
132
|
+
# OOMs on a 24 GB 3090 because the transformer + T5 + VAE all
|
|
133
|
+
# peak together. `enable_sequential_cpu_offload()` offloads at
|
|
134
|
+
# the layer/module level — VRAM peak drops to ~12 GB, at the
|
|
135
|
+
# cost of 2-3x slower inference. The only way to run FLUX on
|
|
136
|
+
# this card without OOM until we move to a 32 GB+ GPU.
|
|
137
|
+
if is_flux and device == "cuda":
|
|
138
|
+
pipe.enable_sequential_cpu_offload()
|
|
139
|
+
else:
|
|
140
|
+
pipe = pipe.to(device)
|
|
141
|
+
# SDXL-Turbo and SD-Turbo benefit from disabling the safety checker
|
|
142
|
+
# (rejects too aggressively at single-step) and CFG.
|
|
143
|
+
if "turbo" in self.model.lower():
|
|
144
|
+
pipe.set_progress_bar_config(disable=True)
|
|
145
|
+
self._pipe = pipe
|
|
146
|
+
self._torch_device = device
|
|
147
|
+
logger.info("Diffusers pipeline loaded.")
|
|
148
|
+
|
|
149
|
+
async def generate(
|
|
150
|
+
self,
|
|
151
|
+
prompt: str,
|
|
152
|
+
size: str = "1024x1024",
|
|
153
|
+
negative_prompt: str = "",
|
|
154
|
+
seed: int | None = None,
|
|
155
|
+
steps: int | None = None,
|
|
156
|
+
guidance_scale: float | None = None,
|
|
157
|
+
) -> bytes:
|
|
158
|
+
# Heavy CPU/GPU work — push off the event loop.
|
|
159
|
+
return await asyncio.to_thread(
|
|
160
|
+
self._sync_generate,
|
|
161
|
+
prompt, size, negative_prompt, seed, steps, guidance_scale,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
def _sync_generate(
|
|
165
|
+
self, prompt: str, size: str, negative_prompt: str,
|
|
166
|
+
seed: int | None, steps: int | None, guidance_scale: float | None,
|
|
167
|
+
) -> bytes:
|
|
168
|
+
import torch
|
|
169
|
+
self._ensure_loaded()
|
|
170
|
+
w, h = _parse_size(size)
|
|
171
|
+
|
|
172
|
+
# Defaults tuned per-model family.
|
|
173
|
+
is_turbo = "turbo" in self.model.lower()
|
|
174
|
+
is_flux_schnell = "flux.1-schnell" in self.model.lower()
|
|
175
|
+
if steps is None:
|
|
176
|
+
steps = 1 if is_turbo else (4 if is_flux_schnell else 25)
|
|
177
|
+
if guidance_scale is None:
|
|
178
|
+
guidance_scale = 0.0 if (is_turbo or is_flux_schnell) else 7.5
|
|
179
|
+
|
|
180
|
+
kwargs: dict[str, Any] = {
|
|
181
|
+
"prompt": prompt,
|
|
182
|
+
"width": w, "height": h,
|
|
183
|
+
"num_inference_steps": steps,
|
|
184
|
+
"guidance_scale": guidance_scale,
|
|
185
|
+
}
|
|
186
|
+
if negative_prompt and not is_turbo:
|
|
187
|
+
kwargs["negative_prompt"] = negative_prompt
|
|
188
|
+
if seed is not None:
|
|
189
|
+
generator = torch.Generator(device=self._torch_device).manual_seed(int(seed))
|
|
190
|
+
kwargs["generator"] = generator
|
|
191
|
+
|
|
192
|
+
result = self._pipe(**kwargs)
|
|
193
|
+
image = result.images[0]
|
|
194
|
+
buf = io.BytesIO()
|
|
195
|
+
image.save(buf, format="PNG")
|
|
196
|
+
return buf.getvalue()
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
# --------------------------------------------------------------------
|
|
200
|
+
# ComfyUI (HTTP proxy)
|
|
201
|
+
# --------------------------------------------------------------------
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
class ComfyUIImageGen:
|
|
205
|
+
"""Calls a running ComfyUI server's `/prompt` API.
|
|
206
|
+
|
|
207
|
+
Uses a minimal text-to-image workflow: CLIPTextEncode → KSampler →
|
|
208
|
+
VAEDecode → SaveImage. The user can override the workflow JSON via
|
|
209
|
+
`workflow=` for custom checkpoints / LoRAs / ControlNet.
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
def __init__(
|
|
213
|
+
self,
|
|
214
|
+
host: str = "http://127.0.0.1:8188",
|
|
215
|
+
checkpoint: str = "sd_xl_base_1.0.safetensors",
|
|
216
|
+
workflow: dict | None = None,
|
|
217
|
+
):
|
|
218
|
+
self.host = host.rstrip("/")
|
|
219
|
+
self.checkpoint = checkpoint
|
|
220
|
+
self.workflow = workflow
|
|
221
|
+
|
|
222
|
+
async def generate(
|
|
223
|
+
self,
|
|
224
|
+
prompt: str,
|
|
225
|
+
size: str = "1024x1024",
|
|
226
|
+
negative_prompt: str = "",
|
|
227
|
+
seed: int | None = None,
|
|
228
|
+
steps: int | None = None,
|
|
229
|
+
guidance_scale: float | None = None,
|
|
230
|
+
) -> bytes:
|
|
231
|
+
return await asyncio.to_thread(
|
|
232
|
+
self._sync_generate, prompt, size, negative_prompt, seed, steps, guidance_scale,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
def _sync_generate(
|
|
236
|
+
self, prompt: str, size: str, negative_prompt: str,
|
|
237
|
+
seed: int | None, steps: int | None, guidance_scale: float | None,
|
|
238
|
+
) -> bytes:
|
|
239
|
+
w, h = _parse_size(size)
|
|
240
|
+
seed = seed if seed is not None else int.from_bytes(uuid.uuid4().bytes[:4], "big")
|
|
241
|
+
steps = steps or 20
|
|
242
|
+
guidance_scale = guidance_scale or 7.0
|
|
243
|
+
|
|
244
|
+
wf = self.workflow or self._default_workflow(
|
|
245
|
+
prompt, negative_prompt, w, h, seed, steps, guidance_scale, self.checkpoint,
|
|
246
|
+
)
|
|
247
|
+
client_id = uuid.uuid4().hex
|
|
248
|
+
prompt_id = self._post_prompt(wf, client_id)
|
|
249
|
+
return self._fetch_result(prompt_id)
|
|
250
|
+
|
|
251
|
+
def _default_workflow(
|
|
252
|
+
self, prompt: str, negative: str, w: int, h: int,
|
|
253
|
+
seed: int, steps: int, cfg: float, checkpoint: str,
|
|
254
|
+
) -> dict:
|
|
255
|
+
# Standard text-to-image graph in ComfyUI's prompt API format.
|
|
256
|
+
return {
|
|
257
|
+
"3": {"inputs": {"seed": seed, "steps": steps, "cfg": cfg,
|
|
258
|
+
"sampler_name": "euler", "scheduler": "normal",
|
|
259
|
+
"denoise": 1, "model": ["4", 0],
|
|
260
|
+
"positive": ["6", 0], "negative": ["7", 0],
|
|
261
|
+
"latent_image": ["5", 0]},
|
|
262
|
+
"class_type": "KSampler"},
|
|
263
|
+
"4": {"inputs": {"ckpt_name": checkpoint},
|
|
264
|
+
"class_type": "CheckpointLoaderSimple"},
|
|
265
|
+
"5": {"inputs": {"width": w, "height": h, "batch_size": 1},
|
|
266
|
+
"class_type": "EmptyLatentImage"},
|
|
267
|
+
"6": {"inputs": {"text": prompt, "clip": ["4", 1]},
|
|
268
|
+
"class_type": "CLIPTextEncode"},
|
|
269
|
+
"7": {"inputs": {"text": negative or "blurry, low quality", "clip": ["4", 1]},
|
|
270
|
+
"class_type": "CLIPTextEncode"},
|
|
271
|
+
"8": {"inputs": {"samples": ["3", 0], "vae": ["4", 2]},
|
|
272
|
+
"class_type": "VAEDecode"},
|
|
273
|
+
"9": {"inputs": {"filename_prefix": "cognos", "images": ["8", 0]},
|
|
274
|
+
"class_type": "SaveImage"},
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
def _post_prompt(self, workflow: dict, client_id: str) -> str:
|
|
278
|
+
body = json.dumps({"prompt": workflow, "client_id": client_id}).encode()
|
|
279
|
+
req = urllib.request.Request(
|
|
280
|
+
f"{self.host}/prompt",
|
|
281
|
+
data=body, headers={"Content-Type": "application/json"},
|
|
282
|
+
)
|
|
283
|
+
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
284
|
+
data = json.loads(resp.read())
|
|
285
|
+
return data["prompt_id"]
|
|
286
|
+
|
|
287
|
+
def _fetch_result(self, prompt_id: str) -> bytes:
|
|
288
|
+
# Poll history endpoint until the run finishes.
|
|
289
|
+
import time
|
|
290
|
+
for _ in range(120):
|
|
291
|
+
time.sleep(0.5)
|
|
292
|
+
with urllib.request.urlopen(
|
|
293
|
+
f"{self.host}/history/{prompt_id}", timeout=10,
|
|
294
|
+
) as resp:
|
|
295
|
+
data = json.loads(resp.read())
|
|
296
|
+
if not data:
|
|
297
|
+
continue
|
|
298
|
+
run = data.get(prompt_id)
|
|
299
|
+
if not run or "outputs" not in run:
|
|
300
|
+
continue
|
|
301
|
+
for node_out in run["outputs"].values():
|
|
302
|
+
for img in node_out.get("images", []):
|
|
303
|
+
qs = urllib.parse.urlencode({
|
|
304
|
+
"filename": img["filename"],
|
|
305
|
+
"subfolder": img.get("subfolder", ""),
|
|
306
|
+
"type": img.get("type", "output"),
|
|
307
|
+
})
|
|
308
|
+
with urllib.request.urlopen(
|
|
309
|
+
f"{self.host}/view?{qs}", timeout=30,
|
|
310
|
+
) as iresp:
|
|
311
|
+
return iresp.read()
|
|
312
|
+
raise RuntimeError(f"ComfyUI run {prompt_id} timed out")
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
# --------------------------------------------------------------------
|
|
316
|
+
# LiteLLM cloud
|
|
317
|
+
# --------------------------------------------------------------------
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
class LiteLLMImageGen:
|
|
321
|
+
"""Cloud image gen via LiteLLM's image-generation interface.
|
|
322
|
+
|
|
323
|
+
Works with model ids like `openai/gpt-image-1`, `dall-e-3`,
|
|
324
|
+
`stability-ai/stable-diffusion-3-large`, etc. The API key for the
|
|
325
|
+
chosen provider must be in the environment.
|
|
326
|
+
"""
|
|
327
|
+
|
|
328
|
+
def __init__(self, model: str = "openai/gpt-image-1"):
|
|
329
|
+
self.model = model
|
|
330
|
+
|
|
331
|
+
async def generate(
|
|
332
|
+
self,
|
|
333
|
+
prompt: str,
|
|
334
|
+
size: str = "1024x1024",
|
|
335
|
+
negative_prompt: str = "",
|
|
336
|
+
seed: int | None = None,
|
|
337
|
+
steps: int | None = None,
|
|
338
|
+
guidance_scale: float | None = None,
|
|
339
|
+
) -> bytes:
|
|
340
|
+
try:
|
|
341
|
+
import litellm
|
|
342
|
+
except ImportError as e:
|
|
343
|
+
raise RuntimeError("LiteLLM backend requires litellm") from e
|
|
344
|
+
|
|
345
|
+
kwargs: dict[str, Any] = {"model": self.model, "prompt": prompt, "size": size}
|
|
346
|
+
if seed is not None:
|
|
347
|
+
kwargs["seed"] = seed
|
|
348
|
+
resp = await litellm.aimage_generation(**kwargs)
|
|
349
|
+
|
|
350
|
+
data = resp.data[0]
|
|
351
|
+
if getattr(data, "b64_json", None):
|
|
352
|
+
return base64.b64decode(data.b64_json)
|
|
353
|
+
url = getattr(data, "url", None)
|
|
354
|
+
if url:
|
|
355
|
+
with urllib.request.urlopen(url, timeout=30) as r:
|
|
356
|
+
return r.read()
|
|
357
|
+
raise RuntimeError(f"LiteLLM returned no image bytes or URL: {data}")
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
# --------------------------------------------------------------------
|
|
361
|
+
# Image edit (img2img + instruction)
|
|
362
|
+
# --------------------------------------------------------------------
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
class ImageEdit(Protocol):
|
|
366
|
+
"""Common shape for any image editor. Two modes:
|
|
367
|
+
|
|
368
|
+
- `reimagine`: img2img — preserves composition, restyles content.
|
|
369
|
+
Strength controls how much to deviate (0=identity, 1=full new).
|
|
370
|
+
- `edit`: instruction — "make the wings blue", "remove the cat",
|
|
371
|
+
etc. The model rewrites only what the instruction describes.
|
|
372
|
+
"""
|
|
373
|
+
|
|
374
|
+
async def edit(
|
|
375
|
+
self,
|
|
376
|
+
image_bytes: bytes,
|
|
377
|
+
prompt: str,
|
|
378
|
+
mode: str = "reimagine",
|
|
379
|
+
strength: float = 0.75,
|
|
380
|
+
size: str = "1024x1024",
|
|
381
|
+
seed: int | None = None,
|
|
382
|
+
steps: int | None = None,
|
|
383
|
+
guidance_scale: float | None = None,
|
|
384
|
+
) -> bytes:
|
|
385
|
+
...
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
class DiffusersImageEdit:
|
|
389
|
+
"""Two-mode FLUX image editor: img2img (FLUX-schnell) + Kontext.
|
|
390
|
+
|
|
391
|
+
Each mode lazy-loads its pipeline on first use. They share the
|
|
392
|
+
same `transformer` weights only when both run img2img-style; in
|
|
393
|
+
practice we keep them separate for clarity.
|
|
394
|
+
"""
|
|
395
|
+
|
|
396
|
+
def __init__(
|
|
397
|
+
self,
|
|
398
|
+
img2img_model: str = "black-forest-labs/FLUX.1-schnell",
|
|
399
|
+
kontext_model: str = "black-forest-labs/FLUX.1-Kontext-dev",
|
|
400
|
+
device: str | None = None,
|
|
401
|
+
dtype: str | None = None,
|
|
402
|
+
):
|
|
403
|
+
self.img2img_model = img2img_model
|
|
404
|
+
self.kontext_model = kontext_model
|
|
405
|
+
self._device = device
|
|
406
|
+
self._dtype = dtype
|
|
407
|
+
self._img2img_pipe = None
|
|
408
|
+
self._kontext_pipe = None
|
|
409
|
+
self._torch_device: str | None = None
|
|
410
|
+
|
|
411
|
+
def _ensure_loaded(self, mode: str) -> Any:
|
|
412
|
+
# Same TF-skip dance as DiffusersImageGen.
|
|
413
|
+
import os
|
|
414
|
+
os.environ.setdefault("USE_TF", "0")
|
|
415
|
+
os.environ.setdefault("TRANSFORMERS_NO_TF", "1")
|
|
416
|
+
import torch
|
|
417
|
+
|
|
418
|
+
if mode == "reimagine" and self._img2img_pipe is not None:
|
|
419
|
+
return self._img2img_pipe
|
|
420
|
+
if mode == "edit" and self._kontext_pipe is not None:
|
|
421
|
+
return self._kontext_pipe
|
|
422
|
+
|
|
423
|
+
device = self._device or ("cuda" if torch.cuda.is_available() else "cpu")
|
|
424
|
+
# FLUX is bf16-native (fp16 produces black/garbled outputs).
|
|
425
|
+
dtype_name = self._dtype or ("bfloat16" if device == "cuda" else "float32")
|
|
426
|
+
torch_dtype = getattr(torch, dtype_name)
|
|
427
|
+
self._torch_device = device
|
|
428
|
+
|
|
429
|
+
if mode == "reimagine":
|
|
430
|
+
from diffusers import FluxImg2ImgPipeline
|
|
431
|
+
logger.info(
|
|
432
|
+
f"Loading FluxImg2ImgPipeline {self.img2img_model!r} "
|
|
433
|
+
f"on {device} ({dtype_name})..."
|
|
434
|
+
)
|
|
435
|
+
pipe = FluxImg2ImgPipeline.from_pretrained(
|
|
436
|
+
self.img2img_model, torch_dtype=torch_dtype,
|
|
437
|
+
)
|
|
438
|
+
elif mode == "edit":
|
|
439
|
+
from diffusers import FluxKontextPipeline
|
|
440
|
+
logger.info(
|
|
441
|
+
f"Loading FluxKontextPipeline {self.kontext_model!r} "
|
|
442
|
+
f"on {device} ({dtype_name})..."
|
|
443
|
+
)
|
|
444
|
+
pipe = FluxKontextPipeline.from_pretrained(
|
|
445
|
+
self.kontext_model, torch_dtype=torch_dtype,
|
|
446
|
+
)
|
|
447
|
+
else:
|
|
448
|
+
raise ValueError(f"Unknown image-edit mode: {mode!r}")
|
|
449
|
+
|
|
450
|
+
# Same sequential-CPU-offload as DiffusersImageGen — required
|
|
451
|
+
# for FLUX on a 24 GB 3090 (model_cpu_offload OOMs).
|
|
452
|
+
if device == "cuda":
|
|
453
|
+
pipe.enable_sequential_cpu_offload()
|
|
454
|
+
else:
|
|
455
|
+
pipe = pipe.to(device)
|
|
456
|
+
|
|
457
|
+
if mode == "reimagine":
|
|
458
|
+
self._img2img_pipe = pipe
|
|
459
|
+
else:
|
|
460
|
+
self._kontext_pipe = pipe
|
|
461
|
+
logger.info(f"FLUX {mode} pipeline loaded.")
|
|
462
|
+
return pipe
|
|
463
|
+
|
|
464
|
+
async def edit(
|
|
465
|
+
self,
|
|
466
|
+
image_bytes: bytes,
|
|
467
|
+
prompt: str,
|
|
468
|
+
mode: str = "reimagine",
|
|
469
|
+
strength: float = 0.75,
|
|
470
|
+
size: str = "1024x1024",
|
|
471
|
+
seed: int | None = None,
|
|
472
|
+
steps: int | None = None,
|
|
473
|
+
guidance_scale: float | None = None,
|
|
474
|
+
) -> bytes:
|
|
475
|
+
return await asyncio.to_thread(
|
|
476
|
+
self._sync_edit, image_bytes, prompt, mode, strength,
|
|
477
|
+
size, seed, steps, guidance_scale,
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
def _sync_edit(
|
|
481
|
+
self, image_bytes: bytes, prompt: str, mode: str,
|
|
482
|
+
strength: float, size: str, seed: int | None,
|
|
483
|
+
steps: int | None, guidance_scale: float | None,
|
|
484
|
+
) -> bytes:
|
|
485
|
+
import torch
|
|
486
|
+
from PIL import Image
|
|
487
|
+
|
|
488
|
+
pipe = self._ensure_loaded(mode)
|
|
489
|
+
w, h = _parse_size(size)
|
|
490
|
+
src = Image.open(io.BytesIO(image_bytes)).convert("RGB")
|
|
491
|
+
# Resize source to target dims; FLUX expects multiples of 8.
|
|
492
|
+
src = src.resize((w - w % 8, h - h % 8), Image.LANCZOS)
|
|
493
|
+
|
|
494
|
+
# FLUX-schnell defaults: 4 steps, cfg=0.0. Kontext-dev
|
|
495
|
+
# is non-distilled — needs ~28 steps and cfg=2.5.
|
|
496
|
+
if steps is None:
|
|
497
|
+
steps = 4 if mode == "reimagine" else 28
|
|
498
|
+
if guidance_scale is None:
|
|
499
|
+
guidance_scale = 0.0 if mode == "reimagine" else 2.5
|
|
500
|
+
|
|
501
|
+
kwargs: dict[str, Any] = {
|
|
502
|
+
"image": src,
|
|
503
|
+
"prompt": prompt,
|
|
504
|
+
"num_inference_steps": steps,
|
|
505
|
+
"guidance_scale": guidance_scale,
|
|
506
|
+
}
|
|
507
|
+
if mode == "reimagine":
|
|
508
|
+
# img2img has explicit strength
|
|
509
|
+
kwargs["strength"] = max(0.0, min(1.0, float(strength)))
|
|
510
|
+
kwargs["width"] = src.width
|
|
511
|
+
kwargs["height"] = src.height
|
|
512
|
+
# Kontext doesn't take strength — it edits by instruction
|
|
513
|
+
# against the source. Width/height come from the source.
|
|
514
|
+
|
|
515
|
+
if seed is not None:
|
|
516
|
+
kwargs["generator"] = torch.Generator(
|
|
517
|
+
device=self._torch_device,
|
|
518
|
+
).manual_seed(int(seed))
|
|
519
|
+
|
|
520
|
+
result = pipe(**kwargs)
|
|
521
|
+
image = result.images[0]
|
|
522
|
+
buf = io.BytesIO()
|
|
523
|
+
image.save(buf, format="PNG")
|
|
524
|
+
return buf.getvalue()
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
# --------------------------------------------------------------------
|
|
528
|
+
# Factory
|
|
529
|
+
# --------------------------------------------------------------------
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def make_image_gen(name: str = "diffusers", **kwargs: Any) -> ImageGen:
|
|
533
|
+
n = name.lower()
|
|
534
|
+
if n == "diffusers":
|
|
535
|
+
return DiffusersImageGen(**kwargs)
|
|
536
|
+
if n in ("comfyui", "comfy"):
|
|
537
|
+
return ComfyUIImageGen(**kwargs)
|
|
538
|
+
if n in ("litellm", "openai", "cloud"):
|
|
539
|
+
return LiteLLMImageGen(**kwargs)
|
|
540
|
+
raise ValueError(
|
|
541
|
+
f"Unknown image backend: {name!r} "
|
|
542
|
+
"(try 'diffusers', 'comfyui', or 'litellm')"
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def make_image_edit(name: str = "diffusers", **kwargs: Any) -> ImageEdit:
|
|
547
|
+
"""Construct an image-editor backend. Currently only `diffusers`
|
|
548
|
+
is implemented (FLUX img2img + Kontext)."""
|
|
549
|
+
n = name.lower()
|
|
550
|
+
if n == "diffusers":
|
|
551
|
+
return DiffusersImageEdit(**kwargs)
|
|
552
|
+
raise ValueError(
|
|
553
|
+
f"Unknown image-edit backend: {name!r} (try 'diffusers')"
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
async def generate_to_file_store(
|
|
558
|
+
backend: ImageGen,
|
|
559
|
+
file_store: Any,
|
|
560
|
+
prompt: str,
|
|
561
|
+
size: str = "1024x1024",
|
|
562
|
+
**kwargs: Any,
|
|
563
|
+
) -> Any:
|
|
564
|
+
"""Generate an image and persist it via FileStore. Returns FileRecord."""
|
|
565
|
+
png = await backend.generate(prompt, size=size, **kwargs)
|
|
566
|
+
# Hand off to FileStore by writing to a tempfile first; FileStore
|
|
567
|
+
# owns the canonical path under data/files/.
|
|
568
|
+
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp:
|
|
569
|
+
tmp.write(png)
|
|
570
|
+
tmp_path = Path(tmp.name)
|
|
571
|
+
try:
|
|
572
|
+
slug = "".join(c for c in prompt[:40] if c.isalnum() or c in " _-").strip()
|
|
573
|
+
slug = slug.replace(" ", "_") or "image"
|
|
574
|
+
record = file_store.upload(tmp_path, filename=f"{slug}.png")
|
|
575
|
+
finally:
|
|
576
|
+
tmp_path.unlink(missing_ok=True)
|
|
577
|
+
return record
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
async def edit_to_file_store(
|
|
581
|
+
backend: ImageEdit,
|
|
582
|
+
file_store: Any,
|
|
583
|
+
image_bytes: bytes,
|
|
584
|
+
prompt: str,
|
|
585
|
+
mode: str = "reimagine",
|
|
586
|
+
**kwargs: Any,
|
|
587
|
+
) -> Any:
|
|
588
|
+
"""Run an edit and persist the result via FileStore."""
|
|
589
|
+
png = await backend.edit(image_bytes, prompt, mode=mode, **kwargs)
|
|
590
|
+
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp:
|
|
591
|
+
tmp.write(png)
|
|
592
|
+
tmp_path = Path(tmp.name)
|
|
593
|
+
try:
|
|
594
|
+
slug = "".join(c for c in prompt[:40] if c.isalnum() or c in " _-").strip()
|
|
595
|
+
slug = slug.replace(" ", "_") or "edited"
|
|
596
|
+
record = file_store.upload(tmp_path, filename=f"{slug}_{mode}.png")
|
|
597
|
+
finally:
|
|
598
|
+
tmp_path.unlink(missing_ok=True)
|
|
599
|
+
return record
|
core/input.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""REPL input — multi-line, history search, slash-command tab completion.
|
|
2
|
+
|
|
3
|
+
Wraps prompt_toolkit if available, falls back to console.input if not.
|
|
4
|
+
Three enhancements over the bare console prompt:
|
|
5
|
+
|
|
6
|
+
- Multi-line: Enter submits; Ctrl+J inserts a newline. Starting a
|
|
7
|
+
line with a triple-quote submits everything up to the matching
|
|
8
|
+
closing triple-quote.
|
|
9
|
+
- History: up/down scroll prior inputs, Ctrl+R reverse-searches.
|
|
10
|
+
Persisted to ~/.cognos/history so it spans sessions.
|
|
11
|
+
- Slash completion: typing '/' pops up a completer over the
|
|
12
|
+
slash-command REGISTRY.
|
|
13
|
+
|
|
14
|
+
The fallback path is plain input() so the REPL still works on a
|
|
15
|
+
machine without prompt_toolkit installed.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import logging
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from rich.console import Console
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _history_path() -> Path:
|
|
29
|
+
p = Path.home() / ".cognos" / "history"
|
|
30
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
31
|
+
return p
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class CognosPrompt:
|
|
35
|
+
"""Adaptive prompt — uses prompt_toolkit if installed, else plain input."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, console: Console):
|
|
38
|
+
self.console = console
|
|
39
|
+
self._pt_session = None
|
|
40
|
+
try:
|
|
41
|
+
from prompt_toolkit import PromptSession
|
|
42
|
+
from prompt_toolkit.history import FileHistory
|
|
43
|
+
from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
|
|
44
|
+
from prompt_toolkit.completion import WordCompleter
|
|
45
|
+
from prompt_toolkit.key_binding import KeyBindings
|
|
46
|
+
|
|
47
|
+
from core.slash_commands import REGISTRY
|
|
48
|
+
|
|
49
|
+
completer = WordCompleter(
|
|
50
|
+
["/" + name for name in sorted(REGISTRY.keys())],
|
|
51
|
+
ignore_case=True,
|
|
52
|
+
sentence=True,
|
|
53
|
+
)
|
|
54
|
+
kb = KeyBindings()
|
|
55
|
+
|
|
56
|
+
@kb.add("c-j") # Ctrl+J
|
|
57
|
+
def _(event):
|
|
58
|
+
event.current_buffer.insert_text("\n")
|
|
59
|
+
|
|
60
|
+
self._pt_session = PromptSession(
|
|
61
|
+
history=FileHistory(str(_history_path())),
|
|
62
|
+
auto_suggest=AutoSuggestFromHistory(),
|
|
63
|
+
completer=completer,
|
|
64
|
+
multiline=False,
|
|
65
|
+
key_bindings=kb,
|
|
66
|
+
complete_while_typing=True,
|
|
67
|
+
)
|
|
68
|
+
except Exception as e:
|
|
69
|
+
logger.debug(f"prompt_toolkit unavailable, using plain input: {e}")
|
|
70
|
+
|
|
71
|
+
def ask(self, label: str = "You> ") -> str:
|
|
72
|
+
if self._pt_session is not None:
|
|
73
|
+
try:
|
|
74
|
+
return self._pt_session.prompt(label)
|
|
75
|
+
except (EOFError, KeyboardInterrupt):
|
|
76
|
+
raise
|
|
77
|
+
except Exception as e:
|
|
78
|
+
logger.debug(f"prompt_toolkit failed mid-call: {e}")
|
|
79
|
+
# Plain fallback — supports a "EOF" sentinel for multi-line via
|
|
80
|
+
# triple-quote start: opening `"""` reads until closing `"""`.
|
|
81
|
+
first = self.console.input(label)
|
|
82
|
+
if first.startswith('"""'):
|
|
83
|
+
buf = [first[3:]]
|
|
84
|
+
while True:
|
|
85
|
+
line = self.console.input("... ")
|
|
86
|
+
if line.endswith('"""'):
|
|
87
|
+
buf.append(line[:-3])
|
|
88
|
+
break
|
|
89
|
+
buf.append(line)
|
|
90
|
+
return "\n".join(buf)
|
|
91
|
+
return first
|