lollms-client 1.1.2__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of lollms-client might be problematic. Click here for more details.
- lollms_client/__init__.py +1 -1
- lollms_client/assets/models_ctx_sizes.json +382 -0
- lollms_client/llm_bindings/lollms/__init__.py +2 -2
- lollms_client/llm_bindings/ollama/__init__.py +56 -0
- lollms_client/llm_bindings/openai/__init__.py +3 -3
- lollms_client/lollms_core.py +285 -131
- lollms_client/lollms_discussion.py +419 -147
- lollms_client/lollms_tti_binding.py +32 -82
- lollms_client/tti_bindings/diffusers/__init__.py +460 -297
- lollms_client/tti_bindings/openai/__init__.py +124 -0
- {lollms_client-1.1.2.dist-info → lollms_client-1.3.0.dist-info}/METADATA +1 -1
- {lollms_client-1.1.2.dist-info → lollms_client-1.3.0.dist-info}/RECORD +15 -14
- lollms_client/tti_bindings/dalle/__init__.py +0 -454
- {lollms_client-1.1.2.dist-info → lollms_client-1.3.0.dist-info}/WHEEL +0 -0
- {lollms_client-1.1.2.dist-info → lollms_client-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {lollms_client-1.1.2.dist-info → lollms_client-1.3.0.dist-info}/top_level.txt +0 -0
|
@@ -2,144 +2,137 @@
|
|
|
2
2
|
import os
|
|
3
3
|
import importlib
|
|
4
4
|
from io import BytesIO
|
|
5
|
-
from typing import Optional, List, Dict, Any, Union
|
|
5
|
+
from typing import Optional, List, Dict, Any, Union, Tuple
|
|
6
6
|
from pathlib import Path
|
|
7
|
+
import base64
|
|
7
8
|
import pipmaster as pm
|
|
8
|
-
# --- Concurrency Imports ---
|
|
9
9
|
import threading
|
|
10
10
|
import queue
|
|
11
11
|
from concurrent.futures import Future
|
|
12
12
|
import time
|
|
13
13
|
import hashlib
|
|
14
|
-
import re
|
|
15
|
-
# -------------------------
|
|
16
|
-
# --- Download Imports ---
|
|
17
14
|
import requests
|
|
18
15
|
from tqdm import tqdm
|
|
19
|
-
|
|
16
|
+
import json
|
|
17
|
+
import shutil
|
|
18
|
+
from lollms_client.lollms_tti_binding import LollmsTTIBinding
|
|
19
|
+
from ascii_colors import trace_exception, ASCIIColors
|
|
20
20
|
|
|
21
21
|
pm.ensure_packages(["torch","torchvision"],index_url="https://download.pytorch.org/whl/cu126")
|
|
22
|
-
pm.ensure_packages(["diffusers","pillow","transformers","safetensors",
|
|
22
|
+
pm.ensure_packages(["diffusers","pillow","transformers","safetensors","requests","tqdm"])
|
|
23
23
|
|
|
24
|
-
# Attempt to import core dependencies and set availability flag
|
|
25
24
|
try:
|
|
26
25
|
import torch
|
|
27
|
-
from diffusers import
|
|
26
|
+
from diffusers import (
|
|
27
|
+
AutoPipelineForText2Image,
|
|
28
|
+
AutoPipelineForImage2Image,
|
|
29
|
+
AutoPipelineForInpainting,
|
|
30
|
+
DiffusionPipeline,
|
|
31
|
+
StableDiffusionPipeline
|
|
32
|
+
)
|
|
28
33
|
from diffusers.utils import load_image
|
|
29
34
|
from PIL import Image
|
|
30
35
|
DIFFUSERS_AVAILABLE = True
|
|
31
36
|
except ImportError:
|
|
32
37
|
torch = None
|
|
33
38
|
AutoPipelineForText2Image = None
|
|
39
|
+
AutoPipelineForImage2Image = None
|
|
40
|
+
AutoPipelineForInpainting = None
|
|
34
41
|
DiffusionPipeline = None
|
|
35
42
|
StableDiffusionPipeline = None
|
|
36
43
|
Image = None
|
|
37
44
|
load_image = None
|
|
38
45
|
DIFFUSERS_AVAILABLE = False
|
|
39
46
|
|
|
40
|
-
from lollms_client.lollms_tti_binding import LollmsTTIBinding
|
|
41
|
-
from ascii_colors import trace_exception, ASCIIColors
|
|
42
|
-
import json
|
|
43
|
-
import shutil
|
|
44
|
-
|
|
45
|
-
# Defines the binding name for the manager
|
|
46
47
|
BindingName = "DiffusersTTIBinding_Impl"
|
|
47
48
|
|
|
48
|
-
# --- START: Civitai Model Definitions ---
|
|
49
|
-
# Expanded list of popular Civitai models (as single .safetensors files)
|
|
50
49
|
CIVITAI_MODELS = {
|
|
51
|
-
# --- Photorealistic ---
|
|
52
50
|
"realistic-vision-v6": {
|
|
53
51
|
"display_name": "Realistic Vision V6.0",
|
|
54
52
|
"url": "https://civitai.com/api/download/models/130072",
|
|
55
53
|
"filename": "realisticVisionV60_v60B1.safetensors",
|
|
56
|
-
"description": "
|
|
54
|
+
"description": "Photorealistic SD1.5 checkpoint.",
|
|
57
55
|
"owned_by": "civitai"
|
|
58
56
|
},
|
|
59
57
|
"absolute-reality": {
|
|
60
58
|
"display_name": "Absolute Reality",
|
|
61
59
|
"url": "https://civitai.com/api/download/models/132760",
|
|
62
60
|
"filename": "absolutereality_v181.safetensors",
|
|
63
|
-
"description": "
|
|
61
|
+
"description": "General realistic SD1.5.",
|
|
64
62
|
"owned_by": "civitai"
|
|
65
63
|
},
|
|
66
|
-
# --- General / Artistic ---
|
|
67
64
|
"dreamshaper-8": {
|
|
68
65
|
"display_name": "DreamShaper 8",
|
|
69
66
|
"url": "https://civitai.com/api/download/models/128713",
|
|
70
67
|
"filename": "dreamshaper_8.safetensors",
|
|
71
|
-
"description": "
|
|
68
|
+
"description": "Versatile SD1.5 style model.",
|
|
72
69
|
"owned_by": "civitai"
|
|
73
70
|
},
|
|
74
71
|
"juggernaut-xl": {
|
|
75
72
|
"display_name": "Juggernaut XL",
|
|
76
73
|
"url": "https://civitai.com/api/download/models/133005",
|
|
77
74
|
"filename": "juggernautXL_version6Rundiffusion.safetensors",
|
|
78
|
-
"description": "
|
|
75
|
+
"description": "Artistic SDXL.",
|
|
79
76
|
"owned_by": "civitai"
|
|
80
77
|
},
|
|
81
78
|
"lyriel-v1.6": {
|
|
82
79
|
"display_name": "Lyriel v1.6",
|
|
83
80
|
"url": "https://civitai.com/api/download/models/92407",
|
|
84
81
|
"filename": "lyriel_v16.safetensors",
|
|
85
|
-
"description": "
|
|
82
|
+
"description": "Fantasy/stylized SD1.5.",
|
|
86
83
|
"owned_by": "civitai"
|
|
87
84
|
},
|
|
88
|
-
# --- Anime / Illustration ---
|
|
89
85
|
"anything-v5": {
|
|
90
86
|
"display_name": "Anything V5",
|
|
91
87
|
"url": "https://civitai.com/api/download/models/9409",
|
|
92
88
|
"filename": "anythingV5_PrtRE.safetensors",
|
|
93
|
-
"description": "
|
|
89
|
+
"description": "Anime SD1.5.",
|
|
94
90
|
"owned_by": "civitai"
|
|
95
91
|
},
|
|
96
92
|
"meinamix": {
|
|
97
93
|
"display_name": "MeinaMix",
|
|
98
94
|
"url": "https://civitai.com/api/download/models/119057",
|
|
99
95
|
"filename": "meinamix_meinaV11.safetensors",
|
|
100
|
-
"description": "
|
|
96
|
+
"description": "Anime/illustration SD1.5.",
|
|
101
97
|
"owned_by": "civitai"
|
|
102
98
|
},
|
|
103
|
-
# --- Game Assets & Specialized Styles ---
|
|
104
99
|
"rpg-v5": {
|
|
105
100
|
"display_name": "RPG v5",
|
|
106
101
|
"url": "https://civitai.com/api/download/models/137379",
|
|
107
102
|
"filename": "rpg_v5.safetensors",
|
|
108
|
-
"description": "
|
|
103
|
+
"description": "RPG assets SD1.5.",
|
|
109
104
|
"owned_by": "civitai"
|
|
110
105
|
},
|
|
111
106
|
"pixel-art-xl": {
|
|
112
107
|
"display_name": "Pixel Art XL",
|
|
113
108
|
"url": "https://civitai.com/api/download/models/252919",
|
|
114
109
|
"filename": "pixelartxl_v11.safetensors",
|
|
115
|
-
"description": "
|
|
110
|
+
"description": "Pixel art SDXL.",
|
|
116
111
|
"owned_by": "civitai"
|
|
117
112
|
},
|
|
118
113
|
"lowpoly-world": {
|
|
119
114
|
"display_name": "Lowpoly World",
|
|
120
115
|
"url": "https://civitai.com/api/download/models/90299",
|
|
121
116
|
"filename": "lowpoly_world_v10.safetensors",
|
|
122
|
-
"description": "
|
|
117
|
+
"description": "Lowpoly style SD1.5.",
|
|
123
118
|
"owned_by": "civitai"
|
|
124
119
|
},
|
|
125
120
|
"toonyou": {
|
|
126
121
|
"display_name": "ToonYou",
|
|
127
122
|
"url": "https://civitai.com/api/download/models/152361",
|
|
128
123
|
"filename": "toonyou_beta6.safetensors",
|
|
129
|
-
"description": "
|
|
124
|
+
"description": "Cartoon/Disney SD1.5.",
|
|
130
125
|
"owned_by": "civitai"
|
|
131
126
|
},
|
|
132
127
|
"papercut": {
|
|
133
128
|
"display_name": "Papercut",
|
|
134
129
|
"url": "https://civitai.com/api/download/models/45579",
|
|
135
130
|
"filename": "papercut_v1.safetensors",
|
|
136
|
-
"description": "
|
|
131
|
+
"description": "Paper cutout SD1.5.",
|
|
137
132
|
"owned_by": "civitai"
|
|
138
133
|
}
|
|
139
134
|
}
|
|
140
|
-
# --- END: Civitai Model Definitions ---
|
|
141
135
|
|
|
142
|
-
# Helper for torch.dtype string conversion
|
|
143
136
|
TORCH_DTYPE_MAP_STR_TO_OBJ = {
|
|
144
137
|
"float16": getattr(torch, 'float16', 'float16'),
|
|
145
138
|
"bfloat16": getattr(torch, 'bfloat16', 'bfloat16'),
|
|
@@ -150,43 +143,63 @@ TORCH_DTYPE_MAP_OBJ_TO_STR = {v: k for k, v in TORCH_DTYPE_MAP_STR_TO_OBJ.items(
|
|
|
150
143
|
if torch:
|
|
151
144
|
TORCH_DTYPE_MAP_OBJ_TO_STR[None] = "None"
|
|
152
145
|
|
|
153
|
-
# Common Schedulers mapping
|
|
154
146
|
SCHEDULER_MAPPING = {
|
|
155
|
-
"default": None,
|
|
156
|
-
"
|
|
157
|
-
"
|
|
158
|
-
"
|
|
159
|
-
"
|
|
160
|
-
"
|
|
161
|
-
"
|
|
162
|
-
"
|
|
163
|
-
"
|
|
164
|
-
"
|
|
147
|
+
"default": None,
|
|
148
|
+
"ddim": "DDIMScheduler",
|
|
149
|
+
"ddpm": "DDPMScheduler",
|
|
150
|
+
"deis_multistep": "DEISMultistepScheduler",
|
|
151
|
+
"dpm_multistep": "DPMSolverMultistepScheduler",
|
|
152
|
+
"dpm_multistep_karras": "DPMSolverMultistepScheduler",
|
|
153
|
+
"dpm_single": "DPMSolverSinglestepScheduler",
|
|
154
|
+
"dpm_adaptive": "DPMSolverPlusPlusScheduler",
|
|
155
|
+
"dpm++_2m": "DPMSolverMultistepScheduler",
|
|
156
|
+
"dpm++_2m_karras": "DPMSolverMultistepScheduler",
|
|
157
|
+
"dpm++_2s_ancestral": "DPMSolverAncestralDiscreteScheduler",
|
|
158
|
+
"dpm++_2s_ancestral_karras": "DPMSolverAncestralDiscreteScheduler",
|
|
159
|
+
"dpm++_sde": "DPMSolverSDEScheduler",
|
|
160
|
+
"dpm++_sde_karras": "DPMSolverSDEScheduler",
|
|
161
|
+
"euler_ancestral_discrete": "EulerAncestralDiscreteScheduler",
|
|
162
|
+
"euler_discrete": "EulerDiscreteScheduler",
|
|
163
|
+
"heun_discrete": "HeunDiscreteScheduler",
|
|
164
|
+
"heun_karras": "HeunDiscreteScheduler",
|
|
165
|
+
"lms_discrete": "LMSDiscreteScheduler",
|
|
166
|
+
"lms_karras": "LMSDiscreteScheduler",
|
|
167
|
+
"pndm": "PNDMScheduler",
|
|
168
|
+
"unipc_multistep": "UniPCMultistepScheduler",
|
|
169
|
+
"dpm++_2m_sde": "DPMSolverMultistepScheduler",
|
|
170
|
+
"dpm++_2m_sde_karras": "DPMSolverMultistepScheduler",
|
|
171
|
+
"dpm2": "KDPM2DiscreteScheduler",
|
|
172
|
+
"dpm2_karras": "KDPM2DiscreteScheduler",
|
|
173
|
+
"dpm2_a": "KDPM2AncestralDiscreteScheduler",
|
|
174
|
+
"dpm2_a_karras": "KDPM2AncestralDiscreteScheduler",
|
|
175
|
+
"euler": "EulerDiscreteScheduler",
|
|
176
|
+
"euler_a": "EulerAncestralDiscreteScheduler",
|
|
177
|
+
"heun": "HeunDiscreteScheduler",
|
|
178
|
+
"lms": "LMSDiscreteScheduler"
|
|
165
179
|
}
|
|
166
180
|
SCHEDULER_USES_KARRAS_SIGMAS = [
|
|
167
|
-
"dpm_multistep_karras",
|
|
168
|
-
"dpm++_sde_karras",
|
|
181
|
+
"dpm_multistep_karras","dpm++_2m_karras","dpm++_2s_ancestral_karras",
|
|
182
|
+
"dpm++_sde_karras","heun_karras","lms_karras",
|
|
183
|
+
"dpm++_2m_sde_karras","dpm2_karras","dpm2_a_karras"
|
|
169
184
|
]
|
|
170
185
|
|
|
171
|
-
# --- START: Concurrency and Singleton Management ---
|
|
172
|
-
|
|
173
186
|
class ModelManager:
|
|
174
|
-
"""
|
|
175
|
-
Manages a single pipeline instance, its generation queue, and a worker thread.
|
|
176
|
-
This ensures all interactions with a specific model are thread-safe.
|
|
177
|
-
"""
|
|
178
187
|
def __init__(self, config: Dict[str, Any], models_path: Path):
|
|
179
188
|
self.config = config
|
|
180
189
|
self.models_path = models_path
|
|
181
190
|
self.pipeline: Optional[DiffusionPipeline] = None
|
|
191
|
+
self.current_task: Optional[str] = None
|
|
182
192
|
self.ref_count = 0
|
|
183
193
|
self.lock = threading.Lock()
|
|
184
194
|
self.queue = queue.Queue()
|
|
185
|
-
self.worker_thread = threading.Thread(target=self._generation_worker, daemon=True)
|
|
186
|
-
self._stop_event = threading.Event()
|
|
187
195
|
self.is_loaded = False
|
|
188
|
-
|
|
196
|
+
self.last_used_time = time.time()
|
|
197
|
+
self._stop_event = threading.Event()
|
|
198
|
+
self.worker_thread = threading.Thread(target=self._generation_worker, daemon=True)
|
|
189
199
|
self.worker_thread.start()
|
|
200
|
+
self._stop_monitor_event = threading.Event()
|
|
201
|
+
self._unload_monitor_thread = None
|
|
202
|
+
self._start_unload_monitor()
|
|
190
203
|
|
|
191
204
|
def acquire(self):
|
|
192
205
|
with self.lock:
|
|
@@ -200,87 +213,165 @@ class ModelManager:
|
|
|
200
213
|
|
|
201
214
|
def stop(self):
|
|
202
215
|
self._stop_event.set()
|
|
216
|
+
if self._unload_monitor_thread:
|
|
217
|
+
self._stop_monitor_event.set()
|
|
218
|
+
self._unload_monitor_thread.join(timeout=2)
|
|
203
219
|
self.queue.put(None)
|
|
204
220
|
self.worker_thread.join(timeout=5)
|
|
205
221
|
|
|
206
|
-
def
|
|
207
|
-
|
|
222
|
+
def _start_unload_monitor(self):
|
|
223
|
+
unload_after = self.config.get("unload_inactive_model_after", 0)
|
|
224
|
+
if unload_after > 0 and self._unload_monitor_thread is None:
|
|
225
|
+
self._stop_monitor_event.clear()
|
|
226
|
+
self._unload_monitor_thread = threading.Thread(target=self._unload_monitor, daemon=True)
|
|
227
|
+
self._unload_monitor_thread.start()
|
|
228
|
+
|
|
229
|
+
def _unload_monitor(self):
|
|
230
|
+
unload_after = self.config.get("unload_inactive_model_after", 0)
|
|
231
|
+
if unload_after <= 0:
|
|
232
|
+
return
|
|
233
|
+
ASCIIColors.info(f"Starting inactivity monitor for '{self.config['model_name']}' (timeout: {unload_after}s).")
|
|
234
|
+
while not self._stop_monitor_event.wait(timeout=5.0):
|
|
235
|
+
with self.lock:
|
|
236
|
+
if not self.is_loaded:
|
|
237
|
+
continue
|
|
238
|
+
if time.time() - self.last_used_time > unload_after:
|
|
239
|
+
ASCIIColors.info(f"Model '{self.config['model_name']}' has been inactive. Unloading.")
|
|
240
|
+
self._unload_pipeline()
|
|
241
|
+
|
|
242
|
+
def _resolve_model_path(self, model_name: str) -> Union[str, Path]:
|
|
243
|
+
path_obj = Path(model_name)
|
|
244
|
+
if path_obj.is_absolute() and path_obj.exists():
|
|
245
|
+
return model_name
|
|
246
|
+
if model_name in CIVITAI_MODELS:
|
|
247
|
+
filename = CIVITAI_MODELS[model_name]["filename"]
|
|
248
|
+
local_path = self.models_path / filename
|
|
249
|
+
if not local_path.exists():
|
|
250
|
+
self._download_civitai_model(model_name)
|
|
251
|
+
return local_path
|
|
252
|
+
local_path = self.models_path / model_name
|
|
253
|
+
if local_path.exists():
|
|
254
|
+
return local_path
|
|
255
|
+
return model_name
|
|
256
|
+
|
|
257
|
+
def _download_civitai_model(self, model_key: str):
|
|
258
|
+
model_info = CIVITAI_MODELS[model_key]
|
|
259
|
+
url = model_info["url"]
|
|
260
|
+
filename = model_info["filename"]
|
|
261
|
+
dest_path = self.models_path / filename
|
|
262
|
+
temp_path = dest_path.with_suffix(".temp")
|
|
263
|
+
ASCIIColors.cyan(f"Downloading '{filename}' from Civitai...")
|
|
264
|
+
try:
|
|
265
|
+
with requests.get(url, stream=True) as r:
|
|
266
|
+
r.raise_for_status()
|
|
267
|
+
total_size = int(r.headers.get('content-length', 0))
|
|
268
|
+
with open(temp_path, 'wb') as f, tqdm(total=total_size, unit='iB', unit_scale=True, desc=f"Downloading {filename}") as bar:
|
|
269
|
+
for chunk in r.iter_content(chunk_size=8192):
|
|
270
|
+
f.write(chunk)
|
|
271
|
+
bar.update(len(chunk))
|
|
272
|
+
shutil.move(temp_path, dest_path)
|
|
273
|
+
ASCIIColors.green(f"Model '{filename}' downloaded successfully.")
|
|
274
|
+
except Exception as e:
|
|
275
|
+
if temp_path.exists():
|
|
276
|
+
temp_path.unlink()
|
|
277
|
+
raise Exception(f"Failed to download model {filename}: {e}") from e
|
|
278
|
+
|
|
279
|
+
def _set_scheduler(self):
|
|
280
|
+
if not self.pipeline:
|
|
208
281
|
return
|
|
282
|
+
scheduler_name_key = self.config["scheduler_name"].lower()
|
|
283
|
+
if scheduler_name_key == "default":
|
|
284
|
+
return
|
|
285
|
+
scheduler_class_name = SCHEDULER_MAPPING.get(scheduler_name_key)
|
|
286
|
+
if scheduler_class_name:
|
|
287
|
+
try:
|
|
288
|
+
SchedulerClass = getattr(importlib.import_module("diffusers.schedulers"), scheduler_class_name)
|
|
289
|
+
scheduler_config = self.pipeline.scheduler.config
|
|
290
|
+
scheduler_config["use_karras_sigmas"] = scheduler_name_key in SCHEDULER_USES_KARRAS_SIGMAS
|
|
291
|
+
self.pipeline.scheduler = SchedulerClass.from_config(scheduler_config)
|
|
292
|
+
ASCIIColors.info(f"Switched scheduler to {scheduler_class_name}")
|
|
293
|
+
except Exception as e:
|
|
294
|
+
ASCIIColors.warning(f"Could not switch scheduler to {scheduler_name_key}: {e}. Using current default.")
|
|
209
295
|
|
|
296
|
+
def _load_pipeline_for_task(self, task: str):
|
|
297
|
+
if self.pipeline and self.current_task == task:
|
|
298
|
+
return
|
|
299
|
+
if self.pipeline:
|
|
300
|
+
self._unload_pipeline()
|
|
210
301
|
model_name = self.config.get("model_name", "")
|
|
211
302
|
if not model_name:
|
|
212
303
|
raise ValueError("Model name cannot be empty for loading.")
|
|
213
|
-
|
|
214
|
-
ASCIIColors.info(f"Loading Diffusers model: {model_name}")
|
|
304
|
+
ASCIIColors.info(f"Loading Diffusers model: {model_name} for task: {task}")
|
|
215
305
|
model_path = self._resolve_model_path(model_name)
|
|
216
306
|
torch_dtype = TORCH_DTYPE_MAP_STR_TO_OBJ.get(self.config["torch_dtype_str"].lower())
|
|
217
|
-
|
|
218
307
|
try:
|
|
308
|
+
load_args = {}
|
|
309
|
+
if self.config.get("hf_cache_path"):
|
|
310
|
+
load_args["cache_dir"] = str(self.config["hf_cache_path"])
|
|
219
311
|
if str(model_path).endswith(".safetensors"):
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
model_path,
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
# Fallback for older diffusers versions
|
|
230
|
-
ASCIIColors.warning("AutoPipelineForText2Image.from_single_file not found. Falling back to StableDiffusionPipeline.")
|
|
231
|
-
ASCIIColors.warning("Consider updating diffusers for better compatibility: pip install --upgrade diffusers")
|
|
232
|
-
self.pipeline = StableDiffusionPipeline.from_single_file(
|
|
233
|
-
model_path,
|
|
234
|
-
torch_dtype=torch_dtype,
|
|
235
|
-
cache_dir=self.config.get("hf_cache_path")
|
|
236
|
-
)
|
|
312
|
+
if task == "text2image":
|
|
313
|
+
try:
|
|
314
|
+
self.pipeline = AutoPipelineForText2Image.from_single_file(model_path, torch_dtype=torch_dtype, cache_dir=load_args.get("cache_dir"))
|
|
315
|
+
except AttributeError:
|
|
316
|
+
self.pipeline = StableDiffusionPipeline.from_single_file(model_path, torch_dtype=torch_dtype, cache_dir=load_args.get("cache_dir"))
|
|
317
|
+
elif task == "image2image":
|
|
318
|
+
self.pipeline = AutoPipelineForImage2Image.from_single_file(model_path, torch_dtype=torch_dtype, cache_dir=load_args.get("cache_dir"))
|
|
319
|
+
elif task == "inpainting":
|
|
320
|
+
self.pipeline = AutoPipelineForInpainting.from_single_file(model_path, torch_dtype=torch_dtype, cache_dir=load_args.get("cache_dir"))
|
|
237
321
|
else:
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
"
|
|
241
|
-
"token": self.config["hf_token"],
|
|
322
|
+
common_args = {
|
|
323
|
+
"torch_dtype": torch_dtype,
|
|
324
|
+
"use_safetensors": self.config["use_safetensors"],
|
|
325
|
+
"token": self.config["hf_token"],
|
|
326
|
+
"local_files_only": self.config["local_files_only"]
|
|
242
327
|
}
|
|
243
|
-
if self.config["hf_variant"]:
|
|
244
|
-
|
|
245
|
-
if
|
|
246
|
-
|
|
247
|
-
|
|
328
|
+
if self.config["hf_variant"]:
|
|
329
|
+
common_args["variant"] = self.config["hf_variant"]
|
|
330
|
+
if not self.config["safety_checker_on"]:
|
|
331
|
+
common_args["safety_checker"] = None
|
|
332
|
+
if self.config.get("hf_cache_path"):
|
|
333
|
+
common_args["cache_dir"] = str(self.config["hf_cache_path"])
|
|
334
|
+
if task == "text2image":
|
|
335
|
+
self.pipeline = AutoPipelineForText2Image.from_pretrained(model_path, **common_args)
|
|
336
|
+
elif task == "image2image":
|
|
337
|
+
self.pipeline = AutoPipelineForImage2Image.from_pretrained(model_path, **common_args)
|
|
338
|
+
elif task == "inpainting":
|
|
339
|
+
self.pipeline = AutoPipelineForInpainting.from_pretrained(model_path, **common_args)
|
|
248
340
|
except Exception as e:
|
|
249
341
|
error_str = str(e).lower()
|
|
250
342
|
if "401" in error_str or "gated" in error_str or "authorization" in error_str:
|
|
251
|
-
|
|
252
|
-
f"AUTHENTICATION FAILED for model '{model_name}'.
|
|
253
|
-
"Please ensure you
|
|
343
|
+
msg = (
|
|
344
|
+
f"AUTHENTICATION FAILED for model '{model_name}'. "
|
|
345
|
+
"Please ensure you accepted the model license and provided a valid HF token."
|
|
254
346
|
)
|
|
255
|
-
raise RuntimeError(
|
|
256
|
-
|
|
257
|
-
raise e
|
|
258
|
-
|
|
347
|
+
raise RuntimeError(msg) from e
|
|
348
|
+
raise e
|
|
259
349
|
self._set_scheduler()
|
|
260
350
|
self.pipeline.to(self.config["device"])
|
|
261
|
-
|
|
262
351
|
if self.config["enable_xformers"]:
|
|
263
352
|
try:
|
|
264
353
|
self.pipeline.enable_xformers_memory_efficient_attention()
|
|
265
354
|
except Exception as e:
|
|
266
355
|
ASCIIColors.warning(f"Could not enable xFormers: {e}.")
|
|
267
|
-
|
|
268
356
|
if self.config["enable_cpu_offload"] and self.config["device"] != "cpu":
|
|
269
357
|
self.pipeline.enable_model_cpu_offload()
|
|
270
358
|
elif self.config["enable_sequential_cpu_offload"] and self.config["device"] != "cpu":
|
|
271
359
|
self.pipeline.enable_sequential_cpu_offload()
|
|
272
|
-
|
|
273
360
|
self.is_loaded = True
|
|
274
|
-
|
|
361
|
+
self.current_task = task
|
|
362
|
+
self.last_used_time = time.time()
|
|
363
|
+
ASCIIColors.green(f"Model '{model_name}' loaded successfully on '{self.config['device']}' for task '{task}'.")
|
|
275
364
|
|
|
276
365
|
def _unload_pipeline(self):
|
|
277
366
|
if self.pipeline:
|
|
367
|
+
model_name = self.config.get('model_name', 'Unknown')
|
|
278
368
|
del self.pipeline
|
|
279
369
|
self.pipeline = None
|
|
280
370
|
if torch and torch.cuda.is_available():
|
|
281
371
|
torch.cuda.empty_cache()
|
|
282
372
|
self.is_loaded = False
|
|
283
|
-
|
|
373
|
+
self.current_task = None
|
|
374
|
+
ASCIIColors.info(f"Model '{model_name}' unloaded and VRAM cleared.")
|
|
284
375
|
|
|
285
376
|
def _generation_worker(self):
|
|
286
377
|
while not self._stop_event.is_set():
|
|
@@ -288,17 +379,18 @@ class ModelManager:
|
|
|
288
379
|
job = self.queue.get(timeout=1)
|
|
289
380
|
if job is None:
|
|
290
381
|
break
|
|
291
|
-
future, pipeline_args = job
|
|
382
|
+
future, task, pipeline_args = job
|
|
292
383
|
try:
|
|
293
384
|
with self.lock:
|
|
294
|
-
|
|
295
|
-
|
|
385
|
+
self.last_used_time = time.time()
|
|
386
|
+
if not self.is_loaded or self.current_task != task:
|
|
387
|
+
self._load_pipeline_for_task(task)
|
|
296
388
|
with torch.no_grad():
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
future.set_result(
|
|
389
|
+
output = self.pipeline(**pipeline_args)
|
|
390
|
+
pil = output.images[0]
|
|
391
|
+
buf = BytesIO()
|
|
392
|
+
pil.save(buf, format="PNG")
|
|
393
|
+
future.set_result(buf.getvalue())
|
|
302
394
|
except Exception as e:
|
|
303
395
|
trace_exception(e)
|
|
304
396
|
future.set_exception(e)
|
|
@@ -307,69 +399,9 @@ class ModelManager:
|
|
|
307
399
|
except queue.Empty:
|
|
308
400
|
continue
|
|
309
401
|
|
|
310
|
-
def _download_civitai_model(self, model_key: str):
|
|
311
|
-
model_info = CIVITAI_MODELS[model_key]
|
|
312
|
-
url = model_info["url"]
|
|
313
|
-
filename = model_info["filename"]
|
|
314
|
-
dest_path = self.models_path / filename
|
|
315
|
-
temp_path = dest_path.with_suffix(".temp")
|
|
316
|
-
|
|
317
|
-
ASCIIColors.cyan(f"Downloading '{filename}' from Civitai...")
|
|
318
|
-
try:
|
|
319
|
-
with requests.get(url, stream=True) as r:
|
|
320
|
-
r.raise_for_status()
|
|
321
|
-
total_size = int(r.headers.get('content-length', 0))
|
|
322
|
-
with open(temp_path, 'wb') as f, tqdm(
|
|
323
|
-
total=total_size, unit='iB', unit_scale=True, desc=f"Downloading {filename}"
|
|
324
|
-
) as bar:
|
|
325
|
-
for chunk in r.iter_content(chunk_size=8192):
|
|
326
|
-
f.write(chunk)
|
|
327
|
-
bar.update(len(chunk))
|
|
328
|
-
|
|
329
|
-
shutil.move(temp_path, dest_path)
|
|
330
|
-
ASCIIColors.green(f"Model '{filename}' downloaded successfully.")
|
|
331
|
-
except Exception as e:
|
|
332
|
-
if temp_path.exists():
|
|
333
|
-
temp_path.unlink()
|
|
334
|
-
raise Exception(f"Failed to download model {filename}: {e}") from e
|
|
335
|
-
|
|
336
|
-
def _resolve_model_path(self, model_name: str) -> Union[str, Path]:
|
|
337
|
-
path_obj = Path(model_name)
|
|
338
|
-
if path_obj.is_absolute() and path_obj.exists():
|
|
339
|
-
return model_name
|
|
340
|
-
|
|
341
|
-
if model_name in CIVITAI_MODELS:
|
|
342
|
-
filename = CIVITAI_MODELS[model_name]["filename"]
|
|
343
|
-
local_path = self.models_path / filename
|
|
344
|
-
if not local_path.exists():
|
|
345
|
-
self._download_civitai_model(model_name)
|
|
346
|
-
return local_path
|
|
347
|
-
|
|
348
|
-
local_path = self.models_path / model_name
|
|
349
|
-
if local_path.exists():
|
|
350
|
-
return local_path
|
|
351
|
-
|
|
352
|
-
return model_name
|
|
353
|
-
|
|
354
|
-
def _set_scheduler(self):
|
|
355
|
-
if not self.pipeline: return
|
|
356
|
-
scheduler_name_key = self.config["scheduler_name"].lower()
|
|
357
|
-
if scheduler_name_key == "default": return
|
|
358
|
-
|
|
359
|
-
scheduler_class_name = SCHEDULER_MAPPING.get(scheduler_name_key)
|
|
360
|
-
if scheduler_class_name:
|
|
361
|
-
try:
|
|
362
|
-
SchedulerClass = getattr(importlib.import_module("diffusers.schedulers"), scheduler_class_name)
|
|
363
|
-
scheduler_config = self.pipeline.scheduler.config
|
|
364
|
-
scheduler_config["use_karras_sigmas"] = scheduler_name_key in SCHEDULER_USES_KARRAS_SIGMAS
|
|
365
|
-
self.pipeline.scheduler = SchedulerClass.from_config(scheduler_config)
|
|
366
|
-
except Exception as e:
|
|
367
|
-
ASCIIColors.warning(f"Could not switch scheduler to {scheduler_name_key}: {e}. Using current default.")
|
|
368
|
-
|
|
369
402
|
class PipelineRegistry:
|
|
370
403
|
_instance = None
|
|
371
404
|
_lock = threading.Lock()
|
|
372
|
-
|
|
373
405
|
def __new__(cls, *args, **kwargs):
|
|
374
406
|
with cls._lock:
|
|
375
407
|
if cls._instance is None:
|
|
@@ -377,24 +409,23 @@ class PipelineRegistry:
|
|
|
377
409
|
cls._instance._managers = {}
|
|
378
410
|
cls._instance._registry_lock = threading.Lock()
|
|
379
411
|
return cls._instance
|
|
380
|
-
|
|
381
|
-
def
|
|
382
|
-
|
|
383
|
-
"model_name",
|
|
384
|
-
"safety_checker_on",
|
|
385
|
-
"enable_sequential_cpu_offload",
|
|
386
|
-
"local_files_only",
|
|
412
|
+
@staticmethod
|
|
413
|
+
def _get_critical_keys():
|
|
414
|
+
return [
|
|
415
|
+
"model_name","device","torch_dtype_str","use_safetensors",
|
|
416
|
+
"safety_checker_on","hf_variant","enable_cpu_offload",
|
|
417
|
+
"enable_sequential_cpu_offload","enable_xformers",
|
|
418
|
+
"local_files_only","hf_cache_path","unload_inactive_model_after"
|
|
387
419
|
]
|
|
388
|
-
|
|
420
|
+
def _get_config_key(self, config: Dict[str, Any]) -> str:
|
|
421
|
+
key_data = tuple(sorted((k, config.get(k)) for k in self._get_critical_keys()))
|
|
389
422
|
return hashlib.sha256(str(key_data).encode('utf-8')).hexdigest()
|
|
390
|
-
|
|
391
423
|
def get_manager(self, config: Dict[str, Any], models_path: Path) -> ModelManager:
|
|
392
424
|
key = self._get_config_key(config)
|
|
393
425
|
with self._registry_lock:
|
|
394
426
|
if key not in self._managers:
|
|
395
427
|
self._managers[key] = ModelManager(config.copy(), models_path)
|
|
396
428
|
return self._managers[key].acquire()
|
|
397
|
-
|
|
398
429
|
def release_manager(self, config: Dict[str, Any]):
|
|
399
430
|
key = self._get_config_key(config)
|
|
400
431
|
with self._registry_lock:
|
|
@@ -402,41 +433,96 @@ class PipelineRegistry:
|
|
|
402
433
|
manager = self._managers[key]
|
|
403
434
|
ref_count = manager.release()
|
|
404
435
|
if ref_count == 0:
|
|
405
|
-
ASCIIColors.info(f"Reference count for model '{config.get('model_name')}' is zero. Cleaning up.")
|
|
436
|
+
ASCIIColors.info(f"Reference count for model '{config.get('model_name')}' is zero. Cleaning up manager.")
|
|
406
437
|
manager.stop()
|
|
407
|
-
manager.
|
|
438
|
+
with manager.lock:
|
|
439
|
+
manager._unload_pipeline()
|
|
408
440
|
del self._managers[key]
|
|
441
|
+
def get_active_managers(self) -> List[ModelManager]:
|
|
442
|
+
with self._registry_lock:
|
|
443
|
+
return [m for m in self._managers.values() if m.is_loaded]
|
|
409
444
|
|
|
410
445
|
class DiffusersTTIBinding_Impl(LollmsTTIBinding):
|
|
411
446
|
DEFAULT_CONFIG = {
|
|
412
|
-
"model_name": "",
|
|
413
|
-
"
|
|
414
|
-
"
|
|
415
|
-
"
|
|
416
|
-
"
|
|
447
|
+
"model_name": "",
|
|
448
|
+
"device": "auto",
|
|
449
|
+
"torch_dtype_str": "auto",
|
|
450
|
+
"use_safetensors": True,
|
|
451
|
+
"scheduler_name": "default",
|
|
452
|
+
"safety_checker_on": True,
|
|
453
|
+
"num_inference_steps": 25,
|
|
454
|
+
"guidance_scale": 7.0,
|
|
455
|
+
"default_width": 512,
|
|
456
|
+
"default_height": 512,
|
|
457
|
+
"seed": -1,
|
|
458
|
+
"enable_cpu_offload": False,
|
|
459
|
+
"enable_sequential_cpu_offload": False,
|
|
460
|
+
"enable_xformers": False,
|
|
461
|
+
"hf_variant": None,
|
|
462
|
+
"hf_token": None,
|
|
463
|
+
"hf_cache_path": None,
|
|
464
|
+
"local_files_only": False,
|
|
465
|
+
"unload_inactive_model_after": 0
|
|
417
466
|
}
|
|
467
|
+
HF_DEFAULT_MODELS = [
|
|
468
|
+
{"family": "SDXL", "model_name": "stabilityai/stable-diffusion-xl-base-1.0", "display_name": "SDXL Base 1.0", "desc": "Text2Image 1024 native."},
|
|
469
|
+
{"family": "SDXL", "model_name": "stabilityai/stable-diffusion-xl-refiner-1.0", "display_name": "SDXL Refiner 1.0", "desc": "Refiner for SDXL."},
|
|
470
|
+
{"family": "SD 1.x", "model_name": "runwayml/stable-diffusion-v1-5", "display_name": "Stable Diffusion 1.5", "desc": "Classic SD1.5."},
|
|
471
|
+
{"family": "SD 2.x", "model_name": "stabilityai/stable-diffusion-2-1", "display_name": "Stable Diffusion 2.1", "desc": "SD2.1 base."},
|
|
472
|
+
{"family": "SD3", "model_name": "stabilityai/stable-diffusion-3-medium-diffusers", "display_name": "Stable Diffusion 3 Medium", "desc": "SD3 medium."},
|
|
473
|
+
{"family": "Specialized", "model_name": "playgroundai/playground-v2.5-1024px-aesthetic", "display_name": "Playground v2.5", "desc": "High aesthetic 1024."},
|
|
474
|
+
{"family": "Editors", "model_name": "Qwen/Qwen-Image-Edit", "display_name": "Qwen Image Edit", "desc": "Dedicated image editing."}
|
|
475
|
+
]
|
|
418
476
|
|
|
419
477
|
def __init__(self, **kwargs):
|
|
420
478
|
super().__init__(binding_name=BindingName)
|
|
421
|
-
|
|
422
479
|
if not DIFFUSERS_AVAILABLE:
|
|
423
|
-
raise ImportError(
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
)
|
|
427
|
-
|
|
428
|
-
self.config = {**self.DEFAULT_CONFIG, **kwargs}
|
|
480
|
+
raise ImportError("Diffusers not available. Please install required packages.")
|
|
481
|
+
self.config = self.DEFAULT_CONFIG.copy()
|
|
482
|
+
self.config.update(kwargs)
|
|
429
483
|
self.model_name = self.config.get("model_name", "")
|
|
430
|
-
|
|
484
|
+
models_path_str = kwargs.get("models_path", str(Path(__file__).parent / "models"))
|
|
485
|
+
self.models_path = Path(models_path_str)
|
|
431
486
|
self.models_path.mkdir(parents=True, exist_ok=True)
|
|
432
|
-
|
|
433
487
|
self.registry = PipelineRegistry()
|
|
434
488
|
self.manager: Optional[ModelManager] = None
|
|
435
|
-
|
|
436
489
|
self._resolve_device_and_dtype()
|
|
437
490
|
if self.model_name:
|
|
438
491
|
self._acquire_manager()
|
|
439
492
|
|
|
493
|
+
def ps(self) -> List[dict]:
|
|
494
|
+
if not self.registry:
|
|
495
|
+
return []
|
|
496
|
+
try:
|
|
497
|
+
active = self.registry.get_active_managers()
|
|
498
|
+
out = []
|
|
499
|
+
for m in active:
|
|
500
|
+
with m.lock:
|
|
501
|
+
cfg = m.config
|
|
502
|
+
pipe = m.pipeline
|
|
503
|
+
vram_usage_bytes = 0
|
|
504
|
+
if torch.cuda.is_available() and cfg.get("device") == "cuda" and pipe:
|
|
505
|
+
for comp in pipe.components.values():
|
|
506
|
+
if hasattr(comp, 'parameters'):
|
|
507
|
+
mem_params = sum(p.nelement() * p.element_size() for p in comp.parameters())
|
|
508
|
+
mem_bufs = sum(b.nelement() * b.element_size() for b in comp.buffers())
|
|
509
|
+
vram_usage_bytes += (mem_params + mem_bufs)
|
|
510
|
+
out.append({
|
|
511
|
+
"model_name": cfg.get("model_name"),
|
|
512
|
+
"vram_size": vram_usage_bytes,
|
|
513
|
+
"device": cfg.get("device"),
|
|
514
|
+
"torch_dtype": str(pipe.dtype) if pipe else cfg.get("torch_dtype_str"),
|
|
515
|
+
"pipeline_type": pipe.__class__.__name__ if pipe else "N/A",
|
|
516
|
+
"scheduler_class": pipe.scheduler.__class__.__name__ if pipe and hasattr(pipe, 'scheduler') else "N/A",
|
|
517
|
+
"status": "Active" if m.is_loaded else "Idle",
|
|
518
|
+
"queue_size": m.queue.qsize(),
|
|
519
|
+
"task": m.current_task or "N/A"
|
|
520
|
+
})
|
|
521
|
+
return out
|
|
522
|
+
except Exception as e:
|
|
523
|
+
ASCIIColors.error(f"Failed to list running models: {e}")
|
|
524
|
+
return []
|
|
525
|
+
|
|
440
526
|
def _acquire_manager(self):
|
|
441
527
|
if self.manager:
|
|
442
528
|
self.registry.release_manager(self.manager.config)
|
|
@@ -446,51 +532,57 @@ class DiffusersTTIBinding_Impl(LollmsTTIBinding):
|
|
|
446
532
|
def _resolve_device_and_dtype(self):
|
|
447
533
|
if self.config["device"].lower() == "auto":
|
|
448
534
|
self.config["device"] = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
|
|
449
|
-
|
|
450
535
|
if self.config["torch_dtype_str"].lower() == "auto":
|
|
451
536
|
self.config["torch_dtype_str"] = "float16" if self.config["device"] != "cpu" else "float32"
|
|
452
537
|
|
|
538
|
+
def _decode_image_input(self, item: str) -> Image.Image:
|
|
539
|
+
s = item.strip()
|
|
540
|
+
if s.startswith("data:image/") and ";base64," in s:
|
|
541
|
+
b64 = s.split(";base64,")[-1]
|
|
542
|
+
raw = base64.b64decode(b64)
|
|
543
|
+
return Image.open(BytesIO(raw)).convert("RGB")
|
|
544
|
+
if re_b64 := (s[:30].replace("\n","")):
|
|
545
|
+
try:
|
|
546
|
+
raw = base64.b64decode(s, validate=True)
|
|
547
|
+
return Image.open(BytesIO(raw)).convert("RGB")
|
|
548
|
+
except Exception:
|
|
549
|
+
pass
|
|
550
|
+
try:
|
|
551
|
+
return load_image(s).convert("RGB")
|
|
552
|
+
except Exception:
|
|
553
|
+
return Image.open(s).convert("RGB")
|
|
554
|
+
|
|
555
|
+
def _prepare_seed(self, kwargs: Dict[str, Any]) -> Optional[torch.Generator]:
|
|
556
|
+
seed = kwargs.pop("seed", self.config["seed"])
|
|
557
|
+
if seed == -1:
|
|
558
|
+
return None
|
|
559
|
+
return torch.Generator(device=self.config["device"]).manual_seed(seed)
|
|
560
|
+
|
|
453
561
|
def list_safetensor_models(self) -> List[str]:
|
|
454
|
-
if not self.models_path.exists():
|
|
562
|
+
if not self.models_path.exists():
|
|
563
|
+
return []
|
|
455
564
|
return sorted([f.name for f in self.models_path.iterdir() if f.is_file() and f.suffix == ".safetensors"])
|
|
456
565
|
|
|
457
566
|
def listModels(self) -> list:
|
|
458
|
-
# Start with hardcoded Civitai and Hugging Face models
|
|
459
567
|
civitai_list = [
|
|
460
568
|
{'model_name': key, 'display_name': info['display_name'], 'description': info['description'], 'owned_by': info['owned_by']}
|
|
461
569
|
for key, info in CIVITAI_MODELS.items()
|
|
462
570
|
]
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
{'model_name': "playgroundai/playground-v2.5-1024px-aesthetic", 'display_name': "Playground v2.5", 'description': "Known for high aesthetic quality. Native resolution is 1024x1024.", 'owned_by': 'HuggingFace'},
|
|
467
|
-
# SD 1.5 Models (512x512 native)
|
|
468
|
-
{'model_name': "runwayml/stable-diffusion-v1-5", 'display_name': "Stable Diffusion 1.5", 'description': "A popular and versatile open-access text-to-image model.", 'owned_by': 'HuggingFace'},
|
|
469
|
-
{'model_name': "dataautogpt3/OpenDalleV1.1", 'display_name': "OpenDalle v1.1", 'description': "An open-source reproduction of DALL-E 3, good for prompt adherence.", 'owned_by': 'HuggingFace'},
|
|
470
|
-
{'model_name': "stabilityai/stable-diffusion-2-1-base", 'display_name': "Stable Diffusion 2.1 (512px)", 'description': "A 512x512 resolution model from Stability AI.", 'owned_by': 'HuggingFace'},
|
|
471
|
-
{'model_name': "CompVis/stable-diffusion-v1-4", 'display_name': "Stable Diffusion 1.4 (Gated)", 'description': "Original SD v1.4. Requires accepting license on Hugging Face and an HF token.", 'owned_by': 'HuggingFace'}
|
|
571
|
+
hf_list = [
|
|
572
|
+
{'model_name': m["model_name"], 'display_name': m["display_name"], 'description': m["desc"], 'owned_by': 'HuggingFace', 'family': m["family"]}
|
|
573
|
+
for m in self.HF_DEFAULT_MODELS
|
|
472
574
|
]
|
|
473
|
-
|
|
474
|
-
# Discover local .safetensors files
|
|
475
|
-
custom_local_models = []
|
|
575
|
+
custom_local = []
|
|
476
576
|
civitai_filenames = {info['filename'] for info in CIVITAI_MODELS.values()}
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
for filename in local_safetensors:
|
|
577
|
+
for filename in self.list_safetensor_models():
|
|
480
578
|
if filename not in civitai_filenames:
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
'display_name': filename,
|
|
484
|
-
'description': 'Local safetensors file from your models folder.',
|
|
485
|
-
'owned_by': 'local_user'
|
|
486
|
-
})
|
|
487
|
-
|
|
488
|
-
return civitai_list + hf_default_list + custom_local_models
|
|
579
|
+
custom_local.append({'model_name': filename, 'display_name': filename, 'description': 'Local safetensors file.', 'owned_by': 'local_user'})
|
|
580
|
+
return hf_list + civitai_list + custom_local
|
|
489
581
|
|
|
490
582
|
def load_model(self):
|
|
491
|
-
ASCIIColors.info("load_model() called. Loading is
|
|
583
|
+
ASCIIColors.info("load_model() called. Loading is automatic on first use.")
|
|
492
584
|
if self.model_name and not self.manager:
|
|
493
|
-
|
|
585
|
+
self._acquire_manager()
|
|
494
586
|
|
|
495
587
|
def unload_model(self):
|
|
496
588
|
if self.manager:
|
|
@@ -498,71 +590,153 @@ class DiffusersTTIBinding_Impl(LollmsTTIBinding):
|
|
|
498
590
|
self.registry.release_manager(self.manager.config)
|
|
499
591
|
self.manager = None
|
|
500
592
|
|
|
501
|
-
def generate_image(self, prompt: str, negative_prompt: str = "", width: int = None, height: int = None, **kwargs) -> bytes:
|
|
593
|
+
def generate_image(self, prompt: str, negative_prompt: str = "", width: int|None = None, height: int|None = None, **kwargs) -> bytes:
|
|
502
594
|
if not self.model_name:
|
|
503
595
|
raise RuntimeError("No model_name configured. Please select a model in settings.")
|
|
504
|
-
|
|
505
596
|
if not self.manager:
|
|
506
597
|
self._acquire_manager()
|
|
507
|
-
|
|
508
|
-
_width = width or self.config["default_width"]
|
|
509
|
-
_height = height or self.config["default_height"]
|
|
510
|
-
_num_inference_steps = kwargs.get("num_inference_steps", self.config["num_inference_steps"])
|
|
511
|
-
_guidance_scale = kwargs.get("guidance_scale", self.config["guidance_scale"])
|
|
512
|
-
_seed = kwargs.get("seed", self.config["seed"])
|
|
513
|
-
|
|
514
|
-
generator = torch.Generator(device=self.config["device"]).manual_seed(_seed) if _seed != -1 else None
|
|
515
|
-
|
|
598
|
+
generator = self._prepare_seed(kwargs)
|
|
516
599
|
pipeline_args = {
|
|
517
|
-
"prompt": prompt,
|
|
518
|
-
"
|
|
519
|
-
"
|
|
600
|
+
"prompt": prompt,
|
|
601
|
+
"negative_prompt": negative_prompt or None,
|
|
602
|
+
"width": width if width is not None else self.config["default_width"],
|
|
603
|
+
"height": height if height is not None else self.config["default_height"],
|
|
604
|
+
"num_inference_steps": kwargs.pop("num_inference_steps", self.config["num_inference_steps"]),
|
|
605
|
+
"guidance_scale": kwargs.pop("guidance_scale", self.config["guidance_scale"]),
|
|
606
|
+
"generator": generator
|
|
520
607
|
}
|
|
521
|
-
|
|
608
|
+
pipeline_args.update(kwargs)
|
|
522
609
|
future = Future()
|
|
523
|
-
self.manager.queue.put((future, pipeline_args))
|
|
524
|
-
ASCIIColors.info(f"Job
|
|
525
|
-
|
|
610
|
+
self.manager.queue.put((future, "text2image", pipeline_args))
|
|
611
|
+
ASCIIColors.info(f"Job (t2i) '{prompt[:50]}...' queued.")
|
|
526
612
|
try:
|
|
527
|
-
|
|
528
|
-
ASCIIColors.green("Image generated successfully.")
|
|
529
|
-
return image_bytes
|
|
613
|
+
return future.result()
|
|
530
614
|
except Exception as e:
|
|
531
615
|
raise Exception(f"Image generation failed: {e}") from e
|
|
532
616
|
|
|
617
|
+
def _encode_image_to_latents(self, pil: Image.Image, width: int, height: int) -> Tuple[torch.Tensor, Tuple[int,int]]:
|
|
618
|
+
pil = pil.convert("RGB").resize((width, height))
|
|
619
|
+
with self.manager.lock:
|
|
620
|
+
self.manager._load_pipeline_for_task("text2image")
|
|
621
|
+
vae = self.manager.pipeline.vae
|
|
622
|
+
img = torch.from_numpy(torch.ByteTensor(bytearray(pil.tobytes())).numpy()).float() # not efficient but avoids np dep
|
|
623
|
+
img = img.view(pil.height, pil.width, 3).permute(2,0,1).unsqueeze(0) / 255.0
|
|
624
|
+
img = (img * 2.0) - 1.0
|
|
625
|
+
img = img.to(self.config["device"], dtype=getattr(torch, self.config["torch_dtype_str"]))
|
|
626
|
+
with torch.no_grad():
|
|
627
|
+
posterior = vae.encode(img)
|
|
628
|
+
latents = posterior.latent_dist.sample()
|
|
629
|
+
sf = getattr(vae.config, "scaling_factor", 0.18215)
|
|
630
|
+
latents = latents * sf
|
|
631
|
+
return latents, (pil.width, pil.height)
|
|
632
|
+
|
|
633
|
+
def edit_image(self,
|
|
634
|
+
images: Union[str, List[str]],
|
|
635
|
+
prompt: str,
|
|
636
|
+
negative_prompt: Optional[str] = "",
|
|
637
|
+
mask: Optional[str] = None,
|
|
638
|
+
width: Optional[int] = None,
|
|
639
|
+
height: Optional[int] = None,
|
|
640
|
+
**kwargs) -> bytes:
|
|
641
|
+
if not self.model_name:
|
|
642
|
+
raise RuntimeError("No model_name configured. Please select a model in settings.")
|
|
643
|
+
if not self.manager:
|
|
644
|
+
self._acquire_manager()
|
|
645
|
+
imgs = [images] if isinstance(images, str) else list(images)
|
|
646
|
+
pil_images = [self._decode_image_input(s) for s in imgs]
|
|
647
|
+
out_w = width if width is not None else self.config["default_width"]
|
|
648
|
+
out_h = height if height is not None else self.config["default_height"]
|
|
649
|
+
generator = self._prepare_seed(kwargs)
|
|
650
|
+
steps = kwargs.pop("num_inference_steps", self.config["num_inference_steps"])
|
|
651
|
+
guidance = kwargs.pop("guidance_scale", self.config["guidance_scale"])
|
|
652
|
+
if mask is not None and len(pil_images) == 1:
|
|
653
|
+
try:
|
|
654
|
+
mask_img = self._decode_image_input(mask).convert("L")
|
|
655
|
+
except Exception as e:
|
|
656
|
+
raise ValueError(f"Failed to decode mask image: {e}") from e
|
|
657
|
+
pipeline_args = {
|
|
658
|
+
"image": pil_images[0],
|
|
659
|
+
"mask_image": mask_img,
|
|
660
|
+
"prompt": prompt,
|
|
661
|
+
"negative_prompt": negative_prompt or None,
|
|
662
|
+
"width": out_w,
|
|
663
|
+
"height": out_h,
|
|
664
|
+
"num_inference_steps": steps,
|
|
665
|
+
"guidance_scale": guidance,
|
|
666
|
+
"generator": generator
|
|
667
|
+
}
|
|
668
|
+
pipeline_args.update(kwargs)
|
|
669
|
+
future = Future()
|
|
670
|
+
self.manager.queue.put((future, "inpainting", pipeline_args))
|
|
671
|
+
ASCIIColors.info("Job (inpaint) queued.")
|
|
672
|
+
return future.result()
|
|
673
|
+
try:
|
|
674
|
+
pipeline_args = {
|
|
675
|
+
"image": pil_images if len(pil_images) > 1 else pil_images[0],
|
|
676
|
+
"prompt": prompt,
|
|
677
|
+
"negative_prompt": negative_prompt or None,
|
|
678
|
+
"strength": kwargs.pop("strength", 0.6),
|
|
679
|
+
"width": out_w,
|
|
680
|
+
"height": out_h,
|
|
681
|
+
"num_inference_steps": steps,
|
|
682
|
+
"guidance_scale": guidance,
|
|
683
|
+
"generator": generator
|
|
684
|
+
}
|
|
685
|
+
pipeline_args.update(kwargs)
|
|
686
|
+
future = Future()
|
|
687
|
+
self.manager.queue.put((future, "image2image", pipeline_args))
|
|
688
|
+
ASCIIColors.info("Job (i2i) queued.")
|
|
689
|
+
return future.result()
|
|
690
|
+
except Exception:
|
|
691
|
+
pass
|
|
692
|
+
try:
|
|
693
|
+
base = pil_images[0]
|
|
694
|
+
latents, _ = self._encode_image_to_latents(base, out_w, out_h)
|
|
695
|
+
pipeline_args = {
|
|
696
|
+
"prompt": prompt,
|
|
697
|
+
"negative_prompt": negative_prompt or None,
|
|
698
|
+
"latents": latents,
|
|
699
|
+
"num_inference_steps": steps,
|
|
700
|
+
"guidance_scale": guidance,
|
|
701
|
+
"generator": generator,
|
|
702
|
+
"width": out_w,
|
|
703
|
+
"height": out_h
|
|
704
|
+
}
|
|
705
|
+
pipeline_args.update(kwargs)
|
|
706
|
+
future = Future()
|
|
707
|
+
self.manager.queue.put((future, "text2image", pipeline_args))
|
|
708
|
+
ASCIIColors.info("Job (t2i with init latents) queued.")
|
|
709
|
+
return future.result()
|
|
710
|
+
except Exception as e:
|
|
711
|
+
raise Exception(f"Image edit failed: {e}") from e
|
|
712
|
+
|
|
533
713
|
def list_local_models(self) -> List[str]:
|
|
534
|
-
if not self.models_path.exists():
|
|
535
|
-
|
|
714
|
+
if not self.models_path.exists():
|
|
715
|
+
return []
|
|
536
716
|
folders = [
|
|
537
717
|
d.name for d in self.models_path.iterdir()
|
|
538
718
|
if d.is_dir() and ((d / "model_index.json").exists() or (d / "unet" / "config.json").exists())
|
|
539
719
|
]
|
|
540
720
|
safetensors = self.list_safetensor_models()
|
|
541
721
|
return sorted(folders + safetensors)
|
|
542
|
-
|
|
722
|
+
|
|
543
723
|
def list_available_models(self) -> List[str]:
|
|
544
|
-
|
|
724
|
+
discoverable = [m['model_name'] for m in self.listModels()]
|
|
545
725
|
local_models = self.list_local_models()
|
|
546
|
-
|
|
547
|
-
combined_list = sorted(list(set(local_models + discoverable_models)))
|
|
548
|
-
return combined_list
|
|
726
|
+
return sorted(list(set(local_models + discoverable)))
|
|
549
727
|
|
|
550
728
|
def list_services(self, **kwargs) -> List[Dict[str, str]]:
|
|
551
729
|
models = self.list_available_models()
|
|
552
730
|
local_models = self.list_local_models()
|
|
553
|
-
|
|
554
731
|
if not models:
|
|
555
732
|
return [{"name": "diffusers_no_models", "caption": "No models found", "help": f"Place models in '{self.models_path.resolve()}'."}]
|
|
556
|
-
|
|
557
733
|
services = []
|
|
558
734
|
for m in models:
|
|
559
735
|
help_text = "Hugging Face model ID"
|
|
560
736
|
if m in local_models:
|
|
561
|
-
|
|
737
|
+
help_text = f"Local model from: {self.models_path.resolve()}"
|
|
562
738
|
elif m in CIVITAI_MODELS:
|
|
563
|
-
|
|
564
|
-
help_text = f"Civitai model (downloads as {filename})"
|
|
565
|
-
|
|
739
|
+
help_text = f"Civitai model (downloads as {CIVITAI_MODELS[m]['filename']})"
|
|
566
740
|
services.append({"name": m, "caption": f"Diffusers: {m}", "help": help_text})
|
|
567
741
|
return services
|
|
568
742
|
|
|
@@ -570,82 +744,71 @@ class DiffusersTTIBinding_Impl(LollmsTTIBinding):
|
|
|
570
744
|
available_models = self.list_available_models()
|
|
571
745
|
return [
|
|
572
746
|
{"name": "model_name", "type": "str", "value": self.model_name, "description": "Local, Civitai, or Hugging Face model.", "options": available_models},
|
|
573
|
-
{"name": "
|
|
574
|
-
{"name": "
|
|
747
|
+
{"name": "unload_inactive_model_after", "type": "int", "value": self.config["unload_inactive_model_after"], "description": "Unload model after X seconds of inactivity (0 to disable)."},
|
|
748
|
+
{"name": "device", "type": "str", "value": self.config["device"], "description": f"Inference device. Resolved: {self.config['device']}", "options": ["auto","cuda","mps","cpu"]},
|
|
749
|
+
{"name": "torch_dtype_str", "type": "str", "value": self.config["torch_dtype_str"], "description": f"Torch dtype. Resolved: {self.config['torch_dtype_str']}", "options": ["auto","float16","bfloat16","float32"]},
|
|
575
750
|
{"name": "hf_variant", "type": "str", "value": self.config["hf_variant"], "description": "HF model variant (e.g., 'fp16')."},
|
|
576
751
|
{"name": "use_safetensors", "type": "bool", "value": self.config["use_safetensors"], "description": "Prefer .safetensors when loading from Hugging Face."},
|
|
577
752
|
{"name": "scheduler_name", "type": "str", "value": self.config["scheduler_name"], "description": "Scheduler for diffusion.", "options": list(SCHEDULER_MAPPING.keys())},
|
|
578
753
|
{"name": "safety_checker_on", "type": "bool", "value": self.config["safety_checker_on"], "description": "Enable the safety checker."},
|
|
579
754
|
{"name": "enable_cpu_offload", "type": "bool", "value": self.config["enable_cpu_offload"], "description": "Enable model CPU offload (saves VRAM, slower)."},
|
|
580
|
-
{"name": "enable_sequential_cpu_offload", "type": "bool", "value": self.config["enable_sequential_cpu_offload"], "description": "Enable sequential CPU offload
|
|
755
|
+
{"name": "enable_sequential_cpu_offload", "type": "bool", "value": self.config["enable_sequential_cpu_offload"], "description": "Enable sequential CPU offload."},
|
|
581
756
|
{"name": "enable_xformers", "type": "bool", "value": self.config["enable_xformers"], "description": "Enable xFormers memory efficient attention."},
|
|
582
|
-
{"name": "default_width", "type": "int", "value": self.config["default_width"], "description": "Default image width.
|
|
583
|
-
{"name": "default_height", "type": "int", "value": self.config["default_height"], "description": "Default image height.
|
|
757
|
+
{"name": "default_width", "type": "int", "value": self.config["default_width"], "description": "Default image width."},
|
|
758
|
+
{"name": "default_height", "type": "int", "value": self.config["default_height"], "description": "Default image height."},
|
|
584
759
|
{"name": "num_inference_steps", "type": "int", "value": self.config["num_inference_steps"], "description": "Default inference steps."},
|
|
585
760
|
{"name": "guidance_scale", "type": "float", "value": self.config["guidance_scale"], "description": "Default guidance scale (CFG)."},
|
|
586
761
|
{"name": "seed", "type": "int", "value": self.config["seed"], "description": "Default seed (-1 for random)."},
|
|
587
762
|
{"name": "hf_token", "type": "str", "value": self.config["hf_token"], "description": "HF API token (for private/gated models).", "is_secret": True},
|
|
588
763
|
{"name": "hf_cache_path", "type": "str", "value": self.config["hf_cache_path"], "description": "Path to HF cache."},
|
|
589
|
-
{"name": "local_files_only", "type": "bool", "value": self.config["local_files_only"], "description": "Do not download from Hugging Face."}
|
|
764
|
+
{"name": "local_files_only", "type": "bool", "value": self.config["local_files_only"], "description": "Do not download from Hugging Face."}
|
|
590
765
|
]
|
|
591
766
|
|
|
592
767
|
def set_settings(self, settings: Union[Dict[str, Any], List[Dict[str, Any]]], **kwargs) -> bool:
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
needs_manager_swap = False
|
|
598
|
-
|
|
599
|
-
for key, value in parsed_settings.items():
|
|
768
|
+
parsed = settings if isinstance(settings, dict) else {i["name"]: i["value"] for i in settings if "name" in i and "value" in i}
|
|
769
|
+
critical_keys = self.registry._get_critical_keys()
|
|
770
|
+
needs_swap = False
|
|
771
|
+
for key, value in parsed.items():
|
|
600
772
|
if self.config.get(key) != value:
|
|
601
773
|
ASCIIColors.info(f"Setting '{key}' changed to: {value}")
|
|
602
774
|
self.config[key] = value
|
|
603
|
-
if key == "model_name":
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
775
|
+
if key == "model_name":
|
|
776
|
+
self.model_name = value
|
|
777
|
+
if key in critical_keys:
|
|
778
|
+
needs_swap = True
|
|
779
|
+
if needs_swap and self.model_name:
|
|
607
780
|
ASCIIColors.info("Critical settings changed. Swapping model manager...")
|
|
608
781
|
self._resolve_device_and_dtype()
|
|
609
782
|
self._acquire_manager()
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
self.manager.
|
|
613
|
-
|
|
614
|
-
with self.manager.lock:
|
|
783
|
+
if not needs_swap and self.manager:
|
|
784
|
+
self.manager.config.update(parsed)
|
|
785
|
+
if 'scheduler_name' in parsed and self.manager.pipeline:
|
|
786
|
+
with self.manager.lock:
|
|
615
787
|
self.manager._set_scheduler()
|
|
616
|
-
|
|
617
788
|
return True
|
|
618
789
|
|
|
619
790
|
def __del__(self):
|
|
620
791
|
self.unload_model()
|
|
621
792
|
|
|
622
|
-
# Example Usage
|
|
623
793
|
if __name__ == '__main__':
|
|
624
794
|
ASCIIColors.magenta("--- Diffusers TTI Binding Test ---")
|
|
625
|
-
|
|
626
795
|
if not DIFFUSERS_AVAILABLE:
|
|
627
796
|
ASCIIColors.error("Diffusers not available. Cannot run test.")
|
|
628
797
|
exit(1)
|
|
629
|
-
|
|
630
|
-
temp_paths_dir = Path(__file__).parent / "temp_lollms_paths_diffusers"
|
|
798
|
+
temp_paths_dir = Path(__file__).parent / "tmp"
|
|
631
799
|
temp_models_path = temp_paths_dir / "models"
|
|
632
|
-
|
|
633
|
-
|
|
800
|
+
if temp_paths_dir.exists():
|
|
801
|
+
shutil.rmtree(temp_paths_dir)
|
|
634
802
|
temp_models_path.mkdir(parents=True, exist_ok=True)
|
|
635
|
-
|
|
636
803
|
try:
|
|
637
|
-
ASCIIColors.cyan("\n--- Test: Loading a
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
binding = DiffusersTTIBinding_Impl(**binding_config)
|
|
641
|
-
|
|
804
|
+
ASCIIColors.cyan("\n--- Test: Loading a small HF model ---")
|
|
805
|
+
cfg = {"models_path": str(temp_models_path), "model_name": "hf-internal-testing/tiny-stable-diffusion-torch"}
|
|
806
|
+
binding = DiffusersTTIBinding_Impl(**cfg)
|
|
642
807
|
img_bytes = binding.generate_image("a tiny robot", width=64, height=64, num_inference_steps=2)
|
|
643
|
-
assert len(img_bytes) > 1000
|
|
644
|
-
ASCIIColors.green("HF
|
|
645
|
-
|
|
808
|
+
assert len(img_bytes) > 1000
|
|
809
|
+
ASCIIColors.green("HF t2i generation OK.")
|
|
646
810
|
del binding
|
|
647
811
|
time.sleep(0.1)
|
|
648
|
-
|
|
649
812
|
except Exception as e:
|
|
650
813
|
trace_exception(e)
|
|
651
814
|
ASCIIColors.error(f"Diffusers binding test failed: {e}")
|
|
@@ -653,4 +816,4 @@ if __name__ == '__main__':
|
|
|
653
816
|
ASCIIColors.cyan("\nCleaning up temporary directories...")
|
|
654
817
|
if temp_paths_dir.exists():
|
|
655
818
|
shutil.rmtree(temp_paths_dir)
|
|
656
|
-
ASCIIColors.magenta("--- Diffusers TTI Binding Test Finished ---")
|
|
819
|
+
ASCIIColors.magenta("--- Diffusers TTI Binding Test Finished ---")
|