agent-comfy-ui-api-workflow 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_comfy_ui_api_workflow/__about__.py +1 -0
- agent_comfy_ui_api_workflow/__init__.py +39 -0
- agent_comfy_ui_api_workflow/_impl/__init__.py +0 -0
- agent_comfy_ui_api_workflow/_impl/agent.py +276 -0
- agent_comfy_ui_api_workflow/_impl/catalog/__init__.py +348 -0
- agent_comfy_ui_api_workflow/_impl/catalog/essential.py +98 -0
- agent_comfy_ui_api_workflow/_impl/catalog/new_object_info.json +58779 -0
- agent_comfy_ui_api_workflow/_impl/env.py +44 -0
- agent_comfy_ui_api_workflow/_impl/errors.py +54 -0
- agent_comfy_ui_api_workflow/_impl/families.py +314 -0
- agent_comfy_ui_api_workflow/_impl/files.py +152 -0
- agent_comfy_ui_api_workflow/_impl/inventory.py +168 -0
- agent_comfy_ui_api_workflow/_impl/middleware.py +39 -0
- agent_comfy_ui_api_workflow/_impl/model.py +32 -0
- agent_comfy_ui_api_workflow/_impl/pipelines/__init__.py +70 -0
- agent_comfy_ui_api_workflow/_impl/pipelines/base.py +87 -0
- agent_comfy_ui_api_workflow/_impl/pipelines/generation.py +1249 -0
- agent_comfy_ui_api_workflow/_impl/pipelines/media.py +370 -0
- agent_comfy_ui_api_workflow/_impl/prompts.py +101 -0
- agent_comfy_ui_api_workflow/_impl/runpod.py +209 -0
- agent_comfy_ui_api_workflow/_impl/session.py +357 -0
- agent_comfy_ui_api_workflow/_impl/tools/__init__.py +25 -0
- agent_comfy_ui_api_workflow/_impl/tools/common.py +56 -0
- agent_comfy_ui_api_workflow/_impl/tools/editing.py +249 -0
- agent_comfy_ui_api_workflow/_impl/tools/nodes.py +98 -0
- agent_comfy_ui_api_workflow/_impl/tools/pipelines.py +92 -0
- agent_comfy_ui_api_workflow/_impl/tools/questions.py +42 -0
- agent_comfy_ui_api_workflow/_impl/validation.py +142 -0
- agent_comfy_ui_api_workflow-0.0.1.dist-info/METADATA +191 -0
- agent_comfy_ui_api_workflow-0.0.1.dist-info/RECORD +32 -0
- agent_comfy_ui_api_workflow-0.0.1.dist-info/WHEEL +4 -0
- agent_comfy_ui_api_workflow-0.0.1.dist-info/licenses/LICENSE +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
version = "0.0.1"
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from .__about__ import version as __version__
|
|
2
|
+
from ._impl.agent import AgentReply, WorkflowAgent
|
|
3
|
+
from ._impl.errors import (
|
|
4
|
+
AgentComfyUIError,
|
|
5
|
+
InputFileError,
|
|
6
|
+
InventoryError,
|
|
7
|
+
JobTimeoutError,
|
|
8
|
+
RunPodError,
|
|
9
|
+
SettingsError,
|
|
10
|
+
)
|
|
11
|
+
from ._impl.files import InputFile
|
|
12
|
+
from ._impl.inventory import ServerInventory
|
|
13
|
+
from ._impl.pipelines import MEDIA_PIPELINES, AgentPipeline
|
|
14
|
+
from ._impl.runpod import JobResult, OutputFile, RunPodClient
|
|
15
|
+
from ._impl.session import Question, WorkflowDiff
|
|
16
|
+
from ._impl.validation import Problem, ValidationReport
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"MEDIA_PIPELINES",
|
|
20
|
+
"AgentComfyUIError",
|
|
21
|
+
"AgentPipeline",
|
|
22
|
+
"AgentReply",
|
|
23
|
+
"InputFile",
|
|
24
|
+
"InputFileError",
|
|
25
|
+
"InventoryError",
|
|
26
|
+
"JobResult",
|
|
27
|
+
"JobTimeoutError",
|
|
28
|
+
"OutputFile",
|
|
29
|
+
"Problem",
|
|
30
|
+
"Question",
|
|
31
|
+
"RunPodClient",
|
|
32
|
+
"RunPodError",
|
|
33
|
+
"ServerInventory",
|
|
34
|
+
"SettingsError",
|
|
35
|
+
"ValidationReport",
|
|
36
|
+
"WorkflowAgent",
|
|
37
|
+
"WorkflowDiff",
|
|
38
|
+
"__version__",
|
|
39
|
+
]
|
|
File without changes
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import copy
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import Iterable, Mapping
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Literal
|
|
8
|
+
|
|
9
|
+
import anthropic
|
|
10
|
+
from langchain.agents import create_agent
|
|
11
|
+
from langchain.agents.middleware import ModelCallLimitMiddleware
|
|
12
|
+
from langchain_anthropic.middleware import AnthropicPromptCachingMiddleware
|
|
13
|
+
from langchain_core.language_models import BaseChatModel
|
|
14
|
+
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
|
|
15
|
+
from langgraph.checkpoint.memory import InMemorySaver
|
|
16
|
+
from langgraph.errors import GraphRecursionError
|
|
17
|
+
|
|
18
|
+
from .catalog import NodeCatalog
|
|
19
|
+
from .env import load_settings
|
|
20
|
+
from .families import family_nodes, find_models
|
|
21
|
+
from .files import FILE_INPUTS, InputFile, vision_block
|
|
22
|
+
from .inventory import ServerInventory
|
|
23
|
+
from .middleware import ValidationGate
|
|
24
|
+
from .model import DEFAULT_EFFORT, DEFAULT_MODEL, build_chat_model
|
|
25
|
+
from .pipelines import AgentPipeline, available_pipelines
|
|
26
|
+
from .prompts import system_prompt
|
|
27
|
+
from .session import Question, WorkflowDiff, WorkflowSession, diff_workflows
|
|
28
|
+
from .tools import build_tools
|
|
29
|
+
from .validation import ValidationReport
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
Status = Literal["completed", "needs_input", "invalid", "refused", "error"]
|
|
34
|
+
|
|
35
|
+
# Nodes that load a file from a server folder: with the folder empty they cannot work.
|
|
36
|
+
_FOLDER_NODES: dict[str, str] = {
|
|
37
|
+
"CheckpointLoaderSimple": "checkpoints",
|
|
38
|
+
"UNETLoader": "diffusion_models",
|
|
39
|
+
"CLIPLoader": "text_encoders",
|
|
40
|
+
"DualCLIPLoader": "text_encoders",
|
|
41
|
+
"VAELoader": "vae",
|
|
42
|
+
"LoraLoader": "loras",
|
|
43
|
+
"LoraLoaderModelOnly": "loras",
|
|
44
|
+
"UpscaleModelLoader": "upscale_models",
|
|
45
|
+
"ImageUpscaleWithModel": "upscale_models",
|
|
46
|
+
"AudioEncoderLoader": "audio_encoders",
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def hidden_nodes(inventory: ServerInventory) -> set[str]:
|
|
51
|
+
return {
|
|
52
|
+
node
|
|
53
|
+
for node, folder in _FOLDER_NODES.items()
|
|
54
|
+
if inventory.knows(folder) and not inventory.folder(folder)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class AgentReply:
|
|
60
|
+
"""What a turn produced: the answer, and the workflow as it is now."""
|
|
61
|
+
|
|
62
|
+
status: Status
|
|
63
|
+
message: str
|
|
64
|
+
workflow: dict[str, Any]
|
|
65
|
+
question: Question | None = None
|
|
66
|
+
changes: WorkflowDiff = field(default_factory=WorkflowDiff)
|
|
67
|
+
validation: ValidationReport = field(default_factory=ValidationReport)
|
|
68
|
+
# The user files the workflow loads; they go to the server with it.
|
|
69
|
+
input_files: tuple[str, ...] = ()
|
|
70
|
+
# Tokens the turn used, added over its model calls.
|
|
71
|
+
usage: dict[str, int] = field(default_factory=dict)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _referenced_files(workflow: Mapping[str, Any]) -> tuple[str, ...]:
|
|
75
|
+
names: list[str] = []
|
|
76
|
+
for node in workflow.values():
|
|
77
|
+
for name, value in node["inputs"].items():
|
|
78
|
+
if (node["class_type"], name) in FILE_INPUTS and isinstance(value, str) and value not in names:
|
|
79
|
+
names.append(value)
|
|
80
|
+
return tuple(names)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _usage(messages: list[BaseMessage]) -> dict[str, int]:
|
|
84
|
+
totals = dict.fromkeys(
|
|
85
|
+
("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens", "model_calls"), 0
|
|
86
|
+
)
|
|
87
|
+
for message in messages:
|
|
88
|
+
if not isinstance(message, AIMessage) or message.usage_metadata is None:
|
|
89
|
+
continue
|
|
90
|
+
usage = message.usage_metadata
|
|
91
|
+
details = usage.get("input_token_details") or {}
|
|
92
|
+
totals["input_tokens"] += usage.get("input_tokens", 0)
|
|
93
|
+
totals["output_tokens"] += usage.get("output_tokens", 0)
|
|
94
|
+
totals["cache_read_tokens"] += details.get("cache_read") or 0
|
|
95
|
+
totals["cache_write_tokens"] += details.get("cache_creation") or 0
|
|
96
|
+
totals["model_calls"] += 1
|
|
97
|
+
return totals
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _final_text(messages: list[BaseMessage]) -> str:
|
|
101
|
+
for message in reversed(messages):
|
|
102
|
+
if isinstance(message, AIMessage) and message.text.strip():
|
|
103
|
+
return message.text.strip()
|
|
104
|
+
if isinstance(message, HumanMessage):
|
|
105
|
+
break
|
|
106
|
+
return ""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class WorkflowAgent:
|
|
110
|
+
"""Builds and edits a ComfyUI API workflow, turn by turn, from what the user asks."""
|
|
111
|
+
|
|
112
|
+
def __init__(
|
|
113
|
+
self,
|
|
114
|
+
inventory: ServerInventory,
|
|
115
|
+
*,
|
|
116
|
+
workflow: Mapping[str, Any] | None = None,
|
|
117
|
+
model: str = DEFAULT_MODEL,
|
|
118
|
+
effort: str = DEFAULT_EFFORT,
|
|
119
|
+
api_key: str | None = None,
|
|
120
|
+
env_file: str | Path = ".env",
|
|
121
|
+
chat_model: BaseChatModel | None = None,
|
|
122
|
+
pipelines: Iterable[type[AgentPipeline]] | None = None,
|
|
123
|
+
refusal_fallback: bool = True,
|
|
124
|
+
vision: bool = True,
|
|
125
|
+
max_model_calls: int = 40,
|
|
126
|
+
) -> None:
|
|
127
|
+
models = find_models(inventory)
|
|
128
|
+
catalog = NodeCatalog(extra=family_nodes(models), hidden=hidden_nodes(inventory))
|
|
129
|
+
if workflow is None:
|
|
130
|
+
session = WorkflowSession(catalog, inventory, models=models)
|
|
131
|
+
else:
|
|
132
|
+
session = WorkflowSession.from_api(workflow, catalog, inventory, models=models)
|
|
133
|
+
# A generation pipeline is offered only when a model on the server can run it.
|
|
134
|
+
capabilities = {capability for model in models for capability in model.capabilities}
|
|
135
|
+
chosen = tuple(
|
|
136
|
+
pipeline
|
|
137
|
+
for pipeline in (available_pipelines(models) if pipelines is None else pipelines)
|
|
138
|
+
if pipeline.CAPABILITY is None or pipeline.CAPABILITY in capabilities
|
|
139
|
+
)
|
|
140
|
+
if chat_model is None:
|
|
141
|
+
key = api_key or load_settings(["ANTHROPIC_API_KEY"], env_file)["ANTHROPIC_API_KEY"]
|
|
142
|
+
chat_model = build_chat_model(
|
|
143
|
+
model, api_key=key, effort=effort, refusal_fallback=refusal_fallback
|
|
144
|
+
)
|
|
145
|
+
profile = getattr(chat_model, "profile", None) or {}
|
|
146
|
+
# The turn's context goes in a system message when the model takes one mid
|
|
147
|
+
# conversation, so it never mixes with the user's words.
|
|
148
|
+
self.__system_context = bool(profile.get("mid_conversation_system_messages"))
|
|
149
|
+
self.__session = session
|
|
150
|
+
self.__vision = vision
|
|
151
|
+
self.__turn = asyncio.Lock()
|
|
152
|
+
self.__turns = 0
|
|
153
|
+
self.__graph = create_agent(
|
|
154
|
+
chat_model,
|
|
155
|
+
build_tools(session, chosen),
|
|
156
|
+
system_prompt=system_prompt(catalog, inventory, chosen, models),
|
|
157
|
+
middleware=[
|
|
158
|
+
AnthropicPromptCachingMiddleware(unsupported_model_behavior="ignore"),
|
|
159
|
+
ValidationGate(session),
|
|
160
|
+
ModelCallLimitMiddleware(run_limit=max_model_calls, exit_behavior="end"),
|
|
161
|
+
],
|
|
162
|
+
checkpointer=InMemorySaver(),
|
|
163
|
+
)
|
|
164
|
+
self.__config = {"configurable": {"thread_id": "workflow"}, "recursion_limit": 10 * max_model_calls}
|
|
165
|
+
|
|
166
|
+
@classmethod
|
|
167
|
+
def from_workflow(
|
|
168
|
+
cls, workflow: Mapping[str, Any], inventory: ServerInventory, **options: Any
|
|
169
|
+
) -> "WorkflowAgent":
|
|
170
|
+
"""An agent that edits a workflow exported with ComfyUI's Export (API)."""
|
|
171
|
+
return cls(inventory, workflow=workflow, **options)
|
|
172
|
+
|
|
173
|
+
@property
|
|
174
|
+
def workflow(self) -> dict[str, Any]:
|
|
175
|
+
return self.__session.snapshot()
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def previous_workflows(self) -> list[dict[str, Any]]:
|
|
179
|
+
"""The workflows the agent set aside to start a new one, oldest first."""
|
|
180
|
+
return copy.deepcopy(self.__session.previous)
|
|
181
|
+
|
|
182
|
+
@property
|
|
183
|
+
def files(self) -> dict[str, bytes]:
|
|
184
|
+
"""The attached files the workflow loads, to send to the server with it."""
|
|
185
|
+
session = self.__session
|
|
186
|
+
names = [name for name in _referenced_files(self.workflow) if name in session.files]
|
|
187
|
+
return session.files.payload(names)
|
|
188
|
+
|
|
189
|
+
def view(self) -> str:
|
|
190
|
+
return self.__session.view()
|
|
191
|
+
|
|
192
|
+
def validate(self) -> ValidationReport:
|
|
193
|
+
return self.__session.validate()
|
|
194
|
+
|
|
195
|
+
async def send(self, text: str, files: Mapping[str, bytes] | None = None) -> AgentReply:
|
|
196
|
+
async with self.__turn:
|
|
197
|
+
session = self.__session
|
|
198
|
+
attached = [session.files.add(name, data) for name, data in (files or {}).items()]
|
|
199
|
+
before = session.snapshot()
|
|
200
|
+
session.start_turn()
|
|
201
|
+
self.__turns += 1
|
|
202
|
+
turn_id = f"user-turn-{self.__turns}"
|
|
203
|
+
try:
|
|
204
|
+
result = await self.__graph.ainvoke(
|
|
205
|
+
{"messages": self.__messages(text, attached, turn_id)}, self.__config
|
|
206
|
+
)
|
|
207
|
+
except (anthropic.APIError, GraphRecursionError) as error:
|
|
208
|
+
logger.warning("The agent turn failed: %s", error)
|
|
209
|
+
return self.__reply("error", f"The agent could not finish the turn: {error}", before)
|
|
210
|
+
messages = result["messages"]
|
|
211
|
+
start = next((index for index, message in enumerate(messages) if message.id == turn_id), 0)
|
|
212
|
+
usage = _usage(messages[start:])
|
|
213
|
+
last = messages[-1]
|
|
214
|
+
if session.question is not None:
|
|
215
|
+
return self.__reply("needs_input", session.question.text, before, usage)
|
|
216
|
+
if isinstance(last, AIMessage) and last.response_metadata.get("stop_reason") == "refusal":
|
|
217
|
+
return self.__reply("refused", "The model declined this request.", before, usage)
|
|
218
|
+
return self.__reply(None, _final_text(messages), before, usage)
|
|
219
|
+
|
|
220
|
+
def __reply(
|
|
221
|
+
self,
|
|
222
|
+
status: Status | None,
|
|
223
|
+
message: str,
|
|
224
|
+
before: dict[str, Any],
|
|
225
|
+
usage: dict[str, int] | None = None,
|
|
226
|
+
) -> AgentReply:
|
|
227
|
+
session = self.__session
|
|
228
|
+
workflow = session.snapshot()
|
|
229
|
+
changes = diff_workflows(before, workflow)
|
|
230
|
+
report = session.validate()
|
|
231
|
+
if status is None:
|
|
232
|
+
status = "invalid" if changes and not report.ok else "completed"
|
|
233
|
+
return AgentReply(
|
|
234
|
+
status=status,
|
|
235
|
+
message=message or changes.text(),
|
|
236
|
+
workflow=workflow,
|
|
237
|
+
question=session.question,
|
|
238
|
+
changes=changes,
|
|
239
|
+
validation=report,
|
|
240
|
+
input_files=_referenced_files(workflow),
|
|
241
|
+
usage=usage or {},
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
def __messages(self, text: str, attached: list[InputFile], turn_id: str) -> list[BaseMessage]:
|
|
245
|
+
images = [file for file in attached if file.kind == "image"] if self.__vision else []
|
|
246
|
+
blocks: list[dict[str, Any]] = []
|
|
247
|
+
for file in images:
|
|
248
|
+
blocks.append({"type": "text", "text": f"Attached image {file.name}:"})
|
|
249
|
+
blocks.append(vision_block(file))
|
|
250
|
+
context = self.__context(attached)
|
|
251
|
+
if self.__system_context:
|
|
252
|
+
content: Any = [*blocks, {"type": "text", "text": text}] if blocks else text
|
|
253
|
+
return [HumanMessage(content, id=turn_id), SystemMessage(context)]
|
|
254
|
+
content = [{"type": "text", "text": context}, *blocks, {"type": "text", "text": text}]
|
|
255
|
+
return [HumanMessage(content, id=turn_id)]
|
|
256
|
+
|
|
257
|
+
def __context(self, attached: list[InputFile]) -> str:
|
|
258
|
+
session = self.__session
|
|
259
|
+
now = ", ".join(file.describe() for file in attached) or "none"
|
|
260
|
+
lines = [
|
|
261
|
+
"Application context for this turn (not written by the user):",
|
|
262
|
+
f"- Files attached with this message: {now}.",
|
|
263
|
+
]
|
|
264
|
+
earlier = [file.describe() for file in session.files if file not in attached]
|
|
265
|
+
if earlier:
|
|
266
|
+
lines.append(f"- Files attached earlier: {', '.join(earlier)}.")
|
|
267
|
+
count = len(session.workflow.nodes)
|
|
268
|
+
if count:
|
|
269
|
+
outputs = ", ".join(session.output_node_ids()) or "none"
|
|
270
|
+
lines.append(
|
|
271
|
+
f"- Current workflow: {count} nodes; output nodes: {outputs}. "
|
|
272
|
+
"Call view_workflow before changing it."
|
|
273
|
+
)
|
|
274
|
+
else:
|
|
275
|
+
lines.append("- Current workflow: empty.")
|
|
276
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import importlib
|
|
3
|
+
import inspect
|
|
4
|
+
import json
|
|
5
|
+
import pkgutil
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import Iterable, Mapping
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from importlib import resources
|
|
10
|
+
from types import NoneType, UnionType
|
|
11
|
+
from typing import Any, Literal, Union, get_args, get_origin
|
|
12
|
+
|
|
13
|
+
from comfy_ui_api_workflow import nodes
|
|
14
|
+
from comfy_ui_api_workflow.nodes.base import BaseNode
|
|
15
|
+
|
|
16
|
+
from .essential import ESSENTIAL_GROUPS
|
|
17
|
+
|
|
18
|
+
__all__ = ["ESSENTIAL_GROUPS", "InputSpec", "NodeCatalog", "get_catalog", "node_types"]
|
|
19
|
+
|
|
20
|
+
_NUMBERED = re.compile(r"^(.*?\D)(\d+)$")
|
|
21
|
+
_MAX_CHOICES = 60
|
|
22
|
+
_SUMMARY_LENGTH = 160
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@functools.cache
|
|
26
|
+
def node_types() -> dict[str, type[BaseNode]]:
|
|
27
|
+
"""Every node class of comfy_ui_api_workflow, by ComfyUI class type."""
|
|
28
|
+
prefix = f"{nodes.__name__}."
|
|
29
|
+
for module in pkgutil.walk_packages(nodes.__path__, prefix):
|
|
30
|
+
importlib.import_module(module.name)
|
|
31
|
+
found: dict[str, type[BaseNode]] = {}
|
|
32
|
+
pending = list(BaseNode.__subclasses__())
|
|
33
|
+
while pending:
|
|
34
|
+
node_type = pending.pop()
|
|
35
|
+
pending.extend(node_type.__subclasses__())
|
|
36
|
+
if node_type.__module__.startswith(prefix):
|
|
37
|
+
found[node_type.CLASS_TYPE] = node_type
|
|
38
|
+
return found
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@functools.cache
|
|
42
|
+
def _lookup() -> dict[str, type[BaseNode]]:
|
|
43
|
+
names: dict[str, type[BaseNode]] = {}
|
|
44
|
+
for class_type, node_type in node_types().items():
|
|
45
|
+
for name in (class_type, node_type.__name__):
|
|
46
|
+
names[name] = node_type
|
|
47
|
+
names.setdefault(name.lower(), node_type)
|
|
48
|
+
return names
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@functools.cache
|
|
52
|
+
def object_info() -> dict[str, Any]:
|
|
53
|
+
data = resources.files(__package__).joinpath("new_object_info.json")
|
|
54
|
+
return json.loads(data.read_text(encoding="utf-8"))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _options(annotation: Any) -> tuple[Any, ...]:
|
|
58
|
+
if get_origin(annotation) in (Union, UnionType):
|
|
59
|
+
return get_args(annotation)
|
|
60
|
+
return (annotation,)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _spec_meta(spec: Any) -> dict[str, Any]:
|
|
64
|
+
return spec[1] if isinstance(spec, list) and len(spec) > 1 and isinstance(spec[1], dict) else {}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _section_inputs(sections: Mapping[str, Any]) -> dict[str, Any]:
|
|
68
|
+
return {**(sections.get("required") or {}), **(sections.get("optional") or {})}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _input_meta(info: Mapping[str, Any], comfy_name: str) -> dict[str, Any]:
|
|
72
|
+
# Dynamic inputs are sent as "<combo>.<input>"; their metadata sits inside the
|
|
73
|
+
# option or the template that opens them.
|
|
74
|
+
parts = comfy_name.split(".")
|
|
75
|
+
spec = _section_inputs(info.get("input") or {}).get(parts[0])
|
|
76
|
+
for part in parts[1:]:
|
|
77
|
+
if spec is None:
|
|
78
|
+
return {}
|
|
79
|
+
kind, meta = spec[0], _spec_meta(spec)
|
|
80
|
+
if kind == "COMFY_DYNAMICCOMBO_V3":
|
|
81
|
+
spec = next(
|
|
82
|
+
(
|
|
83
|
+
inputs[part]
|
|
84
|
+
for option in meta.get("options", [])
|
|
85
|
+
if part in (inputs := _section_inputs(option.get("inputs") or {}))
|
|
86
|
+
),
|
|
87
|
+
None,
|
|
88
|
+
)
|
|
89
|
+
elif kind == "COMFY_AUTOGROW_V3":
|
|
90
|
+
template = meta.get("template") or {}
|
|
91
|
+
inputs = _section_inputs(template.get("input") or {})
|
|
92
|
+
spec = inputs.get(template.get("prefix", "")) or next(iter(inputs.values()), None)
|
|
93
|
+
else:
|
|
94
|
+
return {}
|
|
95
|
+
meta = _spec_meta(spec) if spec is not None else {}
|
|
96
|
+
return {key: meta[key] for key in ("tooltip", "min", "max", "default") if key in meta}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass(frozen=True)
|
|
100
|
+
class InputSpec:
|
|
101
|
+
name: str
|
|
102
|
+
comfy_name: str
|
|
103
|
+
socket: str | None
|
|
104
|
+
annotation: Any
|
|
105
|
+
required: bool
|
|
106
|
+
default: Any
|
|
107
|
+
meta: Mapping[str, Any] = field(default_factory=dict)
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def link_only(self) -> bool:
|
|
111
|
+
options = [option for option in _options(self.annotation) if option is not NoneType]
|
|
112
|
+
return bool(options) and all(get_origin(option) is tuple for option in options)
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def accepts_link(self) -> bool:
|
|
116
|
+
return self.annotation is Any or any(
|
|
117
|
+
get_origin(option) is tuple for option in _options(self.annotation)
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def choices(self) -> tuple[Any, ...]:
|
|
122
|
+
return tuple(
|
|
123
|
+
choice
|
|
124
|
+
for option in _options(self.annotation)
|
|
125
|
+
if get_origin(option) is Literal
|
|
126
|
+
for choice in get_args(option)
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def has_default(self) -> bool:
|
|
131
|
+
return self.default is not inspect.Parameter.empty and self.default is not None
|
|
132
|
+
|
|
133
|
+
def value_text(self) -> str:
|
|
134
|
+
if self.link_only:
|
|
135
|
+
return f"link to {self.socket or 'an output'}"
|
|
136
|
+
if self.choices:
|
|
137
|
+
shown = ", ".join(map(str, self.choices[:_MAX_CHOICES]))
|
|
138
|
+
if len(self.choices) > _MAX_CHOICES:
|
|
139
|
+
shown += f", … ({len(self.choices)} options)"
|
|
140
|
+
return f"one of: {shown}"
|
|
141
|
+
kinds = [option for option in _options(self.annotation) if option in (bool, int, float, str)]
|
|
142
|
+
if bool in kinds:
|
|
143
|
+
return "true or false"
|
|
144
|
+
if int in kinds or float in kinds:
|
|
145
|
+
word = "number" if float in kinds else "int"
|
|
146
|
+
low, high = self.meta.get("min"), self.meta.get("max")
|
|
147
|
+
if low is not None and high is not None:
|
|
148
|
+
return f"{word} from {low} to {high}"
|
|
149
|
+
if low is not None:
|
|
150
|
+
return f"{word}, at least {low}"
|
|
151
|
+
if high is not None:
|
|
152
|
+
return f"{word}, at most {high}"
|
|
153
|
+
return word
|
|
154
|
+
if str in kinds:
|
|
155
|
+
return "text"
|
|
156
|
+
if self.socket:
|
|
157
|
+
return f"link to {self.socket} or a value"
|
|
158
|
+
return "any value"
|
|
159
|
+
|
|
160
|
+
def line(self) -> str:
|
|
161
|
+
parts = [self.value_text()]
|
|
162
|
+
if self.required:
|
|
163
|
+
parts.append("required")
|
|
164
|
+
elif self.has_default:
|
|
165
|
+
parts.append(f"default {self.default!r}")
|
|
166
|
+
else:
|
|
167
|
+
parts.append("optional")
|
|
168
|
+
tooltip = str(self.meta.get("tooltip") or "").strip()
|
|
169
|
+
return f"- {self.name}: {', '.join(parts)}." + (f" {tooltip}" if tooltip else "")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@functools.cache
|
|
173
|
+
def _input_specs(node_type: type[BaseNode]) -> tuple[InputSpec, ...]:
|
|
174
|
+
info = object_info().get(node_type.CLASS_TYPE, {})
|
|
175
|
+
specs: list[InputSpec] = []
|
|
176
|
+
for parameter in inspect.signature(node_type).parameters.values():
|
|
177
|
+
if parameter.name == "node_id" or parameter.kind is parameter.VAR_KEYWORD:
|
|
178
|
+
continue
|
|
179
|
+
comfy_name = node_type.INPUT_NAMES.get(parameter.name, parameter.name)
|
|
180
|
+
specs.append(
|
|
181
|
+
InputSpec(
|
|
182
|
+
name=parameter.name,
|
|
183
|
+
comfy_name=comfy_name,
|
|
184
|
+
socket=node_type.INPUT_SOCKET_TYPES.get(parameter.name),
|
|
185
|
+
annotation=parameter.annotation,
|
|
186
|
+
required=parameter.default is inspect.Parameter.empty,
|
|
187
|
+
default=parameter.default,
|
|
188
|
+
meta=_input_meta(info, comfy_name),
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
return tuple(specs)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _numbered_runs(specs: Iterable[InputSpec]) -> list[list[InputSpec]]:
|
|
195
|
+
# ConcatenateVideo has videos_video0 … videos_video99: one line says it better.
|
|
196
|
+
runs: list[list[InputSpec]] = []
|
|
197
|
+
for spec in specs:
|
|
198
|
+
match = _NUMBERED.match(spec.name)
|
|
199
|
+
if runs and match:
|
|
200
|
+
previous = _NUMBERED.match(runs[-1][-1].name)
|
|
201
|
+
if (
|
|
202
|
+
previous
|
|
203
|
+
and previous.group(1) == match.group(1)
|
|
204
|
+
and int(previous.group(2)) + 1 == int(match.group(2))
|
|
205
|
+
and runs[-1][-1].socket == spec.socket
|
|
206
|
+
):
|
|
207
|
+
runs[-1].append(spec)
|
|
208
|
+
continue
|
|
209
|
+
runs.append([spec])
|
|
210
|
+
return runs
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _run_line(run: list[InputSpec]) -> str:
|
|
214
|
+
first, last = run[0], run[-1]
|
|
215
|
+
required = sum(spec.required for spec in run)
|
|
216
|
+
needed = f"the first {required} required, the rest optional" if required else "all optional"
|
|
217
|
+
tooltip = str(first.meta.get("tooltip") or "").strip()
|
|
218
|
+
line = f"- {first.name} … {last.name}: {first.value_text()}; {needed}, filled in order."
|
|
219
|
+
return line + (f" {tooltip}" if tooltip else "")
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _summary(text: str) -> str:
|
|
223
|
+
first = text.strip().split("\n")[0].strip()
|
|
224
|
+
# A description that opens a list ("Recipes:") says nothing on its own line.
|
|
225
|
+
if first.endswith(":"):
|
|
226
|
+
return ""
|
|
227
|
+
end = first.find(". ")
|
|
228
|
+
if end != -1:
|
|
229
|
+
first = first[: end + 1]
|
|
230
|
+
if len(first) > _SUMMARY_LENGTH:
|
|
231
|
+
first = first[: _SUMMARY_LENGTH - 1].rstrip() + "…"
|
|
232
|
+
return first
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
class NodeCatalog:
|
|
236
|
+
"""The essential nodes the agent may add, with their documentation."""
|
|
237
|
+
|
|
238
|
+
def __init__(
|
|
239
|
+
self,
|
|
240
|
+
*,
|
|
241
|
+
extra: Mapping[str, Iterable[str]] | None = None,
|
|
242
|
+
hidden: Iterable[str] = (),
|
|
243
|
+
) -> None:
|
|
244
|
+
hidden = set(hidden)
|
|
245
|
+
groups: dict[str, tuple[type[BaseNode], ...]] = {}
|
|
246
|
+
for group, names in {**ESSENTIAL_GROUPS, **(extra or {})}.items():
|
|
247
|
+
kept: list[type[BaseNode]] = []
|
|
248
|
+
for name in names:
|
|
249
|
+
node_type = self.node_type(name)
|
|
250
|
+
if node_type is None:
|
|
251
|
+
raise ValueError(f"{name!r} is not a node of comfy_ui_api_workflow")
|
|
252
|
+
if node_type.__name__ not in hidden and node_type.CLASS_TYPE not in hidden:
|
|
253
|
+
kept.append(node_type)
|
|
254
|
+
if kept:
|
|
255
|
+
groups[group] = tuple(kept)
|
|
256
|
+
self.groups = groups
|
|
257
|
+
self.__essential = frozenset(node_type for kept in groups.values() for node_type in kept)
|
|
258
|
+
|
|
259
|
+
@staticmethod
|
|
260
|
+
def node_type(name: str) -> type[BaseNode] | None:
|
|
261
|
+
names = _lookup()
|
|
262
|
+
return names.get(name) or names.get(name.lower())
|
|
263
|
+
|
|
264
|
+
def essential_type(self, name: str) -> type[BaseNode] | None:
|
|
265
|
+
node_type = self.node_type(name)
|
|
266
|
+
return node_type if node_type in self.__essential else None
|
|
267
|
+
|
|
268
|
+
def is_essential(self, node_type: type[BaseNode]) -> bool:
|
|
269
|
+
return node_type in self.__essential
|
|
270
|
+
|
|
271
|
+
@staticmethod
|
|
272
|
+
def info(node_type: type[BaseNode]) -> dict[str, Any]:
|
|
273
|
+
return object_info().get(node_type.CLASS_TYPE, {})
|
|
274
|
+
|
|
275
|
+
@staticmethod
|
|
276
|
+
def inputs(node_type: type[BaseNode]) -> tuple[InputSpec, ...]:
|
|
277
|
+
return _input_specs(node_type)
|
|
278
|
+
|
|
279
|
+
def input_spec(self, node_type: type[BaseNode], name: str) -> InputSpec | None:
|
|
280
|
+
return next(
|
|
281
|
+
(spec for spec in self.inputs(node_type) if name in (spec.name, spec.comfy_name)),
|
|
282
|
+
None,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
@staticmethod
|
|
286
|
+
def is_output_node(class_type: str) -> bool:
|
|
287
|
+
return bool(object_info().get(class_type, {}).get("output_node"))
|
|
288
|
+
|
|
289
|
+
def output_type(self, name: str, index: int) -> str | None:
|
|
290
|
+
node_type = self.node_type(name)
|
|
291
|
+
if node_type is None or not 0 <= index < len(node_type.RETURN_TYPES):
|
|
292
|
+
return None
|
|
293
|
+
return node_type.RETURN_TYPES[index]
|
|
294
|
+
|
|
295
|
+
def describe(self, node_type: type[BaseNode]) -> str:
|
|
296
|
+
info = self.info(node_type)
|
|
297
|
+
title = node_type.__name__
|
|
298
|
+
if node_type.CLASS_TYPE != node_type.__name__:
|
|
299
|
+
title += f" (ComfyUI class_type {node_type.CLASS_TYPE!r})"
|
|
300
|
+
about = str(info.get("description") or info.get("display_name") or "").strip()
|
|
301
|
+
header = f"{title}: {about}" if about else title
|
|
302
|
+
if info.get("category"):
|
|
303
|
+
header += f" (category {info['category']})"
|
|
304
|
+
lines = [header]
|
|
305
|
+
specs = self.inputs(node_type)
|
|
306
|
+
if specs:
|
|
307
|
+
lines.append(
|
|
308
|
+
"Inputs (numbers, text and choices also accept a link to an output of "
|
|
309
|
+
"the same type):"
|
|
310
|
+
)
|
|
311
|
+
for run in _numbered_runs(specs):
|
|
312
|
+
if len(run) >= 3:
|
|
313
|
+
lines.append(_run_line(run))
|
|
314
|
+
else:
|
|
315
|
+
lines.extend(spec.line() for spec in run)
|
|
316
|
+
else:
|
|
317
|
+
lines.append("Inputs: none.")
|
|
318
|
+
tooltips = info.get("output_tooltips") or []
|
|
319
|
+
if node_type.OUTPUTS:
|
|
320
|
+
lines.append('Outputs (link to one as "<node_id>.<output name>"):')
|
|
321
|
+
for name, index in sorted(node_type.OUTPUTS.items(), key=lambda item: item[1]):
|
|
322
|
+
kind = node_type.RETURN_TYPES[index] if index < len(node_type.RETURN_TYPES) else "?"
|
|
323
|
+
tip = tooltips[index] if index < len(tooltips) and tooltips[index] else ""
|
|
324
|
+
lines.append(f"- {name} ({kind})" + (f": {tip}" if tip else ""))
|
|
325
|
+
else:
|
|
326
|
+
lines.append("Outputs: none.")
|
|
327
|
+
return "\n".join(lines)
|
|
328
|
+
|
|
329
|
+
def summary_line(self, node_type: type[BaseNode]) -> str:
|
|
330
|
+
info = self.info(node_type)
|
|
331
|
+
required = ", ".join(spec.name for spec in self.inputs(node_type) if spec.required)
|
|
332
|
+
outputs = ", ".join(sorted(node_type.OUTPUTS, key=node_type.OUTPUTS.__getitem__))
|
|
333
|
+
summary = _summary(info.get("description") or "") or info.get("display_name") or ""
|
|
334
|
+
line = f"- {node_type.__name__}({required}) → {outputs or 'no outputs'}"
|
|
335
|
+
return f"{line}: {summary}" if summary else line
|
|
336
|
+
|
|
337
|
+
def index(self) -> str:
|
|
338
|
+
lines: list[str] = []
|
|
339
|
+
for group, kept in self.groups.items():
|
|
340
|
+
lines.append(f"{group}:")
|
|
341
|
+
lines.extend(self.summary_line(node_type) for node_type in kept)
|
|
342
|
+
lines.append("")
|
|
343
|
+
return "\n".join(lines).strip()
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
@functools.cache
|
|
347
|
+
def get_catalog() -> NodeCatalog:
|
|
348
|
+
return NodeCatalog()
|