flowforge-map 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowforge/__init__.py +0 -0
- flowforge/_classic.html +289 -0
- flowforge/_dashboard/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- flowforge/_dashboard/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
- flowforge/_dashboard/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- flowforge/_dashboard/assets/index-BGT5seIN.css +1 -0
- flowforge/_dashboard/assets/index-BoGikST6.js +52 -0
- flowforge/_dashboard/favicon.svg +1 -0
- flowforge/_dashboard/index.html +16 -0
- flowforge/_example/trip-planner/.env.example +3 -0
- flowforge/_example/trip-planner/.gitignore +2 -0
- flowforge/_example/trip-planner/.mcp.json +20 -0
- flowforge/_example/trip-planner/README.md +48 -0
- flowforge/_example/trip-planner/config.py +27 -0
- flowforge/_example/trip-planner/mcp_servers/currency.py +10 -0
- flowforge/_example/trip-planner/planner.py +113 -0
- flowforge/_example/trip-planner/requirements.txt +3 -0
- flowforge/_example/trip-planner/services/__init__.py +0 -0
- flowforge/_example/trip-planner/services/geo.py +49 -0
- flowforge/_example/trip-planner/services/llm.py +18 -0
- flowforge/_example/trip-planner/services/mcp_tools.py +26 -0
- flowforge/_example/trip-planner/services/weather.py +25 -0
- flowforge/_example/trip-planner/tools/packing_list.py +24 -0
- flowforge/_example/trip-planner/trips/.gitkeep +0 -0
- flowforge/_workflows/diamond_mock.json +71 -0
- flowforge/_workflows/stripe_to_razorpay.json +38 -0
- flowforge/agent/__init__.py +5 -0
- flowforge/agent/server.py +333 -0
- flowforge/api/__init__.py +0 -0
- flowforge/api/agent.py +79 -0
- flowforge/api/checks.py +255 -0
- flowforge/api/connectors.py +289 -0
- flowforge/api/flowmap.py +174 -0
- flowforge/api/runs.py +284 -0
- flowforge/api/scan.py +117 -0
- flowforge/api/security.py +31 -0
- flowforge/api/stats.py +80 -0
- flowforge/checks/__init__.py +13 -0
- flowforge/checks/models.py +41 -0
- flowforge/checks/probes.py +359 -0
- flowforge/cli.py +108 -0
- flowforge/connectors/__init__.py +0 -0
- flowforge/connectors/catalog.py +84 -0
- flowforge/connectors/models.py +149 -0
- flowforge/connectors/presets.py +192 -0
- flowforge/connectors/registry.py +85 -0
- flowforge/connectors/runtime.py +47 -0
- flowforge/connectors/secrets.py +53 -0
- flowforge/example_data.py +247 -0
- flowforge/flow/__init__.py +328 -0
- flowforge/flow/advice.py +132 -0
- flowforge/flow/curly.py +884 -0
- flowforge/flow/index.py +116 -0
- flowforge/flow/order.py +181 -0
- flowforge/flow/reach.py +145 -0
- flowforge/flow/rewrite.py +146 -0
- flowforge/flowmap/__init__.py +1 -0
- flowforge/flowmap/diff.py +145 -0
- flowforge/flowmap/fixture.py +283 -0
- flowforge/flowmap/models.py +258 -0
- flowforge/flowmap/planner.py +131 -0
- flowforge/flowmap/service.py +379 -0
- flowforge/flowmap/validate.py +228 -0
- flowforge/home.py +58 -0
- flowforge/main.py +137 -0
- flowforge/nodes/__init__.py +3 -0
- flowforge/nodes/base.py +74 -0
- flowforge/nodes/check_node.py +50 -0
- flowforge/nodes/http_node.py +161 -0
- flowforge/nodes/llm_node.py +158 -0
- flowforge/nodes/local_node.py +164 -0
- flowforge/nodes/mcp_node.py +243 -0
- flowforge/nodes/mock_node.py +40 -0
- flowforge/scan/__init__.py +6 -0
- flowforge/scan/facts.py +47 -0
- flowforge/scan/files.py +112 -0
- flowforge/scan/manifests.py +65 -0
- flowforge/scan/mcp_config.py +75 -0
- flowforge/scan/models.py +73 -0
- flowforge/scan/patterns.py +86 -0
- flowforge/scan/python_ast.py +246 -0
- flowforge/scan/scanner.py +344 -0
- flowforge/scan/services.py +58 -0
- flowforge/scan/urls.py +60 -0
- flowforge/scheduler/__init__.py +0 -0
- flowforge/scheduler/cache.py +76 -0
- flowforge/scheduler/critical_path.py +65 -0
- flowforge/scheduler/durations.py +35 -0
- flowforge/scheduler/executor.py +390 -0
- flowforge/scheduler/graph.py +108 -0
- flowforge/scheduler/rate_limit.py +36 -0
- flowforge/schema.py +81 -0
- flowforge/security/__init__.py +0 -0
- flowforge/security/finders.py +127 -0
- flowforge/security/findings.py +169 -0
- flowforge/security/logfilter.py +129 -0
- flowforge/spa.py +52 -0
- flowforge/state.py +73 -0
- flowforge/storage.py +481 -0
- flowforge/templating.py +60 -0
- flowforge_map-0.1.0.dist-info/METADATA +335 -0
- flowforge_map-0.1.0.dist-info/RECORD +113 -0
- flowforge_map-0.1.0.dist-info/WHEEL +4 -0
- flowforge_map-0.1.0.dist-info/entry_points.txt +3 -0
- flowforge_map-0.1.0.dist-info/licenses/LICENSE +21 -0
flowforge/api/checks.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""Check now (D24): a health check of every dependency on the map, run by the scheduler, kept over time.
|
|
2
|
+
|
|
3
|
+
GET /checks/plan what a check run would do, the MCP servers it would start, and whether you said yes
|
|
4
|
+
POST /checks {ids?, confirm?} start a check run (201, `run_id`); 409 until starting MCP servers is confirmed
|
|
5
|
+
GET /checks the latest verdict per dependency, in map order, with a summary
|
|
6
|
+
GET /checks/{dep_id} one dependency's checks, newest first
|
|
7
|
+
|
|
8
|
+
Progress streams over GET /runs/{run_id}/events, like any run (D16).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import re
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from fastapi import APIRouter, HTTPException
|
|
21
|
+
from pydantic import BaseModel, ConfigDict
|
|
22
|
+
|
|
23
|
+
from flowforge.api.runs import start_run
|
|
24
|
+
from flowforge.api.scan import repo_path
|
|
25
|
+
from flowforge.checks import CheckTarget, probes, starts_process
|
|
26
|
+
from flowforge.nodes.check_node import MCP_AT_ONCE
|
|
27
|
+
from flowforge.scan.models import Dependency, EnvName, ScanResult
|
|
28
|
+
from flowforge.scan.urls import clean_url
|
|
29
|
+
from flowforge.schema import Step, Workflow
|
|
30
|
+
from flowforge.state import AppState, app_state
|
|
31
|
+
|
|
32
|
+
router = APIRouter()
|
|
33
|
+
|
|
34
|
+
CHECKABLE = ("llm", "api", "mcp", "local")
|
|
35
|
+
CONFIRMED_SETTING = "checks_confirmed" # the MCP commands the user agreed to start, one by one (JSON list)
|
|
36
|
+
LAST_RUN_SETTING = "checks_last_run"
|
|
37
|
+
HISTORY = 12 # checks shown per dependency
|
|
38
|
+
COUNTS = ("up", "warn", "down", "skipped", "unchecked", "unconfirmed")
|
|
39
|
+
# rough check times by kind, so the scheduler starts the slow MCP servers first (D1 weights, D24)
|
|
40
|
+
ESTIMATE_MS = {"mcp": 3000.0, "llm": 300.0, "api": 300.0, "local": 10.0}
|
|
41
|
+
STEP_SLACK_S = 5.0 # on top of a probe's own timeout
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class CheckRequest(BaseModel):
|
|
45
|
+
model_config = ConfigDict(extra="forbid")
|
|
46
|
+
|
|
47
|
+
ids: list[str] | None = None # None = every dependency on the map
|
|
48
|
+
confirm: bool = False # yes, start the MCP servers listed in /checks/plan
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _reads_env(dep: Dependency, scan: ScanResult) -> list[EnvName]:
|
|
52
|
+
"""Env names an MCP server's own script reads, and where, from the scan (for a likely fix when it won't start)."""
|
|
53
|
+
if dep.kind != "mcp" or not dep.command or len(dep.command) < 2:
|
|
54
|
+
return []
|
|
55
|
+
script = dep.command[1]
|
|
56
|
+
return [e for e in scan.env_names if e.ref.rsplit(":", 1)[0] == script]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _address(what: str | None) -> str | None:
|
|
60
|
+
"""A block you added: its text is an address only if it looks like one (no query kept, D24)."""
|
|
61
|
+
if not what or " " in what.strip():
|
|
62
|
+
return None
|
|
63
|
+
what = what.strip()
|
|
64
|
+
return clean_url(what) if "://" in what else (what if "." in what or ":" in what else None)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def target_for(dep: Dependency, scan: ScanResult, name: str | None = None) -> CheckTarget:
|
|
68
|
+
"""What a check of a scanned dependency reaches (D24); shared by Check now and the MCP server (D29)."""
|
|
69
|
+
return CheckTarget(id=dep.id, kind=dep.kind, name=name or dep.name, target=dep.target, command=dep.command,
|
|
70
|
+
model=dep.model, catalog=dep.catalog, env_names=dep.env_names, reads_env=_reads_env(dep, scan))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def confirmed_commands(storage: Any) -> set[str]:
|
|
74
|
+
"""The MCP commands the user said yes to starting (D24)."""
|
|
75
|
+
return set(json.loads(storage.setting(CONFIRMED_SETTING) or "[]"))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _target(node: dict[str, Any], dep: Dependency | None, scan: ScanResult | None) -> CheckTarget:
|
|
79
|
+
if dep is not None and scan is not None:
|
|
80
|
+
return target_for(dep, scan, node["title"])
|
|
81
|
+
what = (node.get("what") or "").strip() # a dependency the user added on the map: its address or command
|
|
82
|
+
address = _address(what)
|
|
83
|
+
if node["kind"] in ("mcp", "local") and what and "://" not in what:
|
|
84
|
+
command = what.split()
|
|
85
|
+
else:
|
|
86
|
+
command = [address] if node["kind"] == "mcp" and address else None
|
|
87
|
+
return CheckTarget(id=node["id"], kind=node["kind"], name=node["title"], catalog=node.get("catalog"),
|
|
88
|
+
target=address, command=command, model=(node.get("op") or {}).get("model"))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _map(s: AppState) -> tuple[list[dict[str, Any]], dict[str, CheckTarget], dict[str, Dependency]]:
|
|
92
|
+
"""The checkable blocks of the dependency map with your edits (hidden ones are gone), and their targets."""
|
|
93
|
+
view = s.flows["deps"].current("mine")
|
|
94
|
+
if not view.get("available"):
|
|
95
|
+
return [], {}, {}
|
|
96
|
+
analysis = s.flows["deps"].analysis() or {}
|
|
97
|
+
scan = ScanResult.model_validate(analysis["scan"]) if analysis.get("scan") else None
|
|
98
|
+
deps = {d.id: d for d in scan.dependencies} if scan else {}
|
|
99
|
+
nodes = [n for n in view["nodes"] if n["kind"] in CHECKABLE]
|
|
100
|
+
targets = {n["id"]: _target(n, deps.get(n["id"]), scan) for n in nodes if n["status"] != "unconfirmed"}
|
|
101
|
+
return nodes, targets, deps
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _origin(command: list[str], root: Path | None) -> str:
|
|
105
|
+
"""Where the code that starts comes from, in plain words."""
|
|
106
|
+
program = Path(command[0]).name
|
|
107
|
+
package = next((a for a in command[1:] if not a.startswith("-")), "")
|
|
108
|
+
if program in ("npx", "bunx", "pnpx"):
|
|
109
|
+
return "downloads from npm" + ("" if package.rfind("@") > 0 else ", any version")
|
|
110
|
+
if program in ("uvx", "pipx"):
|
|
111
|
+
return "downloads from PyPI" + ("" if "@" in package or "==" in package else ", any version")
|
|
112
|
+
if root is not None and package and (root / package).is_file():
|
|
113
|
+
return "runs a script in this repo"
|
|
114
|
+
return f"runs {program} from your PATH"
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def command_text(t: CheckTarget) -> str:
|
|
118
|
+
return " ".join(t.command or [])
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _confirmed(s: AppState) -> set[str]:
|
|
122
|
+
return confirmed_commands(s.storage)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _unconfirmed(s: AppState, targets: list[CheckTarget]) -> list[CheckTarget]:
|
|
126
|
+
"""The servers this check would start that the user hasn't said yes to. Each yes covers the command shown, so
|
|
127
|
+
a re-scan or an edit that changes a command asks again (D24). Example mode starts nothing."""
|
|
128
|
+
if s.example:
|
|
129
|
+
return []
|
|
130
|
+
yes = _confirmed(s)
|
|
131
|
+
return [t for t in targets if starts_process(t) and command_text(t) not in yes]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def check_workflow(targets: list[CheckTarget], root: str) -> Workflow:
|
|
135
|
+
"""One check step per dependency, no edges, no retries, no cache (D24). Only MCP_AT_ONCE servers start
|
|
136
|
+
together (CheckNode), so a server's timeout includes its wait for a turn: every round may take the full
|
|
137
|
+
MCP_TIMEOUT_S."""
|
|
138
|
+
rounds = math.ceil(sum(t.kind == "mcp" for t in targets) / MCP_AT_ONCE) or 1
|
|
139
|
+
timeout = {"mcp": probes.MCP_TIMEOUT_S * rounds, "llm": probes.HTTP_TIMEOUT_S, "api": probes.HTTP_TIMEOUT_S,
|
|
140
|
+
"local": 1.0}
|
|
141
|
+
steps = [Step(id="c_" + re.sub(r"\W", "_", t.id), type="check", retries=0, cache=False,
|
|
142
|
+
description=f"Check {t.name}", estimated_ms=ESTIMATE_MS[t.kind],
|
|
143
|
+
timeout_s=timeout[t.kind] + STEP_SLACK_S,
|
|
144
|
+
params={"target": t.model_dump(mode="json"), "root": root}) for t in targets]
|
|
145
|
+
return Workflow(id="health-check", name="Health check", max_concurrency=8, type_concurrency={"check": 8},
|
|
146
|
+
steps=steps)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@router.get("/checks/plan")
|
|
150
|
+
def check_plan() -> dict[str, Any]:
|
|
151
|
+
s = app_state()
|
|
152
|
+
_, targets, deps = _map(s)
|
|
153
|
+
root = repo_path(s)
|
|
154
|
+
starts = []
|
|
155
|
+
for t in targets.values():
|
|
156
|
+
if starts_process(t):
|
|
157
|
+
dep = deps.get(t.id)
|
|
158
|
+
source = dep.evidence[0].ref if dep and dep.evidence else None
|
|
159
|
+
starts.append({"id": t.id, "name": t.name, "command": " ".join(t.command or []),
|
|
160
|
+
"origin": _origin(t.command or [], root), "source": source})
|
|
161
|
+
confirmed = not _unconfirmed(s, list(targets.values()))
|
|
162
|
+
kinds = {k: sum(t.kind == k for t in targets.values()) for k in CHECKABLE}
|
|
163
|
+
return {"checks": len(targets), "kinds": kinds, "starts": starts, "confirmed": confirmed, "example": s.example}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
async def _record(s: AppState, run_id: str, workflow: Workflow) -> None:
|
|
167
|
+
"""When the run ends, keep a verdict for every check that finished (D24), from the run's events, so a run you
|
|
168
|
+
stopped still keeps what it found. A check that broke says so (skipped), never blaming the dependency; a
|
|
169
|
+
check that never ran records nothing."""
|
|
170
|
+
run = s.runs[run_id]
|
|
171
|
+
if run.task is not None:
|
|
172
|
+
await asyncio.wait([run.task])
|
|
173
|
+
targets = {step.id: step.params["target"] for step in workflow.steps}
|
|
174
|
+
rows = []
|
|
175
|
+
for event in run.events:
|
|
176
|
+
if event.get("type") != "step" or event.get("step_id") not in targets:
|
|
177
|
+
continue
|
|
178
|
+
out = event.get("output")
|
|
179
|
+
if event.get("state") == "succeeded" and isinstance(out, dict) and out.get("status"):
|
|
180
|
+
rows.append(out)
|
|
181
|
+
elif event.get("state") == "failed":
|
|
182
|
+
target = targets[event["step_id"]]
|
|
183
|
+
rows.append({"id": target["id"], "kind": target["kind"], "status": "skipped",
|
|
184
|
+
"summary": "The check couldn't run", "reason": event.get("error")})
|
|
185
|
+
if rows:
|
|
186
|
+
s.storage.add_checks(run_id, rows)
|
|
187
|
+
s.storage.set_setting(LAST_RUN_SETTING, run_id)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@router.post("/checks", status_code=201)
|
|
191
|
+
async def start_checks(req: CheckRequest) -> dict[str, Any]:
|
|
192
|
+
s = app_state()
|
|
193
|
+
_, targets, _ = _map(s)
|
|
194
|
+
root = repo_path(s)
|
|
195
|
+
if not targets or root is None:
|
|
196
|
+
raise HTTPException(409, "There is nothing to check yet. Scan a folder first.")
|
|
197
|
+
chosen = list(targets.values())
|
|
198
|
+
if req.ids is not None:
|
|
199
|
+
unknown = sorted(set(req.ids) - set(targets))
|
|
200
|
+
if unknown:
|
|
201
|
+
raise HTTPException(422, f"not on the map, or unconfirmed: {', '.join(unknown)}")
|
|
202
|
+
chosen = [targets[i] for i in req.ids]
|
|
203
|
+
asking = _unconfirmed(s, chosen)
|
|
204
|
+
if asking:
|
|
205
|
+
if not req.confirm:
|
|
206
|
+
n = len(asking)
|
|
207
|
+
raise HTTPException(409, f"Checking would start {n} MCP server{'s' if n != 1 else ''} on this computer. "
|
|
208
|
+
"See what starts in GET /checks/plan, then send confirm: true.")
|
|
209
|
+
# the yes covers exactly the servers this check starts, nothing else on the map
|
|
210
|
+
s.storage.set_setting(CONFIRMED_SETTING, json.dumps(sorted(_confirmed(s) | {command_text(t) for t in asking})))
|
|
211
|
+
workflow = check_workflow(chosen, str(root))
|
|
212
|
+
run_id = start_run(workflow, use_cache=False)
|
|
213
|
+
task = asyncio.create_task(_record(s, run_id, workflow))
|
|
214
|
+
s.tasks.add(task)
|
|
215
|
+
task.add_done_callback(s.tasks.discard)
|
|
216
|
+
return {"run_id": run_id, "checks": len(workflow.steps)}
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _item(node: dict[str, Any], history: list[dict[str, Any]]) -> dict[str, Any]:
|
|
220
|
+
item = {"id": node["id"], "name": node["title"], "kind": node["kind"], "catalog": node.get("catalog"),
|
|
221
|
+
"history": [h["status"] for h in reversed(history)]} # oldest first, as the squares read
|
|
222
|
+
if node["status"] == "unconfirmed":
|
|
223
|
+
return {**item, "status": "unconfirmed", "summary": "Not checked: unconfirmed", "history": []}
|
|
224
|
+
if not history:
|
|
225
|
+
return {**item, "status": "unchecked", "summary": "Not checked yet"}
|
|
226
|
+
last = history[0]
|
|
227
|
+
return {**item, **{k: last.get(k) for k in ("status", "latency_ms", "summary", "reason", "fix", "at")}}
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
@router.get("/checks")
|
|
231
|
+
def latest_checks() -> dict[str, Any]:
|
|
232
|
+
s = app_state()
|
|
233
|
+
nodes, _, _ = _map(s)
|
|
234
|
+
by_dep: dict[str, list[dict[str, Any]]] = {}
|
|
235
|
+
for row in s.storage.check_history(limit=HISTORY):
|
|
236
|
+
by_dep.setdefault(row["dep_id"], []).append(row)
|
|
237
|
+
items = [_item(n, by_dep.get(n["id"], [])) for n in nodes]
|
|
238
|
+
summary = {k: sum(i["status"] == k for i in items) for k in COUNTS}
|
|
239
|
+
checked = [i["at"] for i in items if i.get("at")]
|
|
240
|
+
run_id = s.storage.setting(LAST_RUN_SETTING)
|
|
241
|
+
last = s.storage.get_run(run_id) if run_id else None
|
|
242
|
+
return {"run_id": run_id, "checked_at": max(checked) if checked else None,
|
|
243
|
+
"took_ms": last.get("makespan_ms") if last else None,
|
|
244
|
+
"summary": summary, "items": items, "example": s.example}
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
@router.get("/checks/{dep_id}")
|
|
248
|
+
def dependency_checks(dep_id: str, limit: int = 50) -> list[dict[str, Any]]:
|
|
249
|
+
s = app_state()
|
|
250
|
+
nodes, _, _ = _map(s)
|
|
251
|
+
history = s.storage.check_history(dep_id, limit=limit)
|
|
252
|
+
if not history and dep_id not in {n["id"] for n in nodes}:
|
|
253
|
+
raise HTTPException(404, f"'{dep_id}' isn't on the map")
|
|
254
|
+
return [{k: h.get(k) for k in ("run_id", "status", "latency_ms", "summary", "reason", "fix", "at")}
|
|
255
|
+
for h in history]
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""Connectors: list, read, create, edit, delete, logo, test, activity, presets (D10, D12, D16).
|
|
2
|
+
|
|
3
|
+
Keys are references only: responses carry `secret: {"ref", "status": "set" | "missing"}`, never
|
|
4
|
+
a value. Bodies are validated by hand so a rejected body is never echoed back (it might hold a
|
|
5
|
+
pasted key). Connectors from connectors.json are read-only here (D16).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from datetime import datetime, timedelta
|
|
14
|
+
from typing import Annotated, Any
|
|
15
|
+
|
|
16
|
+
from fastapi import APIRouter, Body, HTTPException, Response, UploadFile
|
|
17
|
+
from fastapi.responses import FileResponse
|
|
18
|
+
from pydantic import ValidationError
|
|
19
|
+
|
|
20
|
+
from flowforge.checks import CheckTarget
|
|
21
|
+
from flowforge.connectors.models import Connector, parse_connector
|
|
22
|
+
from flowforge.connectors.presets import CUSTOM, DEFAULT_CONNECTORS, PRESETS
|
|
23
|
+
from flowforge.connectors.registry import ConnectorFileError, _describe, check_references
|
|
24
|
+
from flowforge.connectors.runtime import build_node
|
|
25
|
+
from flowforge.home import flowforge_home
|
|
26
|
+
from flowforge.nodes.base import NodeError
|
|
27
|
+
from flowforge.nodes.local_node import LocalNode
|
|
28
|
+
from flowforge.nodes.mcp_node import MCPNode
|
|
29
|
+
from flowforge.security.logfilter import secret_values_for
|
|
30
|
+
from flowforge.state import AppState, app_state
|
|
31
|
+
|
|
32
|
+
MCP_TIMEOUT_S = 15
|
|
33
|
+
MAX_LOGO_BYTES = 1024 * 1024
|
|
34
|
+
LOGO_TYPES = {"png": "image/png", "jpg": "image/jpeg", "webp": "image/webp"}
|
|
35
|
+
|
|
36
|
+
router = APIRouter()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# --- helpers ------------------------------------------------------------------------------------------
|
|
40
|
+
|
|
41
|
+
def secret_status(connector: Connector, example: bool = False) -> dict[str, str] | None:
|
|
42
|
+
""""set" or "missing", never the value. In example mode a missing key reads "example"."""
|
|
43
|
+
ref = connector.secret_ref
|
|
44
|
+
if ref is None:
|
|
45
|
+
return None
|
|
46
|
+
name = ref.split(":", 1)[1]
|
|
47
|
+
if ref.startswith("env:") and os.environ.get(name):
|
|
48
|
+
return {"ref": ref, "status": "set"}
|
|
49
|
+
return {"ref": ref, "status": "example" if example else "missing"}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def view(connector: Connector, s: AppState) -> dict[str, Any]:
|
|
53
|
+
data = connector.model_dump(mode="json")
|
|
54
|
+
data["secret"] = secret_status(connector, s.example)
|
|
55
|
+
data["managed_by"] = "file" if connector.id in s.file_managed else "app"
|
|
56
|
+
data["deletable"] = connector.id not in DEFAULT_CONNECTORS and connector.id not in s.file_managed
|
|
57
|
+
data["is_default"] = connector.id in DEFAULT_CONNECTORS
|
|
58
|
+
if connector.style.logo.type == "upload":
|
|
59
|
+
data["logo_url"] = f"/connectors/{connector.id}/logo"
|
|
60
|
+
return data
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def get_connector(connector_id: str) -> Connector:
|
|
64
|
+
connector = app_state().registry.get(connector_id)
|
|
65
|
+
if connector is None:
|
|
66
|
+
raise HTTPException(404, "unknown connector")
|
|
67
|
+
return connector
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def editable(connector_id: str) -> Connector:
|
|
71
|
+
connector = get_connector(connector_id)
|
|
72
|
+
if connector_id in app_state().file_managed:
|
|
73
|
+
raise HTTPException(409, f"'{connector_id}' is managed by connectors.json; edit that file instead")
|
|
74
|
+
return connector
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def parse_body(body: Any) -> Connector:
|
|
78
|
+
try:
|
|
79
|
+
connector = parse_connector(body)
|
|
80
|
+
except ValidationError as exc:
|
|
81
|
+
raise HTTPException(422, _describe(exc)) from None # field paths and messages only, no values
|
|
82
|
+
if connector.secret_ref and connector.secret_ref.startswith("vault:"):
|
|
83
|
+
raise HTTPException(422, "vault: references need the vault, which arrives in Phase 4; use env:NAME")
|
|
84
|
+
return connector
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def install(connector: Connector, s: AppState) -> None:
|
|
88
|
+
"""Save, then swap in a fresh node and bucket. Running runs keep their snapshot (D16)."""
|
|
89
|
+
others = [c for c in s.registry.all() if c.id != connector.id]
|
|
90
|
+
try:
|
|
91
|
+
check_references([*others, connector])
|
|
92
|
+
except ConnectorFileError as exc:
|
|
93
|
+
raise HTTPException(422, str(exc)) from None
|
|
94
|
+
s.registry.save(connector)
|
|
95
|
+
node, bucket = build_node(connector)
|
|
96
|
+
old = s.nodes.get(connector.id)
|
|
97
|
+
if old is not None:
|
|
98
|
+
s.retired_nodes.append(old)
|
|
99
|
+
s.nodes[connector.id] = node
|
|
100
|
+
if connector.id == "nim":
|
|
101
|
+
s.nodes["llm"] = node
|
|
102
|
+
if bucket is not None and node.rate_limit_key:
|
|
103
|
+
s.rate_limits[node.rate_limit_key] = bucket
|
|
104
|
+
else:
|
|
105
|
+
s.rate_limits.pop(connector.id, None)
|
|
106
|
+
refresh_redaction(s)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def refresh_redaction(s: AppState) -> None:
|
|
110
|
+
s.redactor.set_values(secret_values_for(s.registry.all()))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# --- routes -------------------------------------------------------------------------------------------
|
|
114
|
+
|
|
115
|
+
@router.get("/connectors")
|
|
116
|
+
def list_connectors() -> list[dict[str, Any]]:
|
|
117
|
+
s = app_state()
|
|
118
|
+
return [view(c, s) for c in s.registry.all()]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@router.get("/connectors/{connector_id}")
|
|
122
|
+
def read_connector(connector_id: str) -> dict[str, Any]:
|
|
123
|
+
return view(get_connector(connector_id), app_state())
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@router.post("/connectors", status_code=201)
|
|
127
|
+
def create_connector(body: Annotated[Any, Body()]) -> dict[str, Any]:
|
|
128
|
+
s = app_state()
|
|
129
|
+
connector = parse_body(body)
|
|
130
|
+
if s.registry.get(connector.id) is not None:
|
|
131
|
+
raise HTTPException(409, f"a connector called '{connector.id}' already exists")
|
|
132
|
+
install(connector, s)
|
|
133
|
+
return view(connector, s)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@router.put("/connectors/{connector_id}")
|
|
137
|
+
def update_connector(connector_id: str, body: Annotated[Any, Body()]) -> dict[str, Any]:
|
|
138
|
+
s = app_state()
|
|
139
|
+
current = editable(connector_id)
|
|
140
|
+
connector = parse_body(body)
|
|
141
|
+
if connector.id != connector_id:
|
|
142
|
+
raise HTTPException(422, "the id in the body must match the URL; ids can't be renamed")
|
|
143
|
+
if connector.style.logo.type == "upload" and current.style.logo != connector.style.logo:
|
|
144
|
+
raise HTTPException(422, "upload a logo through /connectors/{id}/logo")
|
|
145
|
+
install(connector.model_copy(update={"created_at": current.created_at}), s)
|
|
146
|
+
return view(connector, s)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@router.delete("/connectors/{connector_id}", status_code=204)
|
|
150
|
+
def delete_connector(connector_id: str) -> Response:
|
|
151
|
+
s = app_state()
|
|
152
|
+
editable(connector_id)
|
|
153
|
+
if connector_id in DEFAULT_CONNECTORS:
|
|
154
|
+
raise HTTPException(409, f"'{connector_id}' is a default connector and can't be deleted")
|
|
155
|
+
users = [c.id for c in s.registry.all() if c.fallback == connector_id]
|
|
156
|
+
if users:
|
|
157
|
+
raise HTTPException(409, f"'{connector_id}' is the fallback for {', '.join(users)}; change those first")
|
|
158
|
+
s.registry.delete(connector_id)
|
|
159
|
+
node = s.nodes.pop(connector_id, None)
|
|
160
|
+
if node is not None:
|
|
161
|
+
s.retired_nodes.append(node)
|
|
162
|
+
s.rate_limits.pop(connector_id, None)
|
|
163
|
+
refresh_redaction(s)
|
|
164
|
+
return Response(status_code=204)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@router.post("/connectors/{connector_id}/logo")
|
|
168
|
+
async def upload_logo(connector_id: str, file: UploadFile) -> dict[str, Any]:
|
|
169
|
+
"""PNG, JPEG or WebP, at most 1 MB, checked by content (CLAUDE.md §6.5). No SVG."""
|
|
170
|
+
s = app_state()
|
|
171
|
+
connector = editable(connector_id)
|
|
172
|
+
data = await file.read(MAX_LOGO_BYTES + 1)
|
|
173
|
+
if len(data) > MAX_LOGO_BYTES:
|
|
174
|
+
raise HTTPException(413, "logos must be 1 MB or smaller")
|
|
175
|
+
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
|
176
|
+
ext = "png"
|
|
177
|
+
elif data.startswith(b"\xff\xd8\xff"):
|
|
178
|
+
ext = "jpg"
|
|
179
|
+
elif data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
|
180
|
+
ext = "webp"
|
|
181
|
+
else:
|
|
182
|
+
raise HTTPException(415, "logos must be PNG, JPEG or WebP images")
|
|
183
|
+
folder = flowforge_home() / "logos"
|
|
184
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
185
|
+
for old in LOGO_TYPES:
|
|
186
|
+
(folder / f"{connector_id}.{old}").unlink(missing_ok=True)
|
|
187
|
+
(folder / f"{connector_id}.{ext}").write_bytes(data)
|
|
188
|
+
data = connector.model_dump(mode="json")
|
|
189
|
+
data["style"]["logo"] = {"type": "upload", "file": f"{connector_id}.{ext}"}
|
|
190
|
+
updated = parse_connector(data)
|
|
191
|
+
s.registry.save(updated)
|
|
192
|
+
return view(updated, s)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@router.get("/connectors/{connector_id}/logo")
|
|
196
|
+
def get_logo(connector_id: str) -> FileResponse:
|
|
197
|
+
logo = get_connector(connector_id).style.logo
|
|
198
|
+
if logo.type != "upload":
|
|
199
|
+
raise HTTPException(404, "this connector has no uploaded logo")
|
|
200
|
+
name, _, ext = logo.file.rpartition(".")
|
|
201
|
+
path = flowforge_home() / "logos" / f"{connector_id}.{ext}"
|
|
202
|
+
if name != connector_id or ext not in LOGO_TYPES or not path.is_file():
|
|
203
|
+
raise HTTPException(404, "logo file missing")
|
|
204
|
+
return FileResponse(path, media_type=LOGO_TYPES[ext],
|
|
205
|
+
headers={"X-Content-Type-Options": "nosniff", "Cache-Control": "no-cache"})
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@router.post("/connectors/{connector_id}/test")
|
|
209
|
+
async def test_connector(connector_id: str) -> dict[str, Any]:
|
|
210
|
+
"""One harmless check. MCP: list its tools. Local: the program and folder exist. LLM and HTTP: the health
|
|
211
|
+
check's one request, with no key (D24)."""
|
|
212
|
+
s = app_state()
|
|
213
|
+
connector = get_connector(connector_id)
|
|
214
|
+
node = s.nodes.get(connector_id)
|
|
215
|
+
if connector.type in ("llm", "http"): # the health check's one request, without the key (D24)
|
|
216
|
+
target = CheckTarget(id=connector.id, kind="llm" if connector.type == "llm" else "api", name=connector.name,
|
|
217
|
+
target=connector.connection.base_url, model=getattr(connector.connection, "model", None))
|
|
218
|
+
out = await s.nodes["check"].run({"target": target.model_dump(mode="json"), "root": "."})
|
|
219
|
+
result = {"ok": out["status"] in ("up", "warn"), "message": out["summary"]}
|
|
220
|
+
s.connector_tests[connector_id] = {**result, "at": datetime.now().astimezone().isoformat(timespec="seconds")}
|
|
221
|
+
return result
|
|
222
|
+
try:
|
|
223
|
+
if isinstance(node, MCPNode):
|
|
224
|
+
tools = await asyncio.wait_for(node.list_tools(), MCP_TIMEOUT_S)
|
|
225
|
+
message = f"Connected. {len(tools)} tool{'s' if len(tools) != 1 else ''} found."
|
|
226
|
+
elif isinstance(node, LocalNode):
|
|
227
|
+
node._program(node._env())
|
|
228
|
+
node._workdir(None)
|
|
229
|
+
message = "Ready. The program and its folder were found."
|
|
230
|
+
else:
|
|
231
|
+
raise HTTPException(409, "this connector has no node; restart FlowForge")
|
|
232
|
+
result = {"ok": True, "message": message}
|
|
233
|
+
except (NodeError, TimeoutError) as exc:
|
|
234
|
+
result = {"ok": False, "message": s.redactor.redact_text(str(exc) or "timed out")}
|
|
235
|
+
s.connector_tests[connector_id] = {**result, "at": datetime.now().astimezone().isoformat(timespec="seconds")}
|
|
236
|
+
return result
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def connector_steps(s: AppState, recent_runs: int = 50) -> dict[str, list[dict[str, Any]]]:
|
|
240
|
+
"""Steps from recent stored runs, grouped by the connector that answered them, newest first."""
|
|
241
|
+
by_connector: dict[str, list[dict[str, Any]]] = {}
|
|
242
|
+
for summary, result in s.storage.recent_results(limit=recent_runs):
|
|
243
|
+
started = datetime.fromisoformat(summary["started_at"])
|
|
244
|
+
titles = {o["id"]: o["title"] for o in json.loads(summary.get("steps_json") or "[]")}
|
|
245
|
+
for step in result["steps"].values():
|
|
246
|
+
connector_id = step.get("answered_by")
|
|
247
|
+
if connector_id is None or step.get("started_at") is None:
|
|
248
|
+
continue
|
|
249
|
+
offset = step.get("finished_at") or step["started_at"]
|
|
250
|
+
duration = None
|
|
251
|
+
if step.get("finished_at") is not None:
|
|
252
|
+
duration = (step["finished_at"] - step["started_at"]) * 1000
|
|
253
|
+
by_connector.setdefault(connector_id, []).append({
|
|
254
|
+
"run_id": summary["id"], "plan": summary["workflow_name"] or summary["workflow_id"],
|
|
255
|
+
"step_id": step["step_id"], "title": titles.get(step["step_id"], step["step_id"]),
|
|
256
|
+
"state": step["state"], "error": step.get("error"),
|
|
257
|
+
"cache_hit": step.get("cache_hit"), "attempts": step.get("attempts", 0), "duration_ms": duration,
|
|
258
|
+
"at": (started + timedelta(seconds=offset)).isoformat(timespec="milliseconds")})
|
|
259
|
+
for items in by_connector.values():
|
|
260
|
+
items.sort(key=lambda i: i["at"], reverse=True)
|
|
261
|
+
return by_connector
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
@router.get("/connectors/{connector_id}/activity")
|
|
265
|
+
def connector_activity(connector_id: str, limit: int = 20) -> list[dict[str, Any]]:
|
|
266
|
+
"""This connector's most recent steps, newest first (from stored results, D16)."""
|
|
267
|
+
get_connector(connector_id)
|
|
268
|
+
return connector_steps(app_state()).get(connector_id, [])[:limit]
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@router.get("/connectors/{connector_id}/tools")
|
|
272
|
+
async def connector_tools(connector_id: str) -> list[dict[str, Any]]:
|
|
273
|
+
node = app_state().nodes.get(connector_id)
|
|
274
|
+
if node is None:
|
|
275
|
+
raise HTTPException(404, "unknown connector")
|
|
276
|
+
if not isinstance(node, MCPNode):
|
|
277
|
+
raise HTTPException(422, f"'{connector_id}' is not an MCP connector")
|
|
278
|
+
try:
|
|
279
|
+
return await asyncio.wait_for(node.list_tools(), MCP_TIMEOUT_S)
|
|
280
|
+
except (NodeError, TimeoutError) as exc:
|
|
281
|
+
raise HTTPException(502, f"could not list tools: {str(exc) or type(exc).__name__}") from exc
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
@router.get("/presets")
|
|
285
|
+
def list_presets() -> list[dict[str, Any]]:
|
|
286
|
+
"""Ready-made connectors and one Custom template per type, for "Add an app"."""
|
|
287
|
+
presets = [{**c.model_dump(mode="json"), "preset": c.id} for c in PRESETS.values()]
|
|
288
|
+
custom = [{"preset": f"custom-{t}", "name": f"Custom {t}", **template} for t, template in CUSTOM.items()]
|
|
289
|
+
return presets + custom
|