flowforge-map 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowforge/__init__.py +0 -0
- flowforge/_classic.html +289 -0
- flowforge/_dashboard/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- flowforge/_dashboard/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
- flowforge/_dashboard/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- flowforge/_dashboard/assets/index-BGT5seIN.css +1 -0
- flowforge/_dashboard/assets/index-BoGikST6.js +52 -0
- flowforge/_dashboard/favicon.svg +1 -0
- flowforge/_dashboard/index.html +16 -0
- flowforge/_example/trip-planner/.env.example +3 -0
- flowforge/_example/trip-planner/.gitignore +2 -0
- flowforge/_example/trip-planner/.mcp.json +20 -0
- flowforge/_example/trip-planner/README.md +48 -0
- flowforge/_example/trip-planner/config.py +27 -0
- flowforge/_example/trip-planner/mcp_servers/currency.py +10 -0
- flowforge/_example/trip-planner/planner.py +113 -0
- flowforge/_example/trip-planner/requirements.txt +3 -0
- flowforge/_example/trip-planner/services/__init__.py +0 -0
- flowforge/_example/trip-planner/services/geo.py +49 -0
- flowforge/_example/trip-planner/services/llm.py +18 -0
- flowforge/_example/trip-planner/services/mcp_tools.py +26 -0
- flowforge/_example/trip-planner/services/weather.py +25 -0
- flowforge/_example/trip-planner/tools/packing_list.py +24 -0
- flowforge/_example/trip-planner/trips/.gitkeep +0 -0
- flowforge/_workflows/diamond_mock.json +71 -0
- flowforge/_workflows/stripe_to_razorpay.json +38 -0
- flowforge/agent/__init__.py +5 -0
- flowforge/agent/server.py +333 -0
- flowforge/api/__init__.py +0 -0
- flowforge/api/agent.py +79 -0
- flowforge/api/checks.py +255 -0
- flowforge/api/connectors.py +289 -0
- flowforge/api/flowmap.py +174 -0
- flowforge/api/runs.py +284 -0
- flowforge/api/scan.py +117 -0
- flowforge/api/security.py +31 -0
- flowforge/api/stats.py +80 -0
- flowforge/checks/__init__.py +13 -0
- flowforge/checks/models.py +41 -0
- flowforge/checks/probes.py +359 -0
- flowforge/cli.py +108 -0
- flowforge/connectors/__init__.py +0 -0
- flowforge/connectors/catalog.py +84 -0
- flowforge/connectors/models.py +149 -0
- flowforge/connectors/presets.py +192 -0
- flowforge/connectors/registry.py +85 -0
- flowforge/connectors/runtime.py +47 -0
- flowforge/connectors/secrets.py +53 -0
- flowforge/example_data.py +247 -0
- flowforge/flow/__init__.py +328 -0
- flowforge/flow/advice.py +132 -0
- flowforge/flow/curly.py +884 -0
- flowforge/flow/index.py +116 -0
- flowforge/flow/order.py +181 -0
- flowforge/flow/reach.py +145 -0
- flowforge/flow/rewrite.py +146 -0
- flowforge/flowmap/__init__.py +1 -0
- flowforge/flowmap/diff.py +145 -0
- flowforge/flowmap/fixture.py +283 -0
- flowforge/flowmap/models.py +258 -0
- flowforge/flowmap/planner.py +131 -0
- flowforge/flowmap/service.py +379 -0
- flowforge/flowmap/validate.py +228 -0
- flowforge/home.py +58 -0
- flowforge/main.py +137 -0
- flowforge/nodes/__init__.py +3 -0
- flowforge/nodes/base.py +74 -0
- flowforge/nodes/check_node.py +50 -0
- flowforge/nodes/http_node.py +161 -0
- flowforge/nodes/llm_node.py +158 -0
- flowforge/nodes/local_node.py +164 -0
- flowforge/nodes/mcp_node.py +243 -0
- flowforge/nodes/mock_node.py +40 -0
- flowforge/scan/__init__.py +6 -0
- flowforge/scan/facts.py +47 -0
- flowforge/scan/files.py +112 -0
- flowforge/scan/manifests.py +65 -0
- flowforge/scan/mcp_config.py +75 -0
- flowforge/scan/models.py +73 -0
- flowforge/scan/patterns.py +86 -0
- flowforge/scan/python_ast.py +246 -0
- flowforge/scan/scanner.py +344 -0
- flowforge/scan/services.py +58 -0
- flowforge/scan/urls.py +60 -0
- flowforge/scheduler/__init__.py +0 -0
- flowforge/scheduler/cache.py +76 -0
- flowforge/scheduler/critical_path.py +65 -0
- flowforge/scheduler/durations.py +35 -0
- flowforge/scheduler/executor.py +390 -0
- flowforge/scheduler/graph.py +108 -0
- flowforge/scheduler/rate_limit.py +36 -0
- flowforge/schema.py +81 -0
- flowforge/security/__init__.py +0 -0
- flowforge/security/finders.py +127 -0
- flowforge/security/findings.py +169 -0
- flowforge/security/logfilter.py +129 -0
- flowforge/spa.py +52 -0
- flowforge/state.py +73 -0
- flowforge/storage.py +481 -0
- flowforge/templating.py +60 -0
- flowforge_map-0.1.0.dist-info/METADATA +335 -0
- flowforge_map-0.1.0.dist-info/RECORD +113 -0
- flowforge_map-0.1.0.dist-info/WHEEL +4 -0
- flowforge_map-0.1.0.dist-info/entry_points.txt +3 -0
- flowforge_map-0.1.0.dist-info/licenses/LICENSE +21 -0
flowforge/api/flowmap.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""The maps and the connector catalog (D17, D21).
|
|
2
|
+
|
|
3
|
+
Every /flow route takes ?map=deps|flow: the dependency map from a scan (the default) or the flow map.
|
|
4
|
+
|
|
5
|
+
GET /flow?layer=mine|planner the map: blocks, lines, positions, critical path, counts
|
|
6
|
+
PUT /flow/positions save block positions (drag end; Re-tidy sends replace=true)
|
|
7
|
+
POST /flow/overrides one edit on top of the Planner's version (never the user's code)
|
|
8
|
+
DELETE /flow/overrides/{id} undo one edit
|
|
9
|
+
DELETE /flow/nodes/{node_id}/overrides back to the Planner's version for one block
|
|
10
|
+
GET /flow/trace the traced test order the map replays
|
|
11
|
+
GET /flow/analysis how the Planner drew the map (stages, log, files, calls, dropped)
|
|
12
|
+
GET /flow/versions every version, newest first
|
|
13
|
+
POST /flow/recheck a new pending version and its diff
|
|
14
|
+
GET /flow/versions/{n} a pending version's diff
|
|
15
|
+
POST /flow/versions/{n}/accept take the chosen changes; older versions stay listed
|
|
16
|
+
POST /flow/versions/{n}/discard keep the current map as it is
|
|
17
|
+
GET /catalog every app, each on_map | connected | none
|
|
18
|
+
POST /catalog/{id}/test a mock test run: offline, canned, no key
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from typing import Annotated, Any
|
|
24
|
+
|
|
25
|
+
from fastapi import APIRouter, Body, HTTPException, Query
|
|
26
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
|
27
|
+
|
|
28
|
+
from flowforge.connectors import catalog as catalog_mod
|
|
29
|
+
from flowforge.connectors.registry import _describe
|
|
30
|
+
from flowforge.flowmap.models import Override
|
|
31
|
+
from flowforge.flowmap.service import FlowService, OverrideError, VersionError
|
|
32
|
+
from flowforge.state import app_state
|
|
33
|
+
|
|
34
|
+
router = APIRouter()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
MapId = Annotated[str, Query(alias="map", pattern="^(deps|flow)$")]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def flows(map_id: str = "deps") -> FlowService:
|
|
41
|
+
service = (app_state().flows or {}).get(map_id)
|
|
42
|
+
if service is None:
|
|
43
|
+
raise HTTPException(503, "the map is not set up")
|
|
44
|
+
return service
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Positions(BaseModel):
|
|
48
|
+
model_config = ConfigDict(extra="forbid")
|
|
49
|
+
positions: dict[str, tuple[float, float]] = Field(max_length=500)
|
|
50
|
+
replace: bool = False
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Choices(BaseModel):
|
|
54
|
+
model_config = ConfigDict(extra="forbid")
|
|
55
|
+
take: dict[str, bool | str] = Field(default_factory=dict)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@router.get("/flow")
|
|
59
|
+
def get_flow(layer: str = Query("mine", pattern="^(mine|planner)$"), map_id: MapId = "deps") -> dict[str, Any]:
|
|
60
|
+
return flows(map_id).current(layer)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@router.put("/flow/positions")
|
|
64
|
+
def put_positions(body: Positions, map_id: MapId = "deps") -> dict[str, int]:
|
|
65
|
+
flows(map_id).set_positions(body.positions, replace=body.replace)
|
|
66
|
+
return {"saved": len(body.positions)}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@router.post("/flow/overrides", status_code=201)
|
|
70
|
+
def add_override(body: Annotated[Any, Body()], map_id: MapId = "deps") -> dict[str, Any]:
|
|
71
|
+
try:
|
|
72
|
+
override = Override.model_validate(body)
|
|
73
|
+
except ValidationError as exc:
|
|
74
|
+
raise HTTPException(422, _describe(exc)) from None
|
|
75
|
+
try:
|
|
76
|
+
return flows(map_id).add_override(override)
|
|
77
|
+
except OverrideError as err:
|
|
78
|
+
raise HTTPException(422, str(err)) from None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@router.delete("/flow/overrides/{override_id}", status_code=204)
|
|
82
|
+
def delete_override(override_id: int, map_id: MapId = "deps") -> None:
|
|
83
|
+
if not flows(map_id).delete_override(override_id):
|
|
84
|
+
raise HTTPException(404, "unknown edit")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@router.delete("/flow/nodes/{node_id}/overrides")
|
|
88
|
+
def reset_node(node_id: str, map_id: MapId = "deps") -> dict[str, int]:
|
|
89
|
+
return {"removed": flows(map_id).reset_node(node_id)}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@router.get("/flow/trace")
|
|
93
|
+
def get_trace(map_id: MapId = "deps") -> dict[str, Any]:
|
|
94
|
+
trace = flows(map_id).trace()
|
|
95
|
+
if trace is None:
|
|
96
|
+
raise HTTPException(404, "there is no traced test run yet")
|
|
97
|
+
return trace
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@router.get("/flow/analysis")
|
|
101
|
+
def get_analysis(map_id: MapId = "deps") -> dict[str, Any]:
|
|
102
|
+
analysis = flows(map_id).analysis()
|
|
103
|
+
if analysis is None:
|
|
104
|
+
raise HTTPException(404, "the Planner hasn't drawn a map yet")
|
|
105
|
+
return analysis
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@router.get("/flow/versions")
|
|
109
|
+
def list_versions(map_id: MapId = "deps") -> list[dict[str, Any]]:
|
|
110
|
+
return flows(map_id).versions()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@router.post("/flow/recheck")
|
|
114
|
+
def recheck(map_id: MapId = "deps") -> dict[str, Any]:
|
|
115
|
+
try:
|
|
116
|
+
return flows(map_id).recheck()
|
|
117
|
+
except VersionError as err:
|
|
118
|
+
raise HTTPException(409, str(err)) from None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@router.get("/flow/versions/{version}")
|
|
122
|
+
def pending_version(version: int, map_id: MapId = "deps") -> dict[str, Any]:
|
|
123
|
+
try:
|
|
124
|
+
return flows(map_id).pending_view(version)
|
|
125
|
+
except VersionError as err:
|
|
126
|
+
raise HTTPException(404, str(err)) from None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@router.post("/flow/versions/{version}/accept")
|
|
130
|
+
def accept_version(version: int, body: Choices, map_id: MapId = "deps") -> dict[str, Any]:
|
|
131
|
+
try:
|
|
132
|
+
return flows(map_id).accept(version, body.take)
|
|
133
|
+
except VersionError as err:
|
|
134
|
+
raise HTTPException(409, str(err)) from None
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@router.post("/flow/versions/{version}/discard", status_code=204)
|
|
138
|
+
def discard_version(version: int, map_id: MapId = "deps") -> None:
|
|
139
|
+
try:
|
|
140
|
+
flows(map_id).discard(version)
|
|
141
|
+
except VersionError as err:
|
|
142
|
+
raise HTTPException(409, str(err)) from None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# --- catalog ----------------------------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
def _connectors() -> dict[str, Any]:
|
|
148
|
+
return {c.id: c for c in app_state().registry.all()}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _on_map() -> set[str]:
|
|
152
|
+
"""Apps the code calls: confirmed blocks with evidence on either map's Planner version (D17, D21). A block
|
|
153
|
+
counts by its connected app or, on the dependency map, by the known service it was recognised as."""
|
|
154
|
+
found: set[str] = set()
|
|
155
|
+
for service in (app_state().flows or {}).values():
|
|
156
|
+
flow = service.planner_flow()
|
|
157
|
+
if flow is None:
|
|
158
|
+
continue
|
|
159
|
+
found |= {app for n in flow.nodes if n.status == "confirmed" and any(ev.confirms for ev in n.evidence)
|
|
160
|
+
for app in (n.connector, n.catalog) if app}
|
|
161
|
+
return found
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@router.get("/catalog")
|
|
165
|
+
def get_catalog() -> dict[str, Any]:
|
|
166
|
+
return catalog_mod.catalog(_connectors(), _on_map())
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@router.post("/catalog/{app_id}/test")
|
|
170
|
+
def test_app(app_id: str) -> dict[str, Any]:
|
|
171
|
+
entry = catalog_mod.find(app_id, _connectors())
|
|
172
|
+
if entry is None:
|
|
173
|
+
raise HTTPException(404, "unknown app")
|
|
174
|
+
return catalog_mod.mock_test(entry)
|
flowforge/api/runs.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
"""Runs: validate, start, list, inspect, stream, stop (D7, D12, D16).
|
|
2
|
+
|
|
3
|
+
POST /validate workflow JSON → {"ok", "order", "levels", "critical_path", "warnings"} or 422
|
|
4
|
+
POST /runs?policy=&use_cache= workflow JSON → {"run_id", "warnings"}; the run executes in the background
|
|
5
|
+
GET /runs?status=&limit=&before= run summaries, newest first
|
|
6
|
+
GET /runs/{run_id} → {"status", "error", "result", "summary"}; served from storage after a restart
|
|
7
|
+
GET /runs/{run_id}/events → Server-Sent Events, replayed from the start (live or stored)
|
|
8
|
+
POST /runs/{run_id}/stop → cancels a running run
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import json
|
|
15
|
+
import uuid
|
|
16
|
+
from collections.abc import AsyncIterator
|
|
17
|
+
from dataclasses import asdict
|
|
18
|
+
from datetime import datetime, timedelta
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from fastapi import APIRouter, HTTPException, Query
|
|
22
|
+
from fastapi.encoders import jsonable_encoder
|
|
23
|
+
from fastapi.sse import EventSourceResponse, ServerSentEvent
|
|
24
|
+
|
|
25
|
+
from flowforge.nodes.base import NodeError, TransientNodeError, current_run_id
|
|
26
|
+
from flowforge.nodes.mcp_node import MCPNode, check_call
|
|
27
|
+
from flowforge.scheduler import graph
|
|
28
|
+
from flowforge.scheduler.critical_path import critical_path
|
|
29
|
+
from flowforge.scheduler.durations import estimate_weights
|
|
30
|
+
from flowforge.scheduler.executor import POLICIES, Policy, check_nodes, run_workflow
|
|
31
|
+
from flowforge.schema import Workflow
|
|
32
|
+
from flowforge.state import Run, app_state, plain
|
|
33
|
+
from flowforge.templating import referenced_steps
|
|
34
|
+
|
|
35
|
+
MCP_CHECK_TIMEOUT_S = 5 # pre-run tool discovery; slower servers get a warning instead (D12)
|
|
36
|
+
|
|
37
|
+
router = APIRouter()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def check_graph(workflow: Workflow) -> graph.DAG:
|
|
41
|
+
try:
|
|
42
|
+
dag = graph.build_dag(workflow)
|
|
43
|
+
check_nodes(workflow, app_state().nodes) # unknown connector, type mismatch, bad fallback (D10)
|
|
44
|
+
except (graph.CycleError, ValueError) as exc:
|
|
45
|
+
raise HTTPException(422, str(exc)) from exc
|
|
46
|
+
return dag
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
async def check_mcp_calls(workflow: Workflow) -> list[str]:
|
|
50
|
+
"""Check each MCP step's tool and literal arguments against its server before a run (D12).
|
|
51
|
+
|
|
52
|
+
A wrong call → 422 with the reason. An unreachable server → a warning; the step is
|
|
53
|
+
still checked when it runs. Templated arguments are checked at run time only.
|
|
54
|
+
"""
|
|
55
|
+
nodes = app_state().nodes
|
|
56
|
+
errors: list[str] = []
|
|
57
|
+
warnings: list[str] = []
|
|
58
|
+
for step in workflow.steps:
|
|
59
|
+
node = nodes.get(step.connector or step.type)
|
|
60
|
+
tool = step.params.get("tool")
|
|
61
|
+
if step.type != "mcp" or not isinstance(node, MCPNode) or not isinstance(tool, str) or referenced_steps(tool):
|
|
62
|
+
continue
|
|
63
|
+
arguments = step.params.get("arguments", {})
|
|
64
|
+
try:
|
|
65
|
+
tools = await asyncio.wait_for(node.list_tools(step.params.get("server")), MCP_CHECK_TIMEOUT_S)
|
|
66
|
+
except (TransientNodeError, TimeoutError) as exc:
|
|
67
|
+
warnings.append(f"step '{step.id}': could not reach the MCP server to check tool '{tool}' "
|
|
68
|
+
f"({str(exc) or type(exc).__name__}); it will be checked when the step runs")
|
|
69
|
+
continue
|
|
70
|
+
except NodeError as exc:
|
|
71
|
+
errors.append(f"step '{step.id}': {exc}")
|
|
72
|
+
continue
|
|
73
|
+
problem = check_call(tools, tool, None if referenced_steps(arguments) else arguments)
|
|
74
|
+
if problem:
|
|
75
|
+
errors.append(f"step '{step.id}': {problem}")
|
|
76
|
+
if errors:
|
|
77
|
+
raise HTTPException(422, "; ".join(errors))
|
|
78
|
+
return warnings
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# --- run summaries (D16) ---------------------------------------------------------------------------
|
|
82
|
+
|
|
83
|
+
def totals(result: dict[str, Any]) -> dict[str, Any]:
|
|
84
|
+
"""Per-run numbers for history and savings, from a (redacted) RunResult dict."""
|
|
85
|
+
tokens = 0
|
|
86
|
+
credits = 0.0
|
|
87
|
+
work_ms = 0.0
|
|
88
|
+
for step in result["steps"].values():
|
|
89
|
+
if step["started_at"] is not None and step["finished_at"] is not None:
|
|
90
|
+
work_ms += (step["finished_at"] - step["started_at"]) * 1000
|
|
91
|
+
usage = step["output"].get("usage") if isinstance(step["output"], dict) else None
|
|
92
|
+
if isinstance(usage, dict) and not step["cache_hit"]:
|
|
93
|
+
tokens += int(usage.get("prompt_tokens") or 0) + int(usage.get("completion_tokens") or 0)
|
|
94
|
+
credits += float(usage.get("credits") or 0)
|
|
95
|
+
return {"makespan_ms": result["makespan_ms"], "api_calls": result["api_calls"],
|
|
96
|
+
"cache_hits": result["cache_hits"], "tokens": tokens, "credits": credits, "work_ms": work_ms}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def failure_reason(workflow: Workflow, result: dict[str, Any]) -> str | None:
|
|
100
|
+
"""The first failure in plain words, and how many later steps it skipped."""
|
|
101
|
+
steps = result["steps"]
|
|
102
|
+
failed = sorted((s for s in steps.values() if s["state"] == "failed"), key=lambda s: s["finished_at"] or 0)
|
|
103
|
+
if not failed:
|
|
104
|
+
return None
|
|
105
|
+
first = failed[0]
|
|
106
|
+
step = workflow.step(first["step_id"])
|
|
107
|
+
title = step.description or step.id
|
|
108
|
+
dag = graph.build_dag(workflow)
|
|
109
|
+
skipped = sum(1 for d in graph.descendants(dag, step.id) if steps[d]["state"] == "skipped")
|
|
110
|
+
text = f'Step "{title}" failed: {first["error"]}.'
|
|
111
|
+
if skipped:
|
|
112
|
+
text += f" {skipped} step{'s' if skipped != 1 else ''} after it {'were' if skipped != 1 else 'was'} skipped."
|
|
113
|
+
if len(failed) > 1:
|
|
114
|
+
text += f" {len(failed) - 1} other step{'s' if len(failed) > 2 else ''} also failed."
|
|
115
|
+
return text
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# a step with no connector runs on its type's default; these are the connectors behind them (D10)
|
|
119
|
+
DEFAULT_CONNECTOR = {"llm": "nim", "mcp": "fetch"}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def outline(workflow: Workflow) -> list[dict[str, Any]]:
|
|
123
|
+
"""What the dashboard needs to draw a run: no params, which could hold anything."""
|
|
124
|
+
return [{"id": s.id, "title": s.description or s.id, "type": s.type, "connector": s.connector,
|
|
125
|
+
"depends_on": list(s.depends_on)} for s in workflow.steps]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def connectors_used(steps: list[dict[str, Any]]) -> list[str]:
|
|
129
|
+
used: list[str] = []
|
|
130
|
+
for step in steps:
|
|
131
|
+
connector = step["connector"] or DEFAULT_CONNECTOR.get(step["type"])
|
|
132
|
+
if connector and connector not in used:
|
|
133
|
+
used.append(connector)
|
|
134
|
+
return used
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def summary(row: dict[str, Any], live: Run | None = None) -> dict[str, Any]:
|
|
138
|
+
status = live.status if live is not None else row["status"]
|
|
139
|
+
steps = json.loads(row["steps_json"]) if row.get("steps_json") else []
|
|
140
|
+
return {
|
|
141
|
+
"id": row["id"], "plan": row["workflow_name"] or row["workflow_id"], "workflow_id": row["workflow_id"],
|
|
142
|
+
"status": status, "policy": row["policy"], "started_at": row["started_at"],
|
|
143
|
+
"finished_at": row["finished_at"], "time_ms": row["makespan_ms"], "api_calls": row["api_calls"],
|
|
144
|
+
"cache_hits": row["cache_hits"], "tokens": row["tokens"], "credits": row["credits"],
|
|
145
|
+
"work_ms": row["work_ms"], "failure_reason": row["failure_reason"], "connectors": connectors_used(steps),
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
# --- routes ----------------------------------------------------------------------------------------
|
|
150
|
+
|
|
151
|
+
@router.post("/validate")
|
|
152
|
+
async def validate(workflow: Workflow) -> dict[str, Any]:
|
|
153
|
+
dag = check_graph(workflow)
|
|
154
|
+
warnings = await check_mcp_calls(workflow)
|
|
155
|
+
s = app_state()
|
|
156
|
+
weights = estimate_weights(workflow, s.storage.duration_history(workflow.id))
|
|
157
|
+
cp = critical_path(dag, weights)
|
|
158
|
+
return {
|
|
159
|
+
"ok": True,
|
|
160
|
+
"order": graph.topological_order(dag),
|
|
161
|
+
"levels": graph.levels(dag),
|
|
162
|
+
"weights": weights,
|
|
163
|
+
"critical_path": cp.path,
|
|
164
|
+
"critical_path_ms": cp.length_ms,
|
|
165
|
+
"warnings": warnings,
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@router.post("/runs")
|
|
170
|
+
async def create_run(workflow: Workflow, policy: Policy = "critical_path", use_cache: bool = True) -> dict[str, Any]:
|
|
171
|
+
if policy not in POLICIES:
|
|
172
|
+
raise HTTPException(422, f"policy must be one of {POLICIES}")
|
|
173
|
+
check_graph(workflow)
|
|
174
|
+
warnings = await check_mcp_calls(workflow)
|
|
175
|
+
return {"run_id": start_run(workflow, policy=policy, use_cache=use_cache), "warnings": warnings}
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def start_run(workflow: Workflow, *, policy: Policy = "critical_path", use_cache: bool = True,
|
|
179
|
+
started_at: datetime | None = None) -> str:
|
|
180
|
+
"""Start `workflow` in the background; it is recorded from its first event (D16).
|
|
181
|
+
|
|
182
|
+
`started_at` back-dates the stored run (example-data history only).
|
|
183
|
+
"""
|
|
184
|
+
s = app_state()
|
|
185
|
+
run_id = uuid.uuid4().hex[:12]
|
|
186
|
+
s.storage.start_run(run_id, workflow.id, workflow.name or workflow.id, policy,
|
|
187
|
+
started_at.isoformat(timespec="milliseconds") if started_at else None, outline(workflow))
|
|
188
|
+
run = s.runs[run_id] = Run(run_id, workflow, s.storage, s.redactor)
|
|
189
|
+
# a snapshot: editing a connector mid-run never swaps a node out from under this run (D16)
|
|
190
|
+
nodes, buckets = dict(s.nodes), dict(s.rate_limits)
|
|
191
|
+
|
|
192
|
+
def finish(status: str, result: dict[str, Any] | None, reason: str | None) -> None:
|
|
193
|
+
run.status, run.result = status, result
|
|
194
|
+
finished_at = None
|
|
195
|
+
if started_at is not None:
|
|
196
|
+
elapsed = timedelta(milliseconds=result["makespan_ms"]) if result else timedelta()
|
|
197
|
+
finished_at = (started_at + elapsed).isoformat(timespec="milliseconds")
|
|
198
|
+
s.storage.finish_run(run_id, status, result, totals(result) if result else {}, reason, finished_at)
|
|
199
|
+
|
|
200
|
+
async def execute() -> None:
|
|
201
|
+
current_run_id.set(run_id) # artifact folder for this run's large outputs (D12)
|
|
202
|
+
try:
|
|
203
|
+
raw = await run_workflow(
|
|
204
|
+
workflow, nodes, policy=policy, use_cache=use_cache,
|
|
205
|
+
rate_limits=buckets, storage=s.storage, on_event=run.push,
|
|
206
|
+
)
|
|
207
|
+
result = plain(asdict(raw), s.redactor)
|
|
208
|
+
finish(raw.status, result, failure_reason(workflow, result))
|
|
209
|
+
except asyncio.CancelledError:
|
|
210
|
+
status = "interrupted" if s.shutting_down else "stopped"
|
|
211
|
+
reason = "The server stopped while this was running." if s.shutting_down else "Stopped by you."
|
|
212
|
+
run.push({"type": "run", "state": status})
|
|
213
|
+
finish(status, None, reason)
|
|
214
|
+
except Exception as exc: # validation passed, so this is an engine bug — surface it
|
|
215
|
+
run.error = s.redactor.redact_text(f"{type(exc).__name__}: {exc}")
|
|
216
|
+
run.push({"type": "run", "state": "error", "error": run.error})
|
|
217
|
+
finish("error", None, run.error)
|
|
218
|
+
finally:
|
|
219
|
+
run.done = True
|
|
220
|
+
run.changed.set()
|
|
221
|
+
|
|
222
|
+
run.task = asyncio.create_task(execute())
|
|
223
|
+
s.tasks.add(run.task)
|
|
224
|
+
run.task.add_done_callback(s.tasks.discard)
|
|
225
|
+
return run_id
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@router.get("/runs")
|
|
229
|
+
def list_runs(status: str | None = None, limit: int = Query(50, ge=1, le=500),
|
|
230
|
+
before: str | None = None) -> list[dict[str, Any]]:
|
|
231
|
+
s = app_state()
|
|
232
|
+
return [summary(row, s.runs.get(row["id"])) for row in s.storage.list_runs(status, limit, before)]
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
@router.get("/runs/{run_id}")
|
|
236
|
+
def run_status(run_id: str) -> dict[str, Any]:
|
|
237
|
+
s = app_state()
|
|
238
|
+
row = s.storage.get_run(run_id)
|
|
239
|
+
if row is None:
|
|
240
|
+
raise HTTPException(404, "unknown run")
|
|
241
|
+
live = s.runs.get(run_id)
|
|
242
|
+
steps = json.loads(row["steps_json"]) if row.get("steps_json") else []
|
|
243
|
+
if live is not None:
|
|
244
|
+
return {"status": live.status, "error": live.error, "result": live.result, "summary": summary(row, live),
|
|
245
|
+
"steps": steps}
|
|
246
|
+
return {"status": row["status"], "error": row["failure_reason"] if row["status"] == "error" else None,
|
|
247
|
+
"result": row["result"], "summary": summary(row), "steps": steps}
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@router.get("/runs/{run_id}/events", response_class=EventSourceResponse)
|
|
251
|
+
async def run_events(run_id: str) -> AsyncIterator[ServerSentEvent]:
|
|
252
|
+
s = app_state()
|
|
253
|
+
run = s.runs.get(run_id)
|
|
254
|
+
if run is None: # finished before this server started: replay what was stored
|
|
255
|
+
if s.storage.get_run(run_id) is None:
|
|
256
|
+
raise HTTPException(404, "unknown run")
|
|
257
|
+
for seq, event in enumerate(s.storage.run_events(run_id)):
|
|
258
|
+
yield ServerSentEvent(data=jsonable_encoder(event), id=str(seq))
|
|
259
|
+
yield ServerSentEvent(data={"type": "end"}, event="end")
|
|
260
|
+
return
|
|
261
|
+
sent = 0
|
|
262
|
+
while True:
|
|
263
|
+
run.changed.clear()
|
|
264
|
+
while sent < len(run.events):
|
|
265
|
+
yield ServerSentEvent(data=jsonable_encoder(run.events[sent]), id=str(sent))
|
|
266
|
+
sent += 1
|
|
267
|
+
if run.done:
|
|
268
|
+
yield ServerSentEvent(data={"type": "end"}, event="end")
|
|
269
|
+
return
|
|
270
|
+
await run.changed.wait()
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
@router.post("/runs/{run_id}/stop")
|
|
274
|
+
def stop_run(run_id: str) -> dict[str, str]:
|
|
275
|
+
s = app_state()
|
|
276
|
+
run = s.runs.get(run_id)
|
|
277
|
+
if run is None:
|
|
278
|
+
if s.storage.get_run(run_id) is None:
|
|
279
|
+
raise HTTPException(404, "unknown run")
|
|
280
|
+
raise HTTPException(409, "this run is not running")
|
|
281
|
+
if run.done or run.task is None:
|
|
282
|
+
raise HTTPException(409, "this run is not running")
|
|
283
|
+
run.task.cancel()
|
|
284
|
+
return {"status": "stopping"}
|
flowforge/api/scan.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Choose the folder whose dependency map you see (D21). One repo per database in V1.
|
|
2
|
+
|
|
3
|
+
GET /scan the folder, whether this is example mode, and a summary of the last scan
|
|
4
|
+
POST /scan {path, replace?} scan a folder and draw its map; a different folder needs replace (a fresh map)
|
|
5
|
+
|
|
6
|
+
Re-scanning the same folder is POST /flow/recheck: a new version and a diff, never an overwrite (D17).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from fastapi import APIRouter, HTTPException
|
|
17
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
18
|
+
|
|
19
|
+
from flowforge.flowmap.planner import CodeFlowPlanner, ScanPlanner
|
|
20
|
+
from flowforge.flowmap.service import FlowService
|
|
21
|
+
from flowforge.home import resource
|
|
22
|
+
from flowforge.state import AppState, app_state
|
|
23
|
+
from flowforge.storage import Storage
|
|
24
|
+
|
|
25
|
+
REPO_SETTING = "repo_path"
|
|
26
|
+
EXAMPLE_REPO = resource("example") # scanned live in example mode; shipped inside the package (D34)
|
|
27
|
+
|
|
28
|
+
router = APIRouter()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ScanRequest(BaseModel):
|
|
32
|
+
model_config = ConfigDict(extra="forbid")
|
|
33
|
+
path: str = Field(min_length=1, max_length=4096)
|
|
34
|
+
replace: bool = False
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class RepoChangeError(ValueError):
|
|
38
|
+
"""A different folder while a map exists: it needs an explicit replace (D21)."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def map_repo(storage: Storage, maps: dict[str, FlowService], path: Path, fresh: bool) -> None:
|
|
42
|
+
"""Both maps of one folder: what it depends on (D21) and the order its code calls them (D25), weighted by the
|
|
43
|
+
latest checks (D24)."""
|
|
44
|
+
maps["deps"].map_folder(ScanPlanner(path), fresh=fresh)
|
|
45
|
+
if "flow" in maps:
|
|
46
|
+
maps["flow"].map_folder(CodeFlowPlanner(path, storage.latest_latencies), fresh=fresh)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def choose_repo(storage: Storage, maps: dict[str, FlowService], path: Path, replace: bool) -> None:
|
|
50
|
+
"""Map `path`: the same folder keeps its maps; a different one starts fresh maps, but only with `replace`."""
|
|
51
|
+
current = storage.setting(REPO_SETTING)
|
|
52
|
+
changed = current != str(path)
|
|
53
|
+
if changed and current and maps["deps"].has_map() and not replace:
|
|
54
|
+
raise RepoChangeError(f"there is already a map of {Path(current).name}; mapping {path.name} starts a fresh map")
|
|
55
|
+
if changed:
|
|
56
|
+
storage.set_setting(REPO_SETTING, str(path))
|
|
57
|
+
if current:
|
|
58
|
+
storage.clear_checks() # another repo's results don't belong to this one (D24)
|
|
59
|
+
map_repo(storage, maps, path, fresh=changed)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def startup_repo(storage: Storage, maps: dict[str, FlowService]) -> None:
|
|
63
|
+
"""At start-up outside example mode: FLOWFORGE_REPO (`flowforge --repo`), else the folder chosen last time.
|
|
64
|
+
A different folder needs FLOWFORGE_REPO_REPLACE=1 (`--replace`); otherwise FlowForge refuses to start."""
|
|
65
|
+
chosen = os.getenv("FLOWFORGE_REPO")
|
|
66
|
+
if chosen:
|
|
67
|
+
path = Path(chosen).expanduser()
|
|
68
|
+
if not path.is_dir():
|
|
69
|
+
raise RuntimeError(f"FLOWFORGE_REPO={chosen} is not a folder")
|
|
70
|
+
try:
|
|
71
|
+
choose_repo(storage, maps, path.resolve(), replace=os.getenv("FLOWFORGE_REPO_REPLACE") == "1")
|
|
72
|
+
except RepoChangeError as err:
|
|
73
|
+
raise RuntimeError(f"{err}. Run with --replace to start it fresh.") from None
|
|
74
|
+
return
|
|
75
|
+
saved = storage.setting(REPO_SETTING)
|
|
76
|
+
if saved and Path(saved).is_dir():
|
|
77
|
+
map_repo(storage, maps, Path(saved), fresh=False)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def repo_path(s: AppState) -> Path | None:
|
|
81
|
+
if s.example:
|
|
82
|
+
return EXAMPLE_REPO
|
|
83
|
+
saved = s.storage.setting(REPO_SETTING)
|
|
84
|
+
return Path(saved) if saved else None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@router.get("/scan")
|
|
88
|
+
def get_scan() -> dict[str, Any]:
|
|
89
|
+
s = app_state()
|
|
90
|
+
repo = repo_path(s)
|
|
91
|
+
analysis = s.flows["deps"].analysis() if "deps" in s.flows else None
|
|
92
|
+
summary = None
|
|
93
|
+
if analysis and analysis.get("scan"):
|
|
94
|
+
deps = analysis["scan"]["dependencies"]
|
|
95
|
+
summary = {"files": analysis["scan"]["files_scanned"], "dependencies": len(deps),
|
|
96
|
+
"unconfirmed": sum(d["status"] == "unconfirmed" for d in deps),
|
|
97
|
+
"kinds": dict(Counter(d["kind"] for d in deps))}
|
|
98
|
+
return {"repo": str(repo) if repo else None, "name": repo.name if repo else None, "example": s.example,
|
|
99
|
+
"summary": summary}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@router.post("/scan", status_code=201)
|
|
103
|
+
def post_scan(body: ScanRequest) -> dict[str, Any]:
|
|
104
|
+
"""Read-only and offline (D20): the folder is read, never run, and nothing is sent anywhere."""
|
|
105
|
+
s = app_state()
|
|
106
|
+
if s.example:
|
|
107
|
+
raise HTTPException(409, "Example mode always maps the sample app. Run FlowForge without --example "
|
|
108
|
+
"to scan your own folder.")
|
|
109
|
+
path = Path(body.path).expanduser()
|
|
110
|
+
if not path.is_dir():
|
|
111
|
+
raise HTTPException(422, "that isn't a folder on this machine")
|
|
112
|
+
path = path.resolve()
|
|
113
|
+
try:
|
|
114
|
+
choose_repo(s.storage, s.flows, path, body.replace)
|
|
115
|
+
except RepoChangeError as err:
|
|
116
|
+
raise HTTPException(409, f"{err} (send replace: true)") from None
|
|
117
|
+
return {"repo": str(path), "name": path.name, "version": s.flows["deps"].current().get("version")}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Security findings (D31): what the map's scan found unsafe, from the stored scan, so a re-scan updates it. A scan
|
|
2
|
+
stored before D31 never ran the finders, so the folder is read again (plain code, nothing run or sent: D20).
|
|
3
|
+
|
|
4
|
+
GET /security {repo, scanned, summary: {high, medium, low}, findings: [...]}
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections import Counter
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from fastapi import APIRouter
|
|
13
|
+
|
|
14
|
+
from flowforge.api.scan import repo_path
|
|
15
|
+
from flowforge.scan import scan as scan_repo
|
|
16
|
+
from flowforge.state import app_state
|
|
17
|
+
|
|
18
|
+
router = APIRouter()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@router.get("/security")
|
|
22
|
+
def security() -> dict[str, Any]:
|
|
23
|
+
s = app_state()
|
|
24
|
+
repo = repo_path(s)
|
|
25
|
+
analysis = s.flows["deps"].analysis() if "deps" in s.flows else None
|
|
26
|
+
scan = (analysis or {}).get("scan")
|
|
27
|
+
if scan and not scan.get("findings_checked") and repo is not None and repo.is_dir():
|
|
28
|
+
scan = scan_repo(repo).model_dump(mode="json")
|
|
29
|
+
findings = scan.get("findings", []) if scan else []
|
|
30
|
+
return {"repo": repo.name if repo else None, "scanned": scan is not None,
|
|
31
|
+
"summary": dict(Counter(f["level"] for f in findings)), "findings": findings}
|