flowforge-map 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowforge/__init__.py +0 -0
- flowforge/_classic.html +289 -0
- flowforge/_dashboard/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- flowforge/_dashboard/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
- flowforge/_dashboard/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- flowforge/_dashboard/assets/index-BGT5seIN.css +1 -0
- flowforge/_dashboard/assets/index-BoGikST6.js +52 -0
- flowforge/_dashboard/favicon.svg +1 -0
- flowforge/_dashboard/index.html +16 -0
- flowforge/_example/trip-planner/.env.example +3 -0
- flowforge/_example/trip-planner/.gitignore +2 -0
- flowforge/_example/trip-planner/.mcp.json +20 -0
- flowforge/_example/trip-planner/README.md +48 -0
- flowforge/_example/trip-planner/config.py +27 -0
- flowforge/_example/trip-planner/mcp_servers/currency.py +10 -0
- flowforge/_example/trip-planner/planner.py +113 -0
- flowforge/_example/trip-planner/requirements.txt +3 -0
- flowforge/_example/trip-planner/services/__init__.py +0 -0
- flowforge/_example/trip-planner/services/geo.py +49 -0
- flowforge/_example/trip-planner/services/llm.py +18 -0
- flowforge/_example/trip-planner/services/mcp_tools.py +26 -0
- flowforge/_example/trip-planner/services/weather.py +25 -0
- flowforge/_example/trip-planner/tools/packing_list.py +24 -0
- flowforge/_example/trip-planner/trips/.gitkeep +0 -0
- flowforge/_workflows/diamond_mock.json +71 -0
- flowforge/_workflows/stripe_to_razorpay.json +38 -0
- flowforge/agent/__init__.py +5 -0
- flowforge/agent/server.py +333 -0
- flowforge/api/__init__.py +0 -0
- flowforge/api/agent.py +79 -0
- flowforge/api/checks.py +255 -0
- flowforge/api/connectors.py +289 -0
- flowforge/api/flowmap.py +174 -0
- flowforge/api/runs.py +284 -0
- flowforge/api/scan.py +117 -0
- flowforge/api/security.py +31 -0
- flowforge/api/stats.py +80 -0
- flowforge/checks/__init__.py +13 -0
- flowforge/checks/models.py +41 -0
- flowforge/checks/probes.py +359 -0
- flowforge/cli.py +108 -0
- flowforge/connectors/__init__.py +0 -0
- flowforge/connectors/catalog.py +84 -0
- flowforge/connectors/models.py +149 -0
- flowforge/connectors/presets.py +192 -0
- flowforge/connectors/registry.py +85 -0
- flowforge/connectors/runtime.py +47 -0
- flowforge/connectors/secrets.py +53 -0
- flowforge/example_data.py +247 -0
- flowforge/flow/__init__.py +328 -0
- flowforge/flow/advice.py +132 -0
- flowforge/flow/curly.py +884 -0
- flowforge/flow/index.py +116 -0
- flowforge/flow/order.py +181 -0
- flowforge/flow/reach.py +145 -0
- flowforge/flow/rewrite.py +146 -0
- flowforge/flowmap/__init__.py +1 -0
- flowforge/flowmap/diff.py +145 -0
- flowforge/flowmap/fixture.py +283 -0
- flowforge/flowmap/models.py +258 -0
- flowforge/flowmap/planner.py +131 -0
- flowforge/flowmap/service.py +379 -0
- flowforge/flowmap/validate.py +228 -0
- flowforge/home.py +58 -0
- flowforge/main.py +137 -0
- flowforge/nodes/__init__.py +3 -0
- flowforge/nodes/base.py +74 -0
- flowforge/nodes/check_node.py +50 -0
- flowforge/nodes/http_node.py +161 -0
- flowforge/nodes/llm_node.py +158 -0
- flowforge/nodes/local_node.py +164 -0
- flowforge/nodes/mcp_node.py +243 -0
- flowforge/nodes/mock_node.py +40 -0
- flowforge/scan/__init__.py +6 -0
- flowforge/scan/facts.py +47 -0
- flowforge/scan/files.py +112 -0
- flowforge/scan/manifests.py +65 -0
- flowforge/scan/mcp_config.py +75 -0
- flowforge/scan/models.py +73 -0
- flowforge/scan/patterns.py +86 -0
- flowforge/scan/python_ast.py +246 -0
- flowforge/scan/scanner.py +344 -0
- flowforge/scan/services.py +58 -0
- flowforge/scan/urls.py +60 -0
- flowforge/scheduler/__init__.py +0 -0
- flowforge/scheduler/cache.py +76 -0
- flowforge/scheduler/critical_path.py +65 -0
- flowforge/scheduler/durations.py +35 -0
- flowforge/scheduler/executor.py +390 -0
- flowforge/scheduler/graph.py +108 -0
- flowforge/scheduler/rate_limit.py +36 -0
- flowforge/schema.py +81 -0
- flowforge/security/__init__.py +0 -0
- flowforge/security/finders.py +127 -0
- flowforge/security/findings.py +169 -0
- flowforge/security/logfilter.py +129 -0
- flowforge/spa.py +52 -0
- flowforge/state.py +73 -0
- flowforge/storage.py +481 -0
- flowforge/templating.py +60 -0
- flowforge_map-0.1.0.dist-info/METADATA +335 -0
- flowforge_map-0.1.0.dist-info/RECORD +113 -0
- flowforge_map-0.1.0.dist-info/WHEEL +4 -0
- flowforge_map-0.1.0.dist-info/entry_points.txt +3 -0
- flowforge_map-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Example-data mode (D16): a lively dashboard with no keys and no network.
|
|
2
|
+
|
|
3
|
+
`flowforge --example` uses its own database (FLOWFORGE_HOME/example.db). It adds example
|
|
4
|
+
connectors, swaps every connector's node for an offline ExampleNode of the same type, and seeds
|
|
5
|
+
a few runs. Seeded runs and demo runs started from the UI go through the real executor, so
|
|
6
|
+
events, critical paths, cache hits, history and savings are genuine. Only the calls are faked.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
from datetime import UTC, datetime, timedelta
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from flowforge.checks import CheckResult, Status
|
|
16
|
+
from flowforge.connectors.models import Connector, parse_connector
|
|
17
|
+
from flowforge.nodes.base import Node, NodeError, TransientNodeError
|
|
18
|
+
from flowforge.schema import Workflow
|
|
19
|
+
from flowforge.state import AppState
|
|
20
|
+
from flowforge.storage import Storage
|
|
21
|
+
|
|
22
|
+
PROJECT_NAME = "Baby-care shop"
|
|
23
|
+
SEED_SCALE = 0.1 # seeded history runs 10x faster than a demo run started from the UI
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _c(data: dict[str, Any]) -> Connector:
|
|
27
|
+
return parse_connector(data)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
CONNECTORS: list[Connector] = [_c(d) for d in (
|
|
31
|
+
{"id": "razorpay", "type": "http", "name": "Razorpay", "role": "payments", "slot": "payments", "mode": "test",
|
|
32
|
+
"style": {"color": "#3395FF", "logo": {"type": "letters", "text": "Rz"}}, "rate_limit_rpm": 60,
|
|
33
|
+
"secret_ref": "env:RAZORPAY_KEY",
|
|
34
|
+
"connection": {"base_url": "https://api.razorpay.com/v1", "auth_header": "Authorization", "auth_scheme": "basic"}},
|
|
35
|
+
{"id": "gemini", "type": "llm", "name": "Gemini", "role": "reasoning", "rate_limit_rpm": 15,
|
|
36
|
+
"style": {"color": "#4285F4", "logo": {"type": "letters", "text": "Ge"}}, "secret_ref": "env:GEMINI_API_KEY",
|
|
37
|
+
"connection": {"provider": "openai_compatible", "model": "gemini-3.8-flash",
|
|
38
|
+
"base_url": "https://generativelanguage.googleapis.com/v1beta/openai/"}},
|
|
39
|
+
{"id": "claude", "type": "llm", "name": "Claude", "role": "writes code", "fallback": "gemini",
|
|
40
|
+
"style": {"color": "#D97757", "logo": {"type": "letters", "text": "Cl"}}, "secret_ref": "env:ANTHROPIC_API_KEY",
|
|
41
|
+
"connection": {"provider": "anthropic", "model": "claude-opus-5-5"}},
|
|
42
|
+
{"id": "sirius", "type": "local", "name": "Sirius", "role": "security scan",
|
|
43
|
+
"style": {"color": "#0B7A70", "logo": {"type": "letters", "text": "Si"}},
|
|
44
|
+
"connection": {"command": ["sirius", "scan"], "cwd": "."}},
|
|
45
|
+
{"id": "stocks", "type": "mcp", "name": "Stock prices", "role": "market data",
|
|
46
|
+
"style": {"color": "#2E9E5B", "logo": {"type": "letters", "text": "Sp"}},
|
|
47
|
+
"connection": {"command": "uvx", "args": ["stock-prices-mcp"]}},
|
|
48
|
+
{"id": "charts", "type": "local", "name": "Chart script", "role": "draws charts",
|
|
49
|
+
"style": {"color": "#C2398D", "logo": {"type": "letters", "text": "Ch"}},
|
|
50
|
+
"connection": {"command": ["python", "scripts/charts.py"], "cwd": "."}},
|
|
51
|
+
# the apps the example flow map uses (D17)
|
|
52
|
+
{"id": "ollama", "type": "llm", "name": "Ollama", "role": "models on this machine",
|
|
53
|
+
"style": {"color": "#1F2328", "logo": {"type": "letters", "text": "Ol"}},
|
|
54
|
+
"connection": {"provider": "openai_compatible", "base_url": "http://localhost:11434/v1", "model": "llama3.2"}},
|
|
55
|
+
{"id": "inventory", "type": "http", "name": "Inventory API", "role": "stock levels",
|
|
56
|
+
"style": {"color": "#2F6FD6", "logo": {"type": "letters", "text": "IA"}}, "secret_ref": "env:INVENTORY_TOKEN",
|
|
57
|
+
"connection": {"base_url": "https://inventory.example.test", "auth_header": "Authorization",
|
|
58
|
+
"auth_scheme": "bearer"}},
|
|
59
|
+
{"id": "higgsfield", "type": "http", "name": "Higgsfield", "role": "order pictures",
|
|
60
|
+
"style": {"color": "#2A2A30", "logo": {"type": "letters", "text": "Hf"}}, "secret_ref": "env:HIGGSFIELD_API_KEY",
|
|
61
|
+
"connection": {"base_url": "https://higgsfield.example.test", "auth_header": "Authorization",
|
|
62
|
+
"auth_scheme": "bearer"}},
|
|
63
|
+
{"id": "gmail", "type": "mcp", "name": "Gmail", "role": "sends email",
|
|
64
|
+
"style": {"color": "#EA4335", "logo": {"type": "letters", "text": "Gm"}},
|
|
65
|
+
"connection": {"command": "npx", "args": ["gmail-mcp"]}},
|
|
66
|
+
{"id": "slack", "type": "mcp", "name": "Slack", "role": "team messages",
|
|
67
|
+
"style": {"color": "#4A154B", "logo": {"type": "letters", "text": "Sl"}},
|
|
68
|
+
"connection": {"command": "npx", "args": ["slack-mcp"]}},
|
|
69
|
+
)]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class ExampleNode(Node):
|
|
73
|
+
"""Offline stand-in for a connector: sleeps, then returns canned output (or fails)."""
|
|
74
|
+
|
|
75
|
+
def __init__(self, type: str, connector_id: str | None, rate_limit_key: str | None, fallback: str | None,
|
|
76
|
+
default_for_type: bool = False) -> None:
|
|
77
|
+
self.type = type
|
|
78
|
+
self.connector_id = connector_id
|
|
79
|
+
self.rate_limit_key = rate_limit_key
|
|
80
|
+
self.fallback = fallback
|
|
81
|
+
self.default_for_type = default_for_type
|
|
82
|
+
|
|
83
|
+
def cacheable_across_runs(self, params: dict[str, Any]) -> bool:
|
|
84
|
+
return False
|
|
85
|
+
|
|
86
|
+
async def run(self, params: dict[str, Any]) -> Any:
|
|
87
|
+
await asyncio.sleep(float(params.get("ms", 500)) / 1000)
|
|
88
|
+
if params.get("fail") == "transient":
|
|
89
|
+
raise TransientNodeError(params.get("error", "the server did not answer"))
|
|
90
|
+
if params.get("fail") == "permanent":
|
|
91
|
+
raise NodeError(params.get("error", "the request was refused"))
|
|
92
|
+
return params.get("output", {"kind": "text", "text": "done"})
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def example_nodes(state: AppState) -> None:
|
|
96
|
+
"""Replace every connector's node (and the type defaults) with an offline ExampleNode."""
|
|
97
|
+
for key, node in list(state.nodes.items()):
|
|
98
|
+
state.nodes[key] = ExampleNode(node.type, node.connector_id, node.rate_limit_key, node.fallback,
|
|
99
|
+
node.default_for_type)
|
|
100
|
+
for key in ("llm", "mcp", "http"):
|
|
101
|
+
connector = {"llm": "nim", "mcp": "fetch"}.get(key)
|
|
102
|
+
if connector and connector in state.nodes:
|
|
103
|
+
state.nodes[key] = state.nodes[connector]
|
|
104
|
+
state.nodes["check"] = ExampleCheckNode() # recorded results, offline (D24)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _llm(text: str, prompt: int, completion: int) -> dict[str, Any]:
|
|
108
|
+
return {"kind": "text", "text": text, "model": "example",
|
|
109
|
+
"usage": {"prompt_tokens": prompt, "completion_tokens": completion, "credits": 0}}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
PAYMENTS = {"kind": "table", "rows": [
|
|
113
|
+
{"payment": "pay_••••1a", "amount": "₹1,299", "status": "captured"},
|
|
114
|
+
{"payment": "pay_••••2c", "amount": "₹849", "status": "captured"},
|
|
115
|
+
{"payment": "pay_••••3e", "amount": "₹2,150", "status": "failed"},
|
|
116
|
+
]}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def workflows(scale: float = 1.0) -> dict[str, dict[str, Any]]:
|
|
120
|
+
"""The example plans, as workflow JSON. `scale` shrinks every step's duration."""
|
|
121
|
+
def ms(value: float) -> int:
|
|
122
|
+
return max(1, round(value * scale))
|
|
123
|
+
|
|
124
|
+
def step(id: str, title: str, type: str, connector: str, duration: float, deps: tuple[str, ...] = (),
|
|
125
|
+
**params: Any) -> dict[str, Any]:
|
|
126
|
+
return {"id": id, "description": title, "type": type, "connector": connector, "depends_on": list(deps),
|
|
127
|
+
"estimated_ms": duration, "retries": params.pop("retries", 0),
|
|
128
|
+
"params": {"ms": ms(duration), **params}}
|
|
129
|
+
|
|
130
|
+
return {
|
|
131
|
+
"nightly_finance_check": {"id": "nightly_finance_check", "name": "Nightly finance check", "steps": [
|
|
132
|
+
step("scan", "Scan the code", "local", "sirius", 6000,
|
|
133
|
+
output={"kind": "text", "stdout": "0 new issues, 1 medium (webhook route)", "exit_code": 0}),
|
|
134
|
+
step("payments", "Fetch today's payments", "http", "razorpay", 1000, output={"status": 200, **PAYMENTS}),
|
|
135
|
+
step("prices", "Update stock prices", "mcp", "stocks", 3000,
|
|
136
|
+
output={"content": "12 prices updated", "structured": {"updated": 12}}),
|
|
137
|
+
step("payments_again", "Fetch payments for the summary", "http", "razorpay", 1000,
|
|
138
|
+
output={"status": 200, **PAYMENTS}),
|
|
139
|
+
step("charts", "Draw the charts", "local", "charts", 2000, ("payments", "prices"),
|
|
140
|
+
output={"kind": "text", "stdout": "wrote charts/today.png", "exit_code": 0}),
|
|
141
|
+
step("summary", "Write the summary", "llm", "gemini", 3000, ("scan", "charts", "payments_again"),
|
|
142
|
+
output=_llm("Revenue ₹4,298 from 3 payments (1 failed). Stock levels steady.", 820, 220)),
|
|
143
|
+
]},
|
|
144
|
+
"payment_risk_check": {"id": "payment_risk_check", "name": "Payment risk check", "steps": [
|
|
145
|
+
step("payment", "Fetch the captured payment", "http", "razorpay", 800, output={"status": 200, **PAYMENTS}),
|
|
146
|
+
step("risk", "Rate the risk", "llm", "claude", 2200, ("payment",),
|
|
147
|
+
output=_llm("Order 1042: high risk (new card, large first order).", 300, 112)),
|
|
148
|
+
]},
|
|
149
|
+
"security_scan": {"id": "security_scan", "name": "Security scan", "steps": [
|
|
150
|
+
step("scan", "Scan the code", "local", "sirius", 6400,
|
|
151
|
+
output={"kind": "text", "stdout": "0 new issues", "exit_code": 0}),
|
|
152
|
+
]},
|
|
153
|
+
"swap_razorpay_to_stripe": {"id": "swap_razorpay_to_stripe", "name": "Swap Razorpay to Stripe", "steps": [
|
|
154
|
+
step("docs", "Fetch Stripe lifecycle", "mcp", "fetch", 1800, fail="transient", retries=2,
|
|
155
|
+
error="could not start its MCP server"),
|
|
156
|
+
step("map", "Map the endpoints", "llm", "claude", 2000, ("docs",), output=_llm("-", 0, 0)),
|
|
157
|
+
step("notes", "Write migration notes", "llm", "claude", 1500, ("map",), output=_llm("-", 0, 0)),
|
|
158
|
+
step("risk", "Write the risk report", "llm", "gemini", 1500, ("map",), output=_llm("-", 0, 0)),
|
|
159
|
+
]},
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# (workflow, how long ago it started)
|
|
164
|
+
SEED_RUNS = [
|
|
165
|
+
("swap_razorpay_to_stripe", timedelta(days=1, hours=2)),
|
|
166
|
+
("nightly_finance_check", timedelta(hours=17)),
|
|
167
|
+
("security_scan", timedelta(hours=6)),
|
|
168
|
+
("payment_risk_check", timedelta(hours=3)),
|
|
169
|
+
("nightly_finance_check", timedelta(minutes=40)),
|
|
170
|
+
]
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _recorded(id: str, status: Status, latency_ms: float | None, summary: str, **fields: str) -> CheckResult:
|
|
174
|
+
return CheckResult(id=id, status=status, latency_ms=latency_ms, summary=summary, detail={"example": True}, **fields)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# Recorded check results for the trip planner (D24): what a real check run gave, so example mode stays offline.
|
|
178
|
+
CHECK_RESULTS: dict[str, CheckResult] = {r.id: r for r in [
|
|
179
|
+
_recorded("mcp-time", "up", 1900, "Up: 2 tools"),
|
|
180
|
+
_recorded("mcp-files", "up", 2400, "Up: 14 tools"),
|
|
181
|
+
_recorded("mcp-fetch", "warn", 6800, "Up: 1 tool, but slow to start: 6.8 s"),
|
|
182
|
+
_recorded("mcp-currency", "down", None, "Stopped while starting",
|
|
183
|
+
reason="currency server: CURRENCY_API_URL is not set",
|
|
184
|
+
fix="Its code reads CURRENCY_API_URL (mcp_servers/currency.py:9), but its MCP config gives it no env."),
|
|
185
|
+
_recorded("local-packing-list", "up", None, "Found, not run"),
|
|
186
|
+
_recorded("llm-ollama", "down", None, "Nothing answers at localhost:11434",
|
|
187
|
+
reason="[Errno 111] Connection refused", fix="Start Ollama (`ollama serve`)."),
|
|
188
|
+
_recorded("api-osrm", "up", 640, "Up"),
|
|
189
|
+
_recorded("api-open-meteo", "up", 180, "Up"),
|
|
190
|
+
_recorded("api-nominatim", "up", 420, "Up"),
|
|
191
|
+
_recorded("api-ip-api-com", "warn", 210, "Up, but plain http://: not encrypted"),
|
|
192
|
+
_recorded("api-google-maps", "up", 230, "Up, needs a key (403)"),
|
|
193
|
+
_recorded("api-wikipedia", "up", 310, "Up"),
|
|
194
|
+
]}
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class ExampleCheckNode(Node):
|
|
198
|
+
"""Offline stand-in for the check node (D24): waits a little, then returns the recorded result."""
|
|
199
|
+
|
|
200
|
+
type = "check"
|
|
201
|
+
|
|
202
|
+
async def run(self, params: dict[str, Any]) -> Any:
|
|
203
|
+
target = params["target"]
|
|
204
|
+
result = CHECK_RESULTS.get(target["id"]) or _recorded(target["id"], "skipped", None,
|
|
205
|
+
"Not checked: no example result")
|
|
206
|
+
await asyncio.sleep(min(result.latency_ms or 150, 1500) / 1000)
|
|
207
|
+
return {**result.model_dump(mode="json"), "kind": target["kind"]}
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def install(state: AppState) -> None:
|
|
211
|
+
"""Add the example connectors (once) and make every node offline."""
|
|
212
|
+
from flowforge.connectors.runtime import build_node
|
|
213
|
+
|
|
214
|
+
for connector in CONNECTORS:
|
|
215
|
+
if state.registry.get(connector.id) is None:
|
|
216
|
+
state.registry.save(connector)
|
|
217
|
+
node, bucket = build_node(connector)
|
|
218
|
+
state.nodes[connector.id] = node
|
|
219
|
+
if bucket is not None and node.rate_limit_key:
|
|
220
|
+
state.rate_limits[node.rate_limit_key] = bucket
|
|
221
|
+
example_nodes(state)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def latencies(storage: Storage) -> dict[str, float]:
|
|
225
|
+
"""Flow weights in example mode (D25): the latest checks, or the recorded results before any check ran."""
|
|
226
|
+
recorded = {r.id: r.latency_ms for r in CHECK_RESULTS.values() if r.latency_ms is not None}
|
|
227
|
+
return {**recorded, **storage.latest_latencies()}
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def seed_flow(state: AppState) -> None:
|
|
231
|
+
"""The example maps (D21, D25): the trip planner's dependency map and its flow, both read from its code."""
|
|
232
|
+
for map_id in ("deps", "flow"):
|
|
233
|
+
if map_id in state.flows:
|
|
234
|
+
state.flows[map_id].first_map()
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
async def seed(state: AppState) -> list[str]:
|
|
238
|
+
"""Run the example history once, on an empty database. Returns the new run ids."""
|
|
239
|
+
from flowforge.api.runs import start_run
|
|
240
|
+
|
|
241
|
+
if state.storage.list_runs(limit=1):
|
|
242
|
+
return []
|
|
243
|
+
plans = workflows(SEED_SCALE)
|
|
244
|
+
now = datetime.now(UTC)
|
|
245
|
+
run_ids = [start_run(Workflow.model_validate(plans[name]), started_at=now - ago) for name, ago in SEED_RUNS]
|
|
246
|
+
await asyncio.gather(*(state.runs[r].task for r in run_ids if state.runs[r].task), return_exceptions=True)
|
|
247
|
+
return run_ids
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""Flow order from code (D25, D32): `flow_of(root)` reads a repo's Python (by `ast`) and Dart, JavaScript and
|
|
2
|
+
TypeScript (by patterns, `curly`) and draws the order its entry points call their dependencies, with the critical
|
|
3
|
+
path's weights and advice on what could run together.
|
|
4
|
+
|
|
5
|
+
Plain code, read-only: nothing in the repo is imported or run, nothing is sent anywhere, no LLM (D20).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import ast
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from flowforge.flow import curly
|
|
16
|
+
from flowforge.flow.advice import advise, parts_by_entry, written_ms
|
|
17
|
+
from flowforge.flow.index import Func, Index
|
|
18
|
+
from flowforge.flow.order import Body, Stmt, flatten, label, preds, reduce
|
|
19
|
+
from flowforge.flow.reach import Hit, Reach, writes
|
|
20
|
+
from flowforge.flowmap.models import Evidence, Flow, FlowEdge, FlowNode, Op
|
|
21
|
+
from flowforge.flowmap.validate import DEFAULT_MS, actions_of
|
|
22
|
+
from flowforge.scan.models import Dependency, ScanResult
|
|
23
|
+
from flowforge.scan.scanner import Places, scan_with_places
|
|
24
|
+
from flowforge.security.logfilter import Redactor, redact_code
|
|
25
|
+
|
|
26
|
+
NO_CODE = "Flow order reads Python, Dart, JavaScript and TypeScript."
|
|
27
|
+
NO_ENTRY = "No function here calls a dependency yet."
|
|
28
|
+
UNWRAP_DEPTH = 5
|
|
29
|
+
_redactor = Redactor()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class FlowResult:
|
|
34
|
+
flow: Flow | None
|
|
35
|
+
message: str | None = None
|
|
36
|
+
scan: ScanResult | None = None
|
|
37
|
+
entries: list[str] = field(default_factory=list)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _excerpt(lines: list[str], line: int) -> tuple[int, list[str]]:
|
|
41
|
+
first = max(1, line - 2)
|
|
42
|
+
return first, [redact_code(raw, _redactor)[:160] for raw in lines[first - 1: line + 2]]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _evidence(path: str, lines: list[str], line: int, text: str) -> Evidence:
|
|
46
|
+
first, excerpt = _excerpt(lines, line)
|
|
47
|
+
return Evidence(type="code", ref=f"{path}:{line}", text=text, first_line=first, lines=excerpt, highlight=line)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _makes_its_own_calls(func: Func, reach: Reach) -> bool:
|
|
51
|
+
"""Every dependency the function reaches is called inside its own lines: a helper, not a flow."""
|
|
52
|
+
lines = range(func.node.lineno, (func.node.end_lineno or func.node.lineno) + 1)
|
|
53
|
+
own = {d for (path, line), ds in reach.at.items() if path == func.path and line in lines for d in ds}
|
|
54
|
+
return {h.dep for h in reach.function(func)} <= own
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _entries(index: Index, reach: Reach, bodies: dict[str, Body]) -> list[Func]:
|
|
58
|
+
"""Functions that reach a dependency and that nothing in the repo calls. A wrapper whose one step is a call into
|
|
59
|
+
another function that works through others is unwrapped (`main` → `asyncio.run(plan())` → `plan`)."""
|
|
60
|
+
callers = index.callers()
|
|
61
|
+
roots = [f for f in index.funcs.values() if reach.function(f) and not callers[f.key]]
|
|
62
|
+
out: list[Func] = []
|
|
63
|
+
for root in sorted(roots, key=lambda f: (f.path, f.node.lineno)):
|
|
64
|
+
func = root
|
|
65
|
+
for _ in range(UNWRAP_DEPTH):
|
|
66
|
+
body = bodies.setdefault(func.key, flatten(func, index, reach))
|
|
67
|
+
steps = [s for s in body.stmts if s.hits]
|
|
68
|
+
if len(steps) != 1:
|
|
69
|
+
break
|
|
70
|
+
module = index.modules[func.module]
|
|
71
|
+
calls = [n for n in ast.walk(steps[0].node) if isinstance(n, ast.Call)]
|
|
72
|
+
inner = [t for t in {index.resolve(c, module) for c in calls} - {None} if reach.function(index.funcs[t])]
|
|
73
|
+
if len(inner) != 1 or _makes_its_own_calls(index.funcs[inner[0]], reach):
|
|
74
|
+
break
|
|
75
|
+
func = index.funcs[inner[0]]
|
|
76
|
+
if func not in out:
|
|
77
|
+
out.append(func)
|
|
78
|
+
return out
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class _Python:
|
|
82
|
+
"""What the builder asks of a Python function, from its `ast` (D25)."""
|
|
83
|
+
|
|
84
|
+
name = "python"
|
|
85
|
+
|
|
86
|
+
def __init__(self, index: Index):
|
|
87
|
+
self.index = index
|
|
88
|
+
|
|
89
|
+
def start_line(self, func: Func) -> int:
|
|
90
|
+
return func.node.lineno
|
|
91
|
+
|
|
92
|
+
def is_async(self, func: Func) -> bool:
|
|
93
|
+
return isinstance(func.node, ast.AsyncFunctionDef)
|
|
94
|
+
|
|
95
|
+
def end_line(self, func: Func, s: Stmt) -> int:
|
|
96
|
+
return getattr(s.node, "end_lineno", None) or s.line
|
|
97
|
+
|
|
98
|
+
def awaited(self, func: Func, s: Stmt) -> bool:
|
|
99
|
+
return any(isinstance(n, ast.Await) for n in ast.walk(s.node))
|
|
100
|
+
|
|
101
|
+
def label(self, func: Func, s: Stmt) -> str:
|
|
102
|
+
return label(s, self.index, func.module)
|
|
103
|
+
|
|
104
|
+
def condition(self, func: Func, s: Stmt) -> str:
|
|
105
|
+
return ast.unparse(s.node)
|
|
106
|
+
|
|
107
|
+
def question(self, func: Func, body: Body, s: Stmt) -> str:
|
|
108
|
+
return _question(func, body, s, self.index)
|
|
109
|
+
|
|
110
|
+
def keep_decision(self, func: Func, body: Body, s: Stmt) -> bool:
|
|
111
|
+
return True
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class _Builder:
|
|
115
|
+
"""Blocks and lines for every entry point of one repo, with ids unique across the map."""
|
|
116
|
+
|
|
117
|
+
def __init__(self, index: Index, reach: Reach, places: Places, deps: dict[str, Dependency],
|
|
118
|
+
sources: dict[str, str], latencies: dict[str, float]):
|
|
119
|
+
self.index, self.reach, self.places, self.deps = index, reach, places, deps
|
|
120
|
+
self.sources, self.latencies = sources, latencies
|
|
121
|
+
self.nodes: list[FlowNode] = []
|
|
122
|
+
self.edges: set[tuple[str, str]] = set()
|
|
123
|
+
self.whens: dict[tuple[str, str], str] = {}
|
|
124
|
+
self.meta: dict[str, dict[str, Any]] = {} # step id -> what the advice needs
|
|
125
|
+
self._counters: dict[str, int] = {}
|
|
126
|
+
|
|
127
|
+
def _id(self, base: str) -> str:
|
|
128
|
+
self._counters[base] = self._counters.get(base, 0) + 1
|
|
129
|
+
return f"{base}-{self._counters[base]}"[:64]
|
|
130
|
+
|
|
131
|
+
def entry(self, func: Any, body: Body, lang: Any) -> None:
|
|
132
|
+
self.lang = lang
|
|
133
|
+
lines = self.sources[func.path].splitlines()
|
|
134
|
+
start = f"start-{func.name.replace('.', '-')}"[:64]
|
|
135
|
+
self.nodes.append(FlowNode(id=start, kind="start", title=f"{func.name}()"[:80],
|
|
136
|
+
what=f"An entry point in {func.path}.",
|
|
137
|
+
evidence=[_evidence(func.path, lines, lang.start_line(func), "starts here")]))
|
|
138
|
+
decisions = {s.idx for s in body.stmts if s.is_if and (s.stops or any(
|
|
139
|
+
t.hits for t in body.stmts[s.idx + 1: s.block_end + 1])) and lang.keep_decision(func, body, s)}
|
|
140
|
+
enters: dict[int, list[str]] = {} # statement -> the blocks a line into it lands on
|
|
141
|
+
leaves: dict[int, list[str]] = {} # statement -> the blocks a line out of it leaves from
|
|
142
|
+
is_async = lang.is_async(func)
|
|
143
|
+
for s in body.stmts:
|
|
144
|
+
ev = _evidence(func.path, lines, s.line, "in your code")
|
|
145
|
+
steps = self._steps(func, lines, s, ev, start, is_async)
|
|
146
|
+
if s.idx in decisions:
|
|
147
|
+
dec = self._decision(func, lines, s, ev, lang.question(func, body, s))
|
|
148
|
+
for step in steps:
|
|
149
|
+
self.edges.add((step, dec))
|
|
150
|
+
enters[s.idx], leaves[s.idx] = (steps or [dec]), [dec]
|
|
151
|
+
elif steps:
|
|
152
|
+
enters[s.idx] = steps if s.gathered else steps[:1]
|
|
153
|
+
leaves[s.idx] = steps if s.gathered else steps[-1:]
|
|
154
|
+
self._lines(start, body, enters, leaves)
|
|
155
|
+
|
|
156
|
+
def _steps(self, func: Func, lines: list[str], s: Stmt, ev: Evidence, start: str, is_async: bool) -> list[str]:
|
|
157
|
+
"""One block per dependency the statement calls; several calls in one statement run one after another
|
|
158
|
+
unless the code gathers them."""
|
|
159
|
+
reached = {h.dep for h in s.hits}
|
|
160
|
+
ids: list[str] = []
|
|
161
|
+
for hit in s.hits:
|
|
162
|
+
dep = self.deps.get(hit.dep)
|
|
163
|
+
if dep is None or dep.via in reached:
|
|
164
|
+
continue # reached through a server this statement calls: drawn off that server below
|
|
165
|
+
node_id = self._id(dep.id)
|
|
166
|
+
measured = dep.kind != "llm" and self.latencies.get(dep.id) is not None
|
|
167
|
+
self.nodes.append(FlowNode(
|
|
168
|
+
id=node_id, kind=dep.kind, title=dep.name[:80], what=_masked(self.lang.label(func, s))[:300],
|
|
169
|
+
catalog=dep.catalog, op=_op(dep, hit), repeats=s.loop, evidence=[ev],
|
|
170
|
+
estimate_ms=float(self.latencies[dep.id]) if measured else DEFAULT_MS.get(dep.kind, 0.0),
|
|
171
|
+
estimate_source="measured" if measured else "estimate"))
|
|
172
|
+
self.meta[node_id] = {"stmt": s.idx, "entry": start, "async": is_async, "line": s.line, "ref": ev.ref,
|
|
173
|
+
"awaited": is_async and self.lang.awaited(func, s),
|
|
174
|
+
"node": s.node if self.lang.name == "python" else s, "func": func,
|
|
175
|
+
"lang": self.lang,
|
|
176
|
+
"name": dep.name, "gathered": s.gathered, "loop": s.loop, "ifs": bool(s.ifs),
|
|
177
|
+
"write": (dep.kind == "api" and hit.method not in (None, "GET", "HEAD"))
|
|
178
|
+
or (dep.kind == "mcp" and writes(hit.tool))}
|
|
179
|
+
if ids and not s.gathered:
|
|
180
|
+
self.edges.add((ids[-1], node_id))
|
|
181
|
+
ids.append(node_id)
|
|
182
|
+
self._through(func, lines, s, dep, node_id)
|
|
183
|
+
return ids
|
|
184
|
+
|
|
185
|
+
def _through(self, func: Func, lines: list[str], s: Stmt, server: Dependency, server_id: str) -> None:
|
|
186
|
+
"""What this server reaches for the app, named inside the same statement (D21's via): its own block."""
|
|
187
|
+
end = self.lang.end_line(func, s)
|
|
188
|
+
for via in self.deps.values():
|
|
189
|
+
if via.via != server.id:
|
|
190
|
+
continue
|
|
191
|
+
at = next((line for path, line, _ in self.places.by_dep.get(via.id, [])
|
|
192
|
+
if path == func.path and s.line <= line <= end), None)
|
|
193
|
+
if at is None:
|
|
194
|
+
continue
|
|
195
|
+
via_id = self._id(via.id)
|
|
196
|
+
self.nodes.append(FlowNode(id=via_id, kind=via.kind, title=via.name[:80], what=f"through {server.name}",
|
|
197
|
+
catalog=via.catalog, estimate_ms=0.0,
|
|
198
|
+
evidence=[_evidence(func.path, lines, at, f"reached through {server.name}")]))
|
|
199
|
+
self.edges.add((server_id, via_id))
|
|
200
|
+
|
|
201
|
+
def _decision(self, func: Func, lines: list[str], s: Stmt, ev: Evidence, question: str) -> str:
|
|
202
|
+
dec = f"if-{sum(1 for n in self.nodes if n.kind == 'decision') + 1}"
|
|
203
|
+
cond = _masked(self.lang.condition(func, s))
|
|
204
|
+
self.nodes.append(FlowNode(id=dec, kind="decision", title=_masked(question)[:80], condition=cond[:200],
|
|
205
|
+
what=f"if {cond}:"[:300], evidence=[ev]))
|
|
206
|
+
if s.stops:
|
|
207
|
+
stop = self._id("stop")
|
|
208
|
+
self.nodes.append(FlowNode(id=stop, kind="end", title="Stop", what="The code returns here.",
|
|
209
|
+
evidence=[_evidence(func.path, lines, s.stop_line, "returns early")]))
|
|
210
|
+
self.edges.add((dec, stop))
|
|
211
|
+
self.whens[(dec, stop)] = s.stops
|
|
212
|
+
return dec
|
|
213
|
+
|
|
214
|
+
def _lines(self, start: str, body: Body, enters: dict[int, list[str]], leaves: dict[int, list[str]]) -> None:
|
|
215
|
+
stmt_preds = preds(body)
|
|
216
|
+
memo: dict[int, set[tuple[int, str | None]]] = {}
|
|
217
|
+
|
|
218
|
+
def drawn(i: int) -> set[tuple[int, str | None]]:
|
|
219
|
+
"""The drawn statements `i` waits for, through plain code in between."""
|
|
220
|
+
if i not in memo:
|
|
221
|
+
memo[i] = set()
|
|
222
|
+
for p, lab in stmt_preds[i]:
|
|
223
|
+
memo[i] |= {(p, lab)} if p in leaves else {(q, None) for q, _ in drawn(p)}
|
|
224
|
+
return memo[i]
|
|
225
|
+
|
|
226
|
+
steps = {i for i, out in leaves.items() if not out[0].startswith("if-")}
|
|
227
|
+
for i, targets in enters.items():
|
|
228
|
+
# does it need a value another call produced? (before reduction hides it behind a decision)
|
|
229
|
+
for b in targets:
|
|
230
|
+
if b in self.meta:
|
|
231
|
+
self.meta[b]["needs_data"] = any(p in steps for p, _ in drawn(i))
|
|
232
|
+
for p, lab in drawn(i):
|
|
233
|
+
for a in leaves[p]:
|
|
234
|
+
for b in targets:
|
|
235
|
+
if a != b:
|
|
236
|
+
self.edges.add((a, b))
|
|
237
|
+
if lab and a.startswith("if-"):
|
|
238
|
+
self.whens[(a, b)] = lab
|
|
239
|
+
writes_in_order = [n for n, m in self.meta.items() if m["entry"] == start and m["write"]]
|
|
240
|
+
for a, b in zip(writes_in_order, writes_in_order[1:], strict=False):
|
|
241
|
+
self.edges.add((a, b)) # two writes keep their written order (D25)
|
|
242
|
+
has_pred = {b for _, b in self.edges}
|
|
243
|
+
for targets in enters.values():
|
|
244
|
+
for b in targets:
|
|
245
|
+
if b not in has_pred:
|
|
246
|
+
self.edges.add((start, b))
|
|
247
|
+
|
|
248
|
+
def flow(self) -> Flow:
|
|
249
|
+
kept = reduce(self.edges)
|
|
250
|
+
edges = [FlowEdge(id=f"{a}__{b}"[:64], source=a, target=b,
|
|
251
|
+
kind="branch" if (a, b) in self.whens else "flow", when=self.whens.get((a, b)))
|
|
252
|
+
for a, b in sorted(kept)]
|
|
253
|
+
flow = Flow(kind="flow", source="code", nodes=self.nodes, edges=edges)
|
|
254
|
+
modules = {m.path: m.tree for m in self.index.modules.values()}
|
|
255
|
+
flow.advice = advise(flow, self.meta, self.sources, modules)
|
|
256
|
+
flow.written_ms = written_ms(flow, self.meta)
|
|
257
|
+
flow.actions = actions_of(flow, parts_by_entry(self.meta))
|
|
258
|
+
return flow
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _masked(text: str) -> str:
|
|
262
|
+
"""Words taken from the code (a condition, a label) are masked like evidence before they are stored (D20)."""
|
|
263
|
+
return redact_code(text, _redactor)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _question(func: Func, body: Body, s: Stmt, index: Index) -> str:
|
|
267
|
+
"""A decision as a whole question, from the code (D25): what it checks, and where that came from. `if place is
|
|
268
|
+
None` after `place = … find the city …` asks 'Did "find the city" fail?'; `if sight` on an input asks 'Was a
|
|
269
|
+
sight given?'. Anything else shows its condition."""
|
|
270
|
+
node, cond = s.node, ast.unparse(s.node)
|
|
271
|
+
name, failed = None, False
|
|
272
|
+
if isinstance(node, ast.Name):
|
|
273
|
+
name = node.id
|
|
274
|
+
elif isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not) and isinstance(node.operand, ast.Name):
|
|
275
|
+
name, failed = node.operand.id, True
|
|
276
|
+
elif isinstance(node, ast.Compare) and len(node.ops) == 1 and isinstance(node.left, ast.Name) \
|
|
277
|
+
and isinstance(node.comparators[0], ast.Constant) and node.comparators[0].value is None:
|
|
278
|
+
name, failed = node.left.id, isinstance(node.ops[0], ast.Is)
|
|
279
|
+
if name is None:
|
|
280
|
+
return f"Is {cond} true?"
|
|
281
|
+
writer = next((t for t in reversed(body.stmts[: s.idx]) if name in t.writes and t.hits), None)
|
|
282
|
+
if writer is not None and (said := label(writer, index, func.module)):
|
|
283
|
+
return f'Did "{said}" {"fail" if failed else "work"}?'
|
|
284
|
+
words = name.replace("_", " ")
|
|
285
|
+
inputs = {a.arg for a in [*func.node.args.posonlyargs, *func.node.args.args, *func.node.args.kwonlyargs]}
|
|
286
|
+
if name in inputs:
|
|
287
|
+
return f"No {words} given?" if failed else f"Was a {words} given?"
|
|
288
|
+
return f"No {words}?" if failed else f"Has a {words}?"
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _op(dep: Dependency, hit: Hit) -> Op | None:
|
|
292
|
+
if dep.kind == "mcp":
|
|
293
|
+
return Op(tool=hit.tool, read_only=not writes(hit.tool))
|
|
294
|
+
if dep.kind == "api":
|
|
295
|
+
return Op(method=hit.method)
|
|
296
|
+
if dep.kind == "llm":
|
|
297
|
+
return Op(model=dep.model)
|
|
298
|
+
return None
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def flow_of(root: str | Path, latencies: dict[str, float] | None = None) -> FlowResult:
|
|
302
|
+
"""The flow of every entry point in `root`. `latencies`: dependency id → its last measured check (D24)."""
|
|
303
|
+
root = Path(root).resolve()
|
|
304
|
+
result, places = scan_with_places(root)
|
|
305
|
+
read = lambda path: (root / path).read_text(encoding="utf-8", errors="replace") # noqa: E731
|
|
306
|
+
sources = {path: read(path) for path in places.facts}
|
|
307
|
+
braced = {path: (lang, read(path)) for path, lang in places.code.items() if lang in curly.LANGUAGES}
|
|
308
|
+
if not sources and not braced:
|
|
309
|
+
return FlowResult(None, NO_CODE, result)
|
|
310
|
+
index = Index(sources)
|
|
311
|
+
reach = Reach(index, places)
|
|
312
|
+
bodies: dict[str, Body] = {}
|
|
313
|
+
entries = _entries(index, reach, bodies) if sources else []
|
|
314
|
+
c_index = curly.Index(braced)
|
|
315
|
+
c_reach = curly.Reach(c_index, places, {d.id for d in result.dependencies})
|
|
316
|
+
c_entries = curly.entries(c_index, c_reach) if braced else []
|
|
317
|
+
if not entries and not c_entries:
|
|
318
|
+
return FlowResult(None, NO_ENTRY, result)
|
|
319
|
+
all_sources = {**sources, **{path: text for path, (_, text) in braced.items()}}
|
|
320
|
+
builder = _Builder(index, reach, places, {d.id: d for d in result.dependencies}, all_sources, latencies or {})
|
|
321
|
+
python, c_lang = _Python(index), curly.Lang(c_index)
|
|
322
|
+
for func in entries:
|
|
323
|
+
builder.entry(func, bodies.setdefault(func.key, flatten(func, index, reach)), python)
|
|
324
|
+
for func in c_entries:
|
|
325
|
+
builder.entry(func, c_lang.read(func, c_reach), c_lang)
|
|
326
|
+
flow = builder.flow()
|
|
327
|
+
flow.patterns = sorted({f.language for f in c_entries})
|
|
328
|
+
return FlowResult(flow, None, result, [f.key for f in entries] + [f.key for f in c_entries])
|