flowforge-map 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowforge/__init__.py +0 -0
- flowforge/_classic.html +289 -0
- flowforge/_dashboard/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- flowforge/_dashboard/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
- flowforge/_dashboard/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- flowforge/_dashboard/assets/index-BGT5seIN.css +1 -0
- flowforge/_dashboard/assets/index-BoGikST6.js +52 -0
- flowforge/_dashboard/favicon.svg +1 -0
- flowforge/_dashboard/index.html +16 -0
- flowforge/_example/trip-planner/.env.example +3 -0
- flowforge/_example/trip-planner/.gitignore +2 -0
- flowforge/_example/trip-planner/.mcp.json +20 -0
- flowforge/_example/trip-planner/README.md +48 -0
- flowforge/_example/trip-planner/config.py +27 -0
- flowforge/_example/trip-planner/mcp_servers/currency.py +10 -0
- flowforge/_example/trip-planner/planner.py +113 -0
- flowforge/_example/trip-planner/requirements.txt +3 -0
- flowforge/_example/trip-planner/services/__init__.py +0 -0
- flowforge/_example/trip-planner/services/geo.py +49 -0
- flowforge/_example/trip-planner/services/llm.py +18 -0
- flowforge/_example/trip-planner/services/mcp_tools.py +26 -0
- flowforge/_example/trip-planner/services/weather.py +25 -0
- flowforge/_example/trip-planner/tools/packing_list.py +24 -0
- flowforge/_example/trip-planner/trips/.gitkeep +0 -0
- flowforge/_workflows/diamond_mock.json +71 -0
- flowforge/_workflows/stripe_to_razorpay.json +38 -0
- flowforge/agent/__init__.py +5 -0
- flowforge/agent/server.py +333 -0
- flowforge/api/__init__.py +0 -0
- flowforge/api/agent.py +79 -0
- flowforge/api/checks.py +255 -0
- flowforge/api/connectors.py +289 -0
- flowforge/api/flowmap.py +174 -0
- flowforge/api/runs.py +284 -0
- flowforge/api/scan.py +117 -0
- flowforge/api/security.py +31 -0
- flowforge/api/stats.py +80 -0
- flowforge/checks/__init__.py +13 -0
- flowforge/checks/models.py +41 -0
- flowforge/checks/probes.py +359 -0
- flowforge/cli.py +108 -0
- flowforge/connectors/__init__.py +0 -0
- flowforge/connectors/catalog.py +84 -0
- flowforge/connectors/models.py +149 -0
- flowforge/connectors/presets.py +192 -0
- flowforge/connectors/registry.py +85 -0
- flowforge/connectors/runtime.py +47 -0
- flowforge/connectors/secrets.py +53 -0
- flowforge/example_data.py +247 -0
- flowforge/flow/__init__.py +328 -0
- flowforge/flow/advice.py +132 -0
- flowforge/flow/curly.py +884 -0
- flowforge/flow/index.py +116 -0
- flowforge/flow/order.py +181 -0
- flowforge/flow/reach.py +145 -0
- flowforge/flow/rewrite.py +146 -0
- flowforge/flowmap/__init__.py +1 -0
- flowforge/flowmap/diff.py +145 -0
- flowforge/flowmap/fixture.py +283 -0
- flowforge/flowmap/models.py +258 -0
- flowforge/flowmap/planner.py +131 -0
- flowforge/flowmap/service.py +379 -0
- flowforge/flowmap/validate.py +228 -0
- flowforge/home.py +58 -0
- flowforge/main.py +137 -0
- flowforge/nodes/__init__.py +3 -0
- flowforge/nodes/base.py +74 -0
- flowforge/nodes/check_node.py +50 -0
- flowforge/nodes/http_node.py +161 -0
- flowforge/nodes/llm_node.py +158 -0
- flowforge/nodes/local_node.py +164 -0
- flowforge/nodes/mcp_node.py +243 -0
- flowforge/nodes/mock_node.py +40 -0
- flowforge/scan/__init__.py +6 -0
- flowforge/scan/facts.py +47 -0
- flowforge/scan/files.py +112 -0
- flowforge/scan/manifests.py +65 -0
- flowforge/scan/mcp_config.py +75 -0
- flowforge/scan/models.py +73 -0
- flowforge/scan/patterns.py +86 -0
- flowforge/scan/python_ast.py +246 -0
- flowforge/scan/scanner.py +344 -0
- flowforge/scan/services.py +58 -0
- flowforge/scan/urls.py +60 -0
- flowforge/scheduler/__init__.py +0 -0
- flowforge/scheduler/cache.py +76 -0
- flowforge/scheduler/critical_path.py +65 -0
- flowforge/scheduler/durations.py +35 -0
- flowforge/scheduler/executor.py +390 -0
- flowforge/scheduler/graph.py +108 -0
- flowforge/scheduler/rate_limit.py +36 -0
- flowforge/schema.py +81 -0
- flowforge/security/__init__.py +0 -0
- flowforge/security/finders.py +127 -0
- flowforge/security/findings.py +169 -0
- flowforge/security/logfilter.py +129 -0
- flowforge/spa.py +52 -0
- flowforge/state.py +73 -0
- flowforge/storage.py +481 -0
- flowforge/templating.py +60 -0
- flowforge_map-0.1.0.dist-info/METADATA +335 -0
- flowforge_map-0.1.0.dist-info/RECORD +113 -0
- flowforge_map-0.1.0.dist-info/WHEEL +4 -0
- flowforge_map-0.1.0.dist-info/entry_points.txt +3 -0
- flowforge_map-0.1.0.dist-info/licenses/LICENSE +21 -0
flowforge/api/stats.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Tool health, money and time, and app meta for the dashboard (D16).
|
|
2
|
+
|
|
3
|
+
GET /health/tools → per connector: status, plain-words label, last OK, last failure, rate left
|
|
4
|
+
GET /stats/savings → time saved vs one at a time, calls skipped by cache, API calls, tokens, credits
|
|
5
|
+
GET /meta → project name, example-data flag, version
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from datetime import UTC, datetime, timedelta
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from fastapi import APIRouter
|
|
15
|
+
|
|
16
|
+
from flowforge import example_data, home
|
|
17
|
+
from flowforge.api.connectors import connector_steps, secret_status
|
|
18
|
+
from flowforge.state import app_state
|
|
19
|
+
|
|
20
|
+
router = APIRouter()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def ago(iso: str, now: datetime) -> str:
|
|
24
|
+
seconds = max(0, (now - datetime.fromisoformat(iso)).total_seconds())
|
|
25
|
+
if seconds < 60:
|
|
26
|
+
return "just now"
|
|
27
|
+
if seconds < 3600:
|
|
28
|
+
return f"{int(seconds // 60)} min ago"
|
|
29
|
+
if seconds < 86400:
|
|
30
|
+
return f"{int(seconds // 3600)} h ago"
|
|
31
|
+
return f"{int(seconds // 86400)} d ago"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@router.get("/health/tools")
|
|
35
|
+
def tool_health() -> list[dict[str, Any]]:
|
|
36
|
+
s = app_state()
|
|
37
|
+
now = datetime.now(UTC)
|
|
38
|
+
steps = connector_steps(s)
|
|
39
|
+
out: list[dict[str, Any]] = []
|
|
40
|
+
for connector in s.registry.all():
|
|
41
|
+
items = steps.get(connector.id, [])
|
|
42
|
+
last_ok = next((i for i in items if i["state"] == "succeeded"), None)
|
|
43
|
+
last_fail = next((i for i in items if i["state"] == "failed"), None)
|
|
44
|
+
secret = secret_status(connector, s.example)
|
|
45
|
+
if secret is not None and secret["status"] == "missing":
|
|
46
|
+
status, label = "missing_key", "Key missing"
|
|
47
|
+
elif last_fail and (not last_ok or last_fail["at"] > last_ok["at"]):
|
|
48
|
+
status, label = "failing", f"Failed {ago(last_fail['at'], now)}"
|
|
49
|
+
elif last_ok:
|
|
50
|
+
status, label = "ok", f"OK, {ago(last_ok['at'], now)}"
|
|
51
|
+
else:
|
|
52
|
+
status, label = "idle", "Not used yet"
|
|
53
|
+
rate = None
|
|
54
|
+
if connector.rate_limit_rpm:
|
|
55
|
+
cutoff = (now - timedelta(minutes=1)).isoformat()
|
|
56
|
+
used = sum(i["attempts"] for i in items if i["at"] >= cutoff and not i["cache_hit"])
|
|
57
|
+
rpm = connector.rate_limit_rpm
|
|
58
|
+
rate = {"rpm": int(rpm) if float(rpm).is_integer() else rpm, "used_last_minute": used,
|
|
59
|
+
"left": max(0, int(rpm - used))}
|
|
60
|
+
out.append({
|
|
61
|
+
"id": connector.id, "name": connector.name, "type": connector.type, "status": status, "label": label,
|
|
62
|
+
"last_ok": last_ok["at"] if last_ok else None,
|
|
63
|
+
"last_failure": last_fail["at"] if last_fail else None,
|
|
64
|
+
"last_error": last_fail["error"] if last_fail else None,
|
|
65
|
+
"rate": rate, "last_test": s.connector_tests.get(connector.id),
|
|
66
|
+
})
|
|
67
|
+
return out
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@router.get("/stats/savings")
|
|
71
|
+
def savings() -> dict[str, Any]:
|
|
72
|
+
return app_state().storage.savings()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@router.get("/meta")
|
|
76
|
+
def meta() -> dict[str, Any]:
|
|
77
|
+
app_version = home.version() # the installed package's, under its PyPI name (D34)
|
|
78
|
+
example = app_state().example
|
|
79
|
+
default = example_data.PROJECT_NAME if example else "My project"
|
|
80
|
+
return {"project": os.getenv("FLOWFORGE_PROJECT_NAME") or default, "example": example, "version": app_version}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Health checks (D24): one harmless check per dependency, with no key and nothing kept but the verdict.
|
|
2
|
+
|
|
3
|
+
`check(target, root)` is the whole interface: an API gets one GET, an LLM its model list, an MCP server is started
|
|
4
|
+
from the repo folder and asked for its tools, and a local program is only looked for. Nothing here reads `.env`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from flowforge.checks import probes
|
|
10
|
+
from flowforge.checks.models import CheckResult, CheckTarget, Status
|
|
11
|
+
from flowforge.checks.probes import check, new_client, starts_process
|
|
12
|
+
|
|
13
|
+
__all__ = ["CheckResult", "CheckTarget", "Status", "check", "new_client", "probes", "starts_process"]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""What a check takes and gives back (D24). Names, addresses and commands only: never a key or a response body."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Literal
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
8
|
+
|
|
9
|
+
from flowforge.scan.models import DependencyKind, EnvName
|
|
10
|
+
|
|
11
|
+
Status = Literal["up", "warn", "down", "skipped"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class _Model(BaseModel):
|
|
15
|
+
model_config = ConfigDict(extra="forbid")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class CheckTarget(_Model):
|
|
19
|
+
"""One dependency to check: from the scan (D21), or a block the user added to the map."""
|
|
20
|
+
|
|
21
|
+
id: str
|
|
22
|
+
kind: DependencyKind
|
|
23
|
+
name: str
|
|
24
|
+
target: str | None = None # a URL without its query, or host:port
|
|
25
|
+
command: list[str] | None = None # mcp and local: the program and its (redacted) arguments
|
|
26
|
+
model: str | None = None
|
|
27
|
+
catalog: str | None = None
|
|
28
|
+
env_names: list[str] = Field(default_factory=list) # env names its MCP config gives it
|
|
29
|
+
reads_env: list[EnvName] = Field(default_factory=list) # env names its own script reads, and where (scan)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class CheckResult(_Model):
|
|
33
|
+
"""One dependency's verdict (D24): up, warn, down, or skipped with the reason nothing was checked."""
|
|
34
|
+
|
|
35
|
+
id: str
|
|
36
|
+
status: Status
|
|
37
|
+
latency_ms: float | None = None
|
|
38
|
+
summary: str # one short sentence, shown on the map
|
|
39
|
+
reason: str | None = None # the error in the dependency's own words, redacted
|
|
40
|
+
fix: str | None = None # a likely fix, when the code shows one
|
|
41
|
+
detail: dict[str, Any] = Field(default_factory=dict) # http status, tool count, model listed…
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
"""One probe per kind of dependency (D24). Thresholds are FlowForge's defaults, named here so they can be tuned."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import shutil
|
|
9
|
+
import socket
|
|
10
|
+
import ssl
|
|
11
|
+
import tempfile
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from datetime import UTC, datetime
|
|
15
|
+
from http.cookiejar import CookieJar
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
from urllib.parse import urlsplit
|
|
19
|
+
|
|
20
|
+
import httpx
|
|
21
|
+
from mcp import ClientSession, StdioServerParameters
|
|
22
|
+
from mcp.client.stdio import get_default_environment, stdio_client
|
|
23
|
+
|
|
24
|
+
from flowforge.checks.models import CheckResult, CheckTarget, Status
|
|
25
|
+
from flowforge.connectors.presets import MODEL_LISTS
|
|
26
|
+
from flowforge.scan.urls import host_of, is_local
|
|
27
|
+
from flowforge.security.logfilter import REDACTED, Redactor
|
|
28
|
+
|
|
29
|
+
SLOW_MS = 2000 # an API or LLM slower than this needs a look
|
|
30
|
+
HTTP_TIMEOUT_S = 10
|
|
31
|
+
CERT_WARN_DAYS = 14
|
|
32
|
+
MCP_SLOW_MS = 5000 # an MCP server slower than this to list its tools needs a look
|
|
33
|
+
MCP_TIMEOUT_S: float = 30
|
|
34
|
+
STDERR_LINES = 3
|
|
35
|
+
KNOCK_TIMEOUT_S = 5 # when a request can't connect: how long each address gets to accept a connection
|
|
36
|
+
MAX_ADDRESSES = 4
|
|
37
|
+
USER_AGENT = "FlowForge health check (one request, no keys)"
|
|
38
|
+
_redactor = Redactor()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class _NoCookies(CookieJar):
|
|
42
|
+
"""A jar that keeps nothing: a cookie a service sets is never sent back on a later check (D24)."""
|
|
43
|
+
|
|
44
|
+
def set_cookie(self, cookie: Any) -> None:
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
def extract_cookies(self, response: Any, request: Any) -> None:
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def new_client(transport: httpx.AsyncBaseTransport | None = None) -> httpx.AsyncClient:
|
|
52
|
+
"""The HTTP client checks share: no cookies kept, no redirects followed, nothing but our User-Agent."""
|
|
53
|
+
return httpx.AsyncClient(transport=transport, cookies=_NoCookies())
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def is_remote(command: list[str]) -> bool:
|
|
57
|
+
return bool(command) and command[0].startswith(("http://", "https://"))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def starts_process(target: CheckTarget) -> bool:
|
|
61
|
+
"""Checking this one starts a process: a stdio MCP server whose arguments are all known (D24)."""
|
|
62
|
+
command = target.command or []
|
|
63
|
+
return (target.kind == "mcp" and bool(command) and not is_remote(command)
|
|
64
|
+
and not any(REDACTED in part for part in command))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def took(ms: float) -> str:
|
|
68
|
+
return f"{ms:.0f} ms" if ms < 1000 else f"{ms / 1000:.1f} s"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def cert_warning(not_after: str, now: datetime) -> str | None:
|
|
72
|
+
"""A certificate's `notAfter` (as `ssl` reports it) turned into a warning when it is close (D24)."""
|
|
73
|
+
expires = datetime.fromtimestamp(ssl.cert_time_to_seconds(not_after), UTC)
|
|
74
|
+
days = (expires - now).days
|
|
75
|
+
if days < 0:
|
|
76
|
+
return "its certificate has expired"
|
|
77
|
+
if days < CERT_WARN_DAYS:
|
|
78
|
+
return f"its certificate expires in {days} day{'s' if days != 1 else ''}"
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _not_after(response: httpx.Response) -> str | None:
|
|
83
|
+
stream = response.extensions.get("network_stream")
|
|
84
|
+
ssl_object = stream.get_extra_info("ssl_object") if stream is not None else None
|
|
85
|
+
cert = ssl_object.getpeercert() if ssl_object is not None else None
|
|
86
|
+
return cert.get("notAfter") if cert else None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _result(t: CheckTarget, status: Status, summary: str, **fields: Any) -> CheckResult:
|
|
90
|
+
if fields.get("reason"):
|
|
91
|
+
fields["reason"] = _redactor.redact_text(fields["reason"])[:400]
|
|
92
|
+
return CheckResult(id=t.id, status=status, summary=summary, **fields)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
async def lookup(host: str, port: int) -> list[str]:
|
|
96
|
+
"""The addresses a name resolves to (DNS). Raises OSError (socket.gaierror) when it doesn't resolve."""
|
|
97
|
+
infos = await asyncio.get_running_loop().getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
|
98
|
+
return list(dict.fromkeys(info[4][0] for info in infos))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
async def knock(ip: str, port: int) -> str:
|
|
102
|
+
"""Does this address accept a connection? "open", "refused", "silent" (no answer) or "unreachable"."""
|
|
103
|
+
try:
|
|
104
|
+
_, writer = await asyncio.wait_for(asyncio.open_connection(ip, port), KNOCK_TIMEOUT_S)
|
|
105
|
+
except ConnectionRefusedError:
|
|
106
|
+
return "refused"
|
|
107
|
+
except TimeoutError:
|
|
108
|
+
return "silent"
|
|
109
|
+
except OSError:
|
|
110
|
+
return "unreachable"
|
|
111
|
+
writer.close()
|
|
112
|
+
return "open"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _addresses(ips: list[str], total: int | None = None) -> str:
|
|
116
|
+
""""Both of its addresses (a, b)", or "2 of its 4 addresses (a, b)" when only some of them are meant."""
|
|
117
|
+
n, total = len(ips), total or len(ips)
|
|
118
|
+
if n < total:
|
|
119
|
+
who = f"{n} of its {total} addresses"
|
|
120
|
+
else:
|
|
121
|
+
who = "Its address" if n == 1 else "Both of its addresses" if n == 2 else f"All {n} of its addresses"
|
|
122
|
+
return f"{who} ({', '.join(ips)})"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
async def _why_not(t: CheckTarget, url: str, error: str) -> CheckResult:
|
|
126
|
+
"""A request that couldn't connect, explained step by step (D24): does the name resolve, and does each address
|
|
127
|
+
accept, refuse or ignore a connection on the port? Only names, addresses and ports: nothing is sent."""
|
|
128
|
+
parts = urlsplit(url)
|
|
129
|
+
host, port = parts.hostname or "", parts.port or (443 if parts.scheme == "https" else 80)
|
|
130
|
+
try:
|
|
131
|
+
ips = (await lookup(host, port))[:MAX_ADDRESSES]
|
|
132
|
+
except OSError as exc:
|
|
133
|
+
why = getattr(exc, "strerror", None) or str(exc)
|
|
134
|
+
return _result(t, "down", "Its name doesn't resolve", reason=f"{host} has no address (DNS): {why}",
|
|
135
|
+
fix="Check the host name in the code. If it is right, the service may be gone, or this "
|
|
136
|
+
"computer's DNS is failing.", detail={"network": "dns"})
|
|
137
|
+
knocks = await asyncio.gather(*(knock(ip, port) for ip in ips))
|
|
138
|
+
by = {k: [ip for ip, r in zip(ips, knocks, strict=True) if r == k] for k in set(knocks)}
|
|
139
|
+
if by.get("open"):
|
|
140
|
+
return _result(t, "down", "Its server answers, but the secure connection failed" if parts.scheme == "https"
|
|
141
|
+
else "Its server answers, but the request failed", reason=error,
|
|
142
|
+
fix="Often a certificate problem, or a proxy or antivirus between you and the service.",
|
|
143
|
+
detail={"network": "request"})
|
|
144
|
+
if by.get("refused"):
|
|
145
|
+
others = len(ips) - len(by["refused"])
|
|
146
|
+
tail = f" The other {others} {'is' if others == 1 else 'are'} unreachable from this network." if others else ""
|
|
147
|
+
return _result(t, "down", "Refused the connection",
|
|
148
|
+
reason=f"{_addresses(by['refused'], len(ips))} refused connections on port {port}.{tail}",
|
|
149
|
+
fix="Nothing there accepts connections: the service is down, or it is blocking this computer "
|
|
150
|
+
"(public servers often block addresses that send too many requests). Try again later; if "
|
|
151
|
+
"it lasts, check the service's status page or use a mirror.",
|
|
152
|
+
detail={"network": "refused"})
|
|
153
|
+
if by.get("silent"):
|
|
154
|
+
server = "server" if len(ips) == 1 else "servers"
|
|
155
|
+
return _result(t, "down", f"No answer from its {server}",
|
|
156
|
+
reason=f"{_addresses(ips)} didn't answer on port {port}.",
|
|
157
|
+
fix="The service may be down, or a firewall between you is dropping the connection.",
|
|
158
|
+
detail={"network": "silent"})
|
|
159
|
+
return _result(t, "down", "Can't reach its servers",
|
|
160
|
+
reason=f"{_addresses(ips)} can't be reached from this network.",
|
|
161
|
+
fix="Check this computer's network or VPN.", detail={"network": "unreachable"})
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass
|
|
165
|
+
class _Answer:
|
|
166
|
+
code: int
|
|
167
|
+
ms: float
|
|
168
|
+
not_after: str | None = None
|
|
169
|
+
body: bytes | None = None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
async def _get(t: CheckTarget, url: str, http: httpx.AsyncClient, *, read: bool = False) -> _Answer | CheckResult:
|
|
173
|
+
"""One GET without a key, cookie or query (D24). The body is read only when `read` (a model list)."""
|
|
174
|
+
start = time.perf_counter()
|
|
175
|
+
try:
|
|
176
|
+
async with http.stream("GET", url, headers={"User-Agent": USER_AGENT}, timeout=HTTP_TIMEOUT_S,
|
|
177
|
+
follow_redirects=False) as response:
|
|
178
|
+
answer = _Answer(response.status_code, (time.perf_counter() - start) * 1000, _not_after(response))
|
|
179
|
+
if read and response.status_code == 200:
|
|
180
|
+
answer.body = await response.aread()
|
|
181
|
+
return answer
|
|
182
|
+
except httpx.TimeoutException:
|
|
183
|
+
return _result(t, "down", f"No answer within {HTTP_TIMEOUT_S} s")
|
|
184
|
+
except httpx.ConnectError as exc:
|
|
185
|
+
return await _why_not(t, url, str(exc) or type(exc).__name__)
|
|
186
|
+
except httpx.HTTPError as exc:
|
|
187
|
+
return _result(t, "down", "Can't connect", reason=str(exc) or type(exc).__name__)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _warnings(url: str, answer: _Answer) -> list[str]:
|
|
191
|
+
out = []
|
|
192
|
+
if answer.ms > SLOW_MS:
|
|
193
|
+
out.append(f"slow: {took(answer.ms)}")
|
|
194
|
+
if url.startswith("http://") and not is_local(host_of(url)):
|
|
195
|
+
out.append("plain http://: not encrypted")
|
|
196
|
+
if answer.not_after and (cert := cert_warning(answer.not_after, datetime.now(UTC))):
|
|
197
|
+
out.append(cert)
|
|
198
|
+
return out
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
async def api(t: CheckTarget, http: httpx.AsyncClient) -> CheckResult:
|
|
202
|
+
if not t.target:
|
|
203
|
+
return _result(t, "skipped", "Not checked: no address in the code")
|
|
204
|
+
url = t.target if "://" in t.target else f"https://{t.target}"
|
|
205
|
+
answer = await _get(t, url, http)
|
|
206
|
+
if isinstance(answer, CheckResult):
|
|
207
|
+
return answer
|
|
208
|
+
detail = {"http_status": answer.code}
|
|
209
|
+
if answer.code >= 500:
|
|
210
|
+
return _result(t, "down", f"Answered {answer.code}", latency_ms=answer.ms, detail=detail)
|
|
211
|
+
up = f"Up, needs a key ({answer.code})" if answer.code in (401, 403) else "Up"
|
|
212
|
+
if warnings := _warnings(url, answer):
|
|
213
|
+
return _result(t, "warn", f"{up}, but {'; '.join(warnings)}", latency_ms=answer.ms, detail=detail)
|
|
214
|
+
return _result(t, "up", up, latency_ms=answer.ms, detail=detail)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _model_names(body: bytes) -> set[str]:
|
|
218
|
+
"""Model names from an Ollama, OpenAI-style or Gemini model list, without "models/" or ":latest"."""
|
|
219
|
+
try:
|
|
220
|
+
data = json.loads(body)
|
|
221
|
+
except ValueError:
|
|
222
|
+
return set()
|
|
223
|
+
items = data.get("models") or data.get("data") or [] if isinstance(data, dict) else []
|
|
224
|
+
names = {str(i.get("name") or i.get("id") or "") for i in items if isinstance(i, dict)}
|
|
225
|
+
return {n.removeprefix("models/").removesuffix(":latest") for n in names if n}
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
async def llm(t: CheckTarget, http: httpx.AsyncClient) -> CheckResult:
|
|
229
|
+
host = host_of(f"http://{t.target}") if t.target and "://" not in t.target else host_of(t.target or "")
|
|
230
|
+
if t.target and "://" in t.target: # a base URL (a connector's): its model list is under it
|
|
231
|
+
url = f"{t.target.rstrip('/')}/models"
|
|
232
|
+
elif t.catalog in MODEL_LISTS:
|
|
233
|
+
url = MODEL_LISTS[t.catalog].format(host=host or "localhost:11434")
|
|
234
|
+
elif host:
|
|
235
|
+
url = f"{'http' if is_local(host) else 'https'}://{host}/v1/models"
|
|
236
|
+
else:
|
|
237
|
+
return _result(t, "skipped", "Not checked: no address in the code")
|
|
238
|
+
where = host_of(url)
|
|
239
|
+
answer = await _get(t, url, http, read=True)
|
|
240
|
+
if isinstance(answer, CheckResult):
|
|
241
|
+
if answer.status == "down" and is_local(where):
|
|
242
|
+
fix = "Start Ollama (`ollama serve`)." if t.catalog == "ollama" else None
|
|
243
|
+
return answer.model_copy(update={"summary": f"Nothing answers at {where}", "fix": fix})
|
|
244
|
+
return answer
|
|
245
|
+
detail = {"http_status": answer.code}
|
|
246
|
+
if answer.code >= 500:
|
|
247
|
+
return _result(t, "down", f"Answered {answer.code}", latency_ms=answer.ms, detail=detail)
|
|
248
|
+
if answer.code in (401, 403):
|
|
249
|
+
return _result(t, "up", f"Up, needs a key ({answer.code}) to list its models", latency_ms=answer.ms,
|
|
250
|
+
detail=detail)
|
|
251
|
+
if t.model and answer.body is not None:
|
|
252
|
+
listed = t.model.removesuffix(":latest") in _model_names(answer.body)
|
|
253
|
+
detail["model_listed"] = listed
|
|
254
|
+
if not listed:
|
|
255
|
+
fix = f"Run `ollama pull {t.model}`." if t.catalog == "ollama" else None
|
|
256
|
+
what = "isn't pulled" if t.catalog == "ollama" else "isn't in its model list"
|
|
257
|
+
return _result(t, "warn", f"Up, but {t.model} {what}", latency_ms=answer.ms, detail=detail, fix=fix)
|
|
258
|
+
up = f"Up, {t.model} is ready" if t.model and detail.get("model_listed") else "Up"
|
|
259
|
+
if warnings := _warnings(url, answer):
|
|
260
|
+
return _result(t, "warn", f"{up}, but {'; '.join(warnings)}", latency_ms=answer.ms, detail=detail)
|
|
261
|
+
return _result(t, "up", up, latency_ms=answer.ms, detail=detail)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _program_missing(program: str, root: Path) -> bool:
|
|
265
|
+
if os.path.dirname(program):
|
|
266
|
+
return not (root / program).is_file() and not Path(program).is_file()
|
|
267
|
+
return shutil.which(program) is None
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _env_fix(t: CheckTarget) -> str | None:
|
|
271
|
+
"""A likely fix when the server's own script reads env names its config doesn't give it (D24)."""
|
|
272
|
+
missing = [e for e in t.reads_env if e.name not in t.env_names]
|
|
273
|
+
if not missing:
|
|
274
|
+
return None
|
|
275
|
+
gives = "gives it no env" if not t.env_names else f"doesn't set {'it' if len(missing) == 1 else 'them'}"
|
|
276
|
+
return f"Its code reads {', '.join(f'{e.name} ({e.ref})' for e in missing)}, but its MCP config {gives}."
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _tail(errlog: Any) -> str:
|
|
280
|
+
errlog.seek(0)
|
|
281
|
+
lines = [line.rstrip() for line in errlog.read().splitlines() if line.strip()]
|
|
282
|
+
return "\n".join(lines[-STDERR_LINES:])
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
async def _list_tools(params: StdioServerParameters, errlog: Any) -> list[str]:
|
|
286
|
+
async with stdio_client(params, errlog=errlog) as (read, write), ClientSession(read, write) as session:
|
|
287
|
+
await session.initialize()
|
|
288
|
+
names: list[str] = []
|
|
289
|
+
cursor: str | None = None
|
|
290
|
+
while True:
|
|
291
|
+
page = await session.list_tools(cursor=cursor) if cursor else await session.list_tools()
|
|
292
|
+
names += [tool.name for tool in page.tools]
|
|
293
|
+
if not (cursor := page.next_cursor):
|
|
294
|
+
return sorted(names)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
async def mcp(t: CheckTarget, root: Path, http: httpx.AsyncClient) -> CheckResult:
|
|
298
|
+
"""Start the server in the repo folder with the SDK's minimal environment, list its tools, stop it (D24)."""
|
|
299
|
+
command = t.command or []
|
|
300
|
+
if not command:
|
|
301
|
+
return _result(t, "skipped", "Not checked: no command in its config")
|
|
302
|
+
if is_remote(command): # a remote server: reachable, like an API
|
|
303
|
+
return await api(t.model_copy(update={"target": command[0]}), http)
|
|
304
|
+
if not starts_process(t):
|
|
305
|
+
return _result(t, "skipped", "Not checked: its arguments hold a masked secret")
|
|
306
|
+
program, args = command[0], command[1:]
|
|
307
|
+
if _program_missing(program, root):
|
|
308
|
+
return _result(t, "down", f"Can't start: {program} isn't installed")
|
|
309
|
+
params = StdioServerParameters(command=program, args=args, cwd=str(root), env=get_default_environment())
|
|
310
|
+
with tempfile.TemporaryFile("w+", encoding="utf-8", errors="replace") as errlog:
|
|
311
|
+
start = time.perf_counter()
|
|
312
|
+
try:
|
|
313
|
+
async with asyncio.timeout(MCP_TIMEOUT_S):
|
|
314
|
+
tools = await _list_tools(params, errlog)
|
|
315
|
+
except TimeoutError:
|
|
316
|
+
return _result(t, "down", f"No answer within {MCP_TIMEOUT_S:g} s", reason=_tail(errlog) or None)
|
|
317
|
+
except Exception as exc: # anything the server or the SDK raised: the server didn't come up
|
|
318
|
+
while isinstance(exc, BaseExceptionGroup) and exc.exceptions:
|
|
319
|
+
exc = exc.exceptions[0]
|
|
320
|
+
reason = _tail(errlog) or str(exc) or type(exc).__name__
|
|
321
|
+
if t.env_names: # the names its config sets, never their values (D24)
|
|
322
|
+
reason += f"\nIts config sets {', '.join(t.env_names)}."
|
|
323
|
+
return _result(t, "down", "Stopped while starting", reason=reason, fix=_env_fix(t))
|
|
324
|
+
ms = (time.perf_counter() - start) * 1000
|
|
325
|
+
up = f"Up: {len(tools)} tool{'s' if len(tools) != 1 else ''}"
|
|
326
|
+
detail = {"tools": tools}
|
|
327
|
+
if ms > MCP_SLOW_MS:
|
|
328
|
+
return _result(t, "warn", f"{up}, but slow to start: {took(ms)}", latency_ms=ms, detail=detail)
|
|
329
|
+
return _result(t, "up", up, latency_ms=ms, detail=detail)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def local(t: CheckTarget, root: Path) -> CheckResult:
|
|
333
|
+
"""The program is on PATH and, when an interpreter runs a script, the script is in the repo. Never run (D24)."""
|
|
334
|
+
command = t.command or []
|
|
335
|
+
if not command:
|
|
336
|
+
return _result(t, "skipped", "Not checked: no command in the code")
|
|
337
|
+
program = command[0]
|
|
338
|
+
if _program_missing(program, root):
|
|
339
|
+
return _result(t, "down", f"Missing: {program} isn't installed")
|
|
340
|
+
script = command[1] if len(command) > 1 and not command[1].startswith("-") else None
|
|
341
|
+
if script and not (root / script).is_file():
|
|
342
|
+
return _result(t, "down", f"Missing: {script} isn't in the repo")
|
|
343
|
+
return _result(t, "up", "Found, not run")
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
async def check(target: CheckTarget, root: Path, *, http: httpx.AsyncClient | None = None) -> CheckResult:
|
|
347
|
+
"""Check one dependency (D24). `root` is the scanned folder, where MCP servers start and scripts are looked for."""
|
|
348
|
+
if http is None:
|
|
349
|
+
async with new_client() as own:
|
|
350
|
+
return await check(target, root, http=own)
|
|
351
|
+
if target.kind == "api":
|
|
352
|
+
return await api(target, http)
|
|
353
|
+
if target.kind == "llm":
|
|
354
|
+
return await llm(target, http)
|
|
355
|
+
if target.kind == "mcp":
|
|
356
|
+
return await mcp(target, root, http)
|
|
357
|
+
if target.kind == "local":
|
|
358
|
+
return local(target, root)
|
|
359
|
+
raise ValueError(f"no check for kind '{target.kind}'")
|
flowforge/cli.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""`flowforge`: start the app and open the dashboard (D15, D16).
|
|
2
|
+
|
|
3
|
+
uv run flowforge your data (flowforge.db), opens http://127.0.0.1:8000
|
|
4
|
+
uv run flowforge --repo PATH the same, mapping the folder PATH; a different folder needs --replace (D21)
|
|
5
|
+
uv run flowforge --example example data in FLOWFORGE_HOME/example.db, no keys needed
|
|
6
|
+
uv run flowforge scan PATH print the folder's dependencies as JSON; nothing stored, nothing sent
|
|
7
|
+
uv run flowforge mcp [--repo P] an MCP server for your coding agent, over stdio (D29); P defaults to here
|
|
8
|
+
uv run flowforge serve --reload (used by `npm run dev`; `serve` is optional)
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import threading
|
|
18
|
+
import webbrowser
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from flowforge.home import flowforge_home, resource, version
|
|
22
|
+
|
|
23
|
+
DIST = resource("dashboard") / "index.html"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parse(argv: list[str] | None) -> argparse.Namespace:
|
|
27
|
+
parser = argparse.ArgumentParser(prog="flowforge", description="Run the FlowForge dashboard.")
|
|
28
|
+
parser.add_argument("command", nargs="?", default="serve", choices=["serve", "scan", "mcp"])
|
|
29
|
+
parser.add_argument("path", nargs="?", help="scan: the folder to read")
|
|
30
|
+
parser.add_argument("--repo", help="serve: map this folder; mcp: the repo to answer about (default: here)")
|
|
31
|
+
parser.add_argument("--db", help="mcp: the dashboard's database (default: FLOWFORGE_DB, else flowforge.db)")
|
|
32
|
+
parser.add_argument("--replace", action="store_true",
|
|
33
|
+
help="serve: --repo names a different folder than last time, so start a fresh map")
|
|
34
|
+
parser.add_argument("--example", action="store_true", help="use example data (no keys, no network)")
|
|
35
|
+
parser.add_argument("--version", action="version", version=f"flowforge {version()}")
|
|
36
|
+
parser.add_argument("--host", default=os.getenv("FLOWFORGE_HOST", "127.0.0.1"))
|
|
37
|
+
parser.add_argument("--port", type=int, default=8000)
|
|
38
|
+
parser.add_argument("--no-open", action="store_true", help="don't open the browser")
|
|
39
|
+
parser.add_argument("--reload", action="store_true", help="restart when Python files change (development)")
|
|
40
|
+
return parser.parse_args(argv)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def main(argv: list[str] | None = None) -> None:
|
|
44
|
+
import uvicorn
|
|
45
|
+
|
|
46
|
+
args = parse(argv)
|
|
47
|
+
if args.command == "scan":
|
|
48
|
+
scan_folder(args.path)
|
|
49
|
+
return
|
|
50
|
+
if args.command == "mcp":
|
|
51
|
+
serve_agent(args.repo, args.db)
|
|
52
|
+
return
|
|
53
|
+
if args.repo:
|
|
54
|
+
if not Path(args.repo).expanduser().is_dir():
|
|
55
|
+
raise SystemExit(f"flowforge: {args.repo} is not a folder")
|
|
56
|
+
os.environ["FLOWFORGE_REPO"] = str(Path(args.repo).expanduser().resolve())
|
|
57
|
+
if args.replace:
|
|
58
|
+
os.environ["FLOWFORGE_REPO_REPLACE"] = "1"
|
|
59
|
+
if args.example:
|
|
60
|
+
os.environ["FLOWFORGE_EXAMPLE"] = "1"
|
|
61
|
+
os.environ.setdefault("FLOWFORGE_DB", str(flowforge_home() / "example.db"))
|
|
62
|
+
flowforge_home().mkdir(parents=True, exist_ok=True)
|
|
63
|
+
url = f"http://{args.host}:{args.port}/"
|
|
64
|
+
if not DIST.exists():
|
|
65
|
+
print("The dashboard isn't built yet: run `npm run build` in frontend/ (the classic page is at /classic).")
|
|
66
|
+
print(f"FlowForge{' (example data)' if args.example else ''} at {url}")
|
|
67
|
+
if not args.no_open:
|
|
68
|
+
threading.Timer(1.5, webbrowser.open, args=(url,)).start()
|
|
69
|
+
uvicorn.run("flowforge.main:app", host=args.host, port=args.port, reload=args.reload,
|
|
70
|
+
reload_dirs=[str(Path(__file__).parent)] if args.reload else None)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def serve_agent(repo: str | None, db: str | None = None) -> None:
|
|
74
|
+
"""The MCP server a coding agent starts (D29). stdout is the agent's pipe, so everything else goes to stderr."""
|
|
75
|
+
import logging
|
|
76
|
+
|
|
77
|
+
from flowforge.agent import build_server
|
|
78
|
+
from flowforge.home import database_path
|
|
79
|
+
from flowforge.security.logfilter import Redactor, install_log_redaction
|
|
80
|
+
from flowforge.storage import Storage
|
|
81
|
+
|
|
82
|
+
root = Path(repo or ".").expanduser().resolve()
|
|
83
|
+
if not root.is_dir():
|
|
84
|
+
print(f"flowforge mcp: {root} is not a folder", file=sys.stderr)
|
|
85
|
+
raise SystemExit(2)
|
|
86
|
+
logging.basicConfig(stream=sys.stderr, level=logging.WARNING)
|
|
87
|
+
redactor = Redactor()
|
|
88
|
+
install_log_redaction(redactor)
|
|
89
|
+
path = Path(db).expanduser() if db else database_path()
|
|
90
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
build_server(root, Storage(path, redact=redactor.redact)).run("stdio")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def scan_folder(path: str | None) -> None:
|
|
95
|
+
"""Read-only and offline (D20): print what the folder depends on, with evidence."""
|
|
96
|
+
from flowforge.scan import scan
|
|
97
|
+
|
|
98
|
+
if not path:
|
|
99
|
+
print("flowforge scan: name the folder to scan, for example `flowforge scan .`", file=sys.stderr)
|
|
100
|
+
raise SystemExit(2)
|
|
101
|
+
if not Path(path).expanduser().is_dir():
|
|
102
|
+
print(f"flowforge scan: {path} is not a folder", file=sys.stderr)
|
|
103
|
+
raise SystemExit(2)
|
|
104
|
+
print(json.dumps(scan(Path(path).expanduser()).model_dump(mode="json"), indent=2))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
if __name__ == "__main__":
|
|
108
|
+
main()
|
|
File without changes
|