flowforge-map 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowforge/__init__.py +0 -0
- flowforge/_classic.html +289 -0
- flowforge/_dashboard/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- flowforge/_dashboard/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- flowforge/_dashboard/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-ext-wght-normal-X_5orZeX.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-cyrillic-wght-normal-DiZS0aHC.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-ext-wght-normal-Bwz-egvJ.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-latin-wght-normal-XN7g48iV.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-symbols2-wght-normal-CO5SzqOn.woff2 +0 -0
- flowforge/_dashboard/assets/geist-mono-vietnamese-wght-normal-DadHysG0.woff2 +0 -0
- flowforge/_dashboard/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- flowforge/_dashboard/assets/index-BGT5seIN.css +1 -0
- flowforge/_dashboard/assets/index-BoGikST6.js +52 -0
- flowforge/_dashboard/favicon.svg +1 -0
- flowforge/_dashboard/index.html +16 -0
- flowforge/_example/trip-planner/.env.example +3 -0
- flowforge/_example/trip-planner/.gitignore +2 -0
- flowforge/_example/trip-planner/.mcp.json +20 -0
- flowforge/_example/trip-planner/README.md +48 -0
- flowforge/_example/trip-planner/config.py +27 -0
- flowforge/_example/trip-planner/mcp_servers/currency.py +10 -0
- flowforge/_example/trip-planner/planner.py +113 -0
- flowforge/_example/trip-planner/requirements.txt +3 -0
- flowforge/_example/trip-planner/services/__init__.py +0 -0
- flowforge/_example/trip-planner/services/geo.py +49 -0
- flowforge/_example/trip-planner/services/llm.py +18 -0
- flowforge/_example/trip-planner/services/mcp_tools.py +26 -0
- flowforge/_example/trip-planner/services/weather.py +25 -0
- flowforge/_example/trip-planner/tools/packing_list.py +24 -0
- flowforge/_example/trip-planner/trips/.gitkeep +0 -0
- flowforge/_workflows/diamond_mock.json +71 -0
- flowforge/_workflows/stripe_to_razorpay.json +38 -0
- flowforge/agent/__init__.py +5 -0
- flowforge/agent/server.py +333 -0
- flowforge/api/__init__.py +0 -0
- flowforge/api/agent.py +79 -0
- flowforge/api/checks.py +255 -0
- flowforge/api/connectors.py +289 -0
- flowforge/api/flowmap.py +174 -0
- flowforge/api/runs.py +284 -0
- flowforge/api/scan.py +117 -0
- flowforge/api/security.py +31 -0
- flowforge/api/stats.py +80 -0
- flowforge/checks/__init__.py +13 -0
- flowforge/checks/models.py +41 -0
- flowforge/checks/probes.py +359 -0
- flowforge/cli.py +108 -0
- flowforge/connectors/__init__.py +0 -0
- flowforge/connectors/catalog.py +84 -0
- flowforge/connectors/models.py +149 -0
- flowforge/connectors/presets.py +192 -0
- flowforge/connectors/registry.py +85 -0
- flowforge/connectors/runtime.py +47 -0
- flowforge/connectors/secrets.py +53 -0
- flowforge/example_data.py +247 -0
- flowforge/flow/__init__.py +328 -0
- flowforge/flow/advice.py +132 -0
- flowforge/flow/curly.py +884 -0
- flowforge/flow/index.py +116 -0
- flowforge/flow/order.py +181 -0
- flowforge/flow/reach.py +145 -0
- flowforge/flow/rewrite.py +146 -0
- flowforge/flowmap/__init__.py +1 -0
- flowforge/flowmap/diff.py +145 -0
- flowforge/flowmap/fixture.py +283 -0
- flowforge/flowmap/models.py +258 -0
- flowforge/flowmap/planner.py +131 -0
- flowforge/flowmap/service.py +379 -0
- flowforge/flowmap/validate.py +228 -0
- flowforge/home.py +58 -0
- flowforge/main.py +137 -0
- flowforge/nodes/__init__.py +3 -0
- flowforge/nodes/base.py +74 -0
- flowforge/nodes/check_node.py +50 -0
- flowforge/nodes/http_node.py +161 -0
- flowforge/nodes/llm_node.py +158 -0
- flowforge/nodes/local_node.py +164 -0
- flowforge/nodes/mcp_node.py +243 -0
- flowforge/nodes/mock_node.py +40 -0
- flowforge/scan/__init__.py +6 -0
- flowforge/scan/facts.py +47 -0
- flowforge/scan/files.py +112 -0
- flowforge/scan/manifests.py +65 -0
- flowforge/scan/mcp_config.py +75 -0
- flowforge/scan/models.py +73 -0
- flowforge/scan/patterns.py +86 -0
- flowforge/scan/python_ast.py +246 -0
- flowforge/scan/scanner.py +344 -0
- flowforge/scan/services.py +58 -0
- flowforge/scan/urls.py +60 -0
- flowforge/scheduler/__init__.py +0 -0
- flowforge/scheduler/cache.py +76 -0
- flowforge/scheduler/critical_path.py +65 -0
- flowforge/scheduler/durations.py +35 -0
- flowforge/scheduler/executor.py +390 -0
- flowforge/scheduler/graph.py +108 -0
- flowforge/scheduler/rate_limit.py +36 -0
- flowforge/schema.py +81 -0
- flowforge/security/__init__.py +0 -0
- flowforge/security/finders.py +127 -0
- flowforge/security/findings.py +169 -0
- flowforge/security/logfilter.py +129 -0
- flowforge/spa.py +52 -0
- flowforge/state.py +73 -0
- flowforge/storage.py +481 -0
- flowforge/templating.py +60 -0
- flowforge_map-0.1.0.dist-info/METADATA +335 -0
- flowforge_map-0.1.0.dist-info/RECORD +113 -0
- flowforge_map-0.1.0.dist-info/WHEEL +4 -0
- flowforge_map-0.1.0.dist-info/entry_points.txt +3 -0
- flowforge_map-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""MCP servers from config files (D21): .mcp.json, mcp.json, .vscode/.cursor mcp.json, claude_desktop_config.json.
|
|
2
|
+
|
|
3
|
+
Only the command, its arguments (redacted) and env var *names* are kept. Env values are never read out; a key-like
|
|
4
|
+
name whose value is written in the file rather than referenced is noted by name and line (D31).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from pathlib import PurePosixPath
|
|
12
|
+
|
|
13
|
+
from flowforge.scan.urls import clean_url
|
|
14
|
+
from flowforge.security.logfilter import REDACTED, SECRET_NAME, Redactor, redact_code
|
|
15
|
+
|
|
16
|
+
CONFIG_NAMES = frozenset({".mcp.json", "mcp.json", "claude_desktop_config.json"})
|
|
17
|
+
_redactor = Redactor()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class Server:
|
|
22
|
+
name: str
|
|
23
|
+
line: int
|
|
24
|
+
command: list[str] = field(default_factory=list)
|
|
25
|
+
env_names: list[str] = field(default_factory=list)
|
|
26
|
+
env_literals: list[tuple[str, int]] = field(default_factory=list) # (key-like name, line): a value written in
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def is_config(rel: str) -> bool:
|
|
30
|
+
return PurePosixPath(rel).name in CONFIG_NAMES
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def safe_args(args: list[str]) -> list[str]:
|
|
34
|
+
"""Arguments with key-shaped values and the values of key-like flags (--api-key X, TOKEN=X) masked."""
|
|
35
|
+
out: list[str] = []
|
|
36
|
+
for i, arg in enumerate(args):
|
|
37
|
+
if i and SECRET_NAME.search(args[i - 1]) and args[i - 1].startswith("-"):
|
|
38
|
+
out.append(REDACTED)
|
|
39
|
+
elif "=" in arg and SECRET_NAME.search(arg.split("=", 1)[0]):
|
|
40
|
+
out.append(f"{arg.split('=', 1)[0]}={REDACTED}")
|
|
41
|
+
else:
|
|
42
|
+
out.append(redact_code(arg, _redactor))
|
|
43
|
+
return out
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _line_of(lines: list[str], key: str, after: int) -> int:
|
|
47
|
+
return next((i for i, raw in enumerate(lines, 1) if i > after and raw.lstrip().startswith(key)), after)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def parse(text: str) -> list[Server]:
|
|
51
|
+
try:
|
|
52
|
+
data = json.loads(text)
|
|
53
|
+
except ValueError:
|
|
54
|
+
return []
|
|
55
|
+
block = data.get("mcpServers") or data.get("servers") if isinstance(data, dict) else None
|
|
56
|
+
if not isinstance(block, dict):
|
|
57
|
+
return []
|
|
58
|
+
lines = text.splitlines()
|
|
59
|
+
servers = []
|
|
60
|
+
for name, spec in block.items():
|
|
61
|
+
if not isinstance(spec, dict):
|
|
62
|
+
continue
|
|
63
|
+
key = f'"{name}"'
|
|
64
|
+
line = next((i for i, raw in enumerate(lines, 1) if raw.lstrip().startswith(key)), 1)
|
|
65
|
+
args = [str(a) for a in spec.get("args", []) if isinstance(a, (str, int, float))]
|
|
66
|
+
command = [str(spec["command"]), *args] if spec.get("command") else []
|
|
67
|
+
if not command and spec.get("url"): # a remote (HTTP/SSE) server: scheme, host and path only (D21)
|
|
68
|
+
url = clean_url(str(spec["url"]))
|
|
69
|
+
command = [url] if url else []
|
|
70
|
+
env = spec.get("env") if isinstance(spec.get("env"), dict) else {}
|
|
71
|
+
literals = [(n, _line_of(lines, f'"{n}"', line)) for n, v in env.items()
|
|
72
|
+
if SECRET_NAME.search(n) and isinstance(v, str) and v.strip() and not v.strip().startswith("$")]
|
|
73
|
+
servers.append(Server(name=name, line=line, command=safe_args(command), env_names=sorted(env),
|
|
74
|
+
env_literals=literals))
|
|
75
|
+
return servers
|
flowforge/scan/models.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""The scan's output (D21). Names, hosts, paths and `file:line` only: never a key, query string or env value."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Literal
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
8
|
+
|
|
9
|
+
DependencyKind = Literal["llm", "api", "mcp", "local"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class _Model(BaseModel):
|
|
13
|
+
model_config = ConfigDict(extra="forbid")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ScanEvidence(_Model):
|
|
17
|
+
"""One place in the repo that shows the dependency. `lines` is a redacted excerpt around `ref`."""
|
|
18
|
+
|
|
19
|
+
type: Literal["code"] = "code"
|
|
20
|
+
ref: str # "services/llm.py:5"
|
|
21
|
+
text: str = "" # one short sentence: what this line shows
|
|
22
|
+
first_line: int | None = None # the excerpt's first line number
|
|
23
|
+
lines: list[str] = Field(default_factory=list, max_length=5)
|
|
24
|
+
highlight: int | None = None # the line `ref` points at
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Dependency(_Model):
|
|
28
|
+
id: str # stable: llm-<provider>, api-<host or service>, mcp-<name>, local-<program>
|
|
29
|
+
kind: DependencyKind
|
|
30
|
+
name: str
|
|
31
|
+
catalog: str | None = None # a known service's id (presets.SERVICES), for its tile
|
|
32
|
+
target: str | None = None # what a check would reach: a URL without its query, host:port, or a command
|
|
33
|
+
targets: list[str] = Field(default_factory=list) # every distinct URL seen, for a service on several hosts
|
|
34
|
+
model: str | None = None # llm only
|
|
35
|
+
command: list[str] | None = None # mcp and local: the program and its arguments (redacted)
|
|
36
|
+
status: Literal["confirmed", "unconfirmed"] = "confirmed"
|
|
37
|
+
scope: Literal["app", "dev_tool"] = "app" # dev_tool: an MCP server only your coding tools use
|
|
38
|
+
via: str | None = None # reached through this MCP server, e.g. "mcp-fetch"
|
|
39
|
+
evidence: list[ScanEvidence] = Field(default_factory=list) # at most 10
|
|
40
|
+
places: int = 0 # how many places show it in all
|
|
41
|
+
env_names: list[str] = Field(default_factory=list)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Finding(_Model):
|
|
45
|
+
"""Something unsafe the scan saw (D31). Never holds the value it found: `excerpt` is masked like evidence."""
|
|
46
|
+
|
|
47
|
+
id: str # stable: <kind>-<path>-<line>
|
|
48
|
+
kind: Literal["key_in_code", "key_in_config", "plain_http", "unpinned_server", "broad_files"]
|
|
49
|
+
level: Literal["high", "medium", "low"]
|
|
50
|
+
title: str
|
|
51
|
+
ref: str # "config.py:23"
|
|
52
|
+
line: int
|
|
53
|
+
dep_id: str | None = None # the dependency it belongs to, when one does
|
|
54
|
+
first_line: int | None = None
|
|
55
|
+
excerpt: list[str] = Field(default_factory=list, max_length=5)
|
|
56
|
+
fix: str # one sentence
|
|
57
|
+
prompt: str # the whole change, for a coding agent (D27's rule)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class EnvName(_Model):
|
|
61
|
+
name: str
|
|
62
|
+
ref: str
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class ScanResult(_Model):
|
|
66
|
+
root: str # the folder's name, not its full path
|
|
67
|
+
files_scanned: int = 0
|
|
68
|
+
skipped: dict[str, int] = Field(default_factory=dict)
|
|
69
|
+
languages: dict[str, int] = Field(default_factory=dict)
|
|
70
|
+
dependencies: list[Dependency] = Field(default_factory=list)
|
|
71
|
+
env_names: list[EnvName] = Field(default_factory=list)
|
|
72
|
+
findings: list[Finding] = Field(default_factory=list) # D31
|
|
73
|
+
findings_checked: bool = False # the finders ran: a scan stored before D31 has an empty list that means nothing
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Facts from JS/TS and Dart files by patterns, line by line (D21). Less exact than Python's AST, and said so.
|
|
2
|
+
|
|
3
|
+
A URL counts only when a quote opens right before it (so URLs inside longer strings and bare comment links don't),
|
|
4
|
+
and it is confirmed by a call or key on the same line: fetch(, .get(, Uri.parse(, urlTemplate:, baseURL: …
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
from flowforge.scan import services
|
|
12
|
+
from flowforge.scan.facts import FileFacts, LLMClient, Ref, UrlHit
|
|
13
|
+
from flowforge.scan.urls import clean_url
|
|
14
|
+
|
|
15
|
+
_JS_IMPORT = re.compile(r"""^\s*import\s+(?:[^'"]*?\s+from\s+)?['"]([^'"]+)['"]|require\(\s*['"]([^'"]+)['"]\s*\)"""
|
|
16
|
+
r"""|import\(\s*['"]([^'"]+)['"]\s*\)""")
|
|
17
|
+
_DART_IMPORT = re.compile(r"""^\s*import\s+['"]package:([A-Za-z0-9_]+)/""")
|
|
18
|
+
_URL = re.compile(r"""['"`](https?://[^'"`\s]+)""")
|
|
19
|
+
_DART_URI = re.compile(r"""Uri\.https?\(\s*['"]([^'"]+)['"]\s*(?:,\s*['"]([^'"]*)['"])?""")
|
|
20
|
+
_CONFIRMS = re.compile(r"fetch\(|axios|\.(get|post|put|patch|delete|request)\(|Uri\.(parse|https?)\(|urlTemplate|"
|
|
21
|
+
r"base_?url|baseURL|endpoint|\burl\s*[:=]|http\.|dio\.", re.IGNORECASE)
|
|
22
|
+
_BASE_URL = re.compile(r"base_?url|baseURL", re.IGNORECASE)
|
|
23
|
+
_MODEL = re.compile(r"""\bmodel\s*[:=]\s*['"]([^'"]+)['"]""")
|
|
24
|
+
_ENV = re.compile(r"""process\.env\.([A-Za-z_][A-Za-z0-9_]*)|process\.env\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]"""
|
|
25
|
+
r"""|import\.meta\.env\.([A-Za-z_][A-Za-z0-9_]*)"""
|
|
26
|
+
r"""|String\.fromEnvironment\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]"""
|
|
27
|
+
r"""|Platform\.environment\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]"""
|
|
28
|
+
r"""|dotenv\.(?:env\[|get\()\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]""")
|
|
29
|
+
# child_process calls, bare or on child_process/cp, but never a method like regex.exec("…")
|
|
30
|
+
_COMMAND = re.compile(r"""(?:(?<![\w.$])|(?<=child_process\.)|(?<=\bcp\.))"""
|
|
31
|
+
r"""(?:spawn|spawnSync|execFile|execFileSync|exec|execSync)\(\s*['"]([^'"\s]+)"""
|
|
32
|
+
r"""|Process\.(?:run|start|runSync)\(\s*['"]([^'"\s]+)['"]""")
|
|
33
|
+
_STDIO = re.compile(r"""StdioClientTransport\(\s*\{([^}]*)\}""")
|
|
34
|
+
_STDIO_COMMAND = re.compile(r"""command:\s*['"]([^'"]+)['"]""")
|
|
35
|
+
_STDIO_ARGS = re.compile(r"""args:\s*\[([^\]]*)\]""")
|
|
36
|
+
_QUOTED = re.compile(r"""['"]([^'"]*)['"]""")
|
|
37
|
+
_STRING = re.compile(r"""['"]([^'"\n]{1,120})['"]""")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _package(spec: str) -> str | None:
|
|
41
|
+
"""npm package name from an import path; None for relative or built-in imports."""
|
|
42
|
+
if spec.startswith((".", "/", "node:")):
|
|
43
|
+
return None
|
|
44
|
+
parts = spec.split("/")
|
|
45
|
+
return "/".join(parts[:2]) if spec.startswith("@") else parts[0]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def extract(path: str, text: str, language: str) -> FileFacts:
|
|
49
|
+
facts = FileFacts(path=path, language=language)
|
|
50
|
+
dart = language == "dart"
|
|
51
|
+
for n, line in enumerate(text.splitlines(), 1):
|
|
52
|
+
stripped = line.lstrip()
|
|
53
|
+
if stripped.startswith(("//", "/*", "*")):
|
|
54
|
+
continue
|
|
55
|
+
if dart:
|
|
56
|
+
if m := _DART_IMPORT.match(line):
|
|
57
|
+
facts.imports.append((m.group(1), n))
|
|
58
|
+
else:
|
|
59
|
+
for m in _JS_IMPORT.finditer(line):
|
|
60
|
+
pkg = _package(next(g for g in m.groups() if g))
|
|
61
|
+
if pkg:
|
|
62
|
+
facts.imports.append((pkg, n))
|
|
63
|
+
confirmed = bool(_CONFIRMS.search(line))
|
|
64
|
+
for m in _URL.finditer(line):
|
|
65
|
+
if url := clean_url(m.group(1)):
|
|
66
|
+
facts.urls.append(UrlHit(line=n, url=url, confirmed=confirmed))
|
|
67
|
+
if _BASE_URL.search(line):
|
|
68
|
+
facts.clients.append(LLMClient(line=n, sdk="", base_url=Ref(literal=url)))
|
|
69
|
+
for m in _DART_URI.finditer(line):
|
|
70
|
+
if url := clean_url(f"https://{m.group(1)}{m.group(2) or ''}"):
|
|
71
|
+
facts.urls.append(UrlHit(line=n, url=url, confirmed=True))
|
|
72
|
+
if m := _MODEL.search(line):
|
|
73
|
+
facts.models.append((Ref(literal=m.group(1)), n))
|
|
74
|
+
for m in _ENV.finditer(line):
|
|
75
|
+
facts.env.append((next(g for g in m.groups() if g), n))
|
|
76
|
+
for m in _COMMAND.finditer(line):
|
|
77
|
+
facts.commands.append(([next(g for g in m.groups() if g)], n))
|
|
78
|
+
for m in _STDIO.finditer(line): # an MCP server started from code, on one line
|
|
79
|
+
if command := _STDIO_COMMAND.search(m.group(1)):
|
|
80
|
+
args = _STDIO_ARGS.search(m.group(1))
|
|
81
|
+
facts.stdio.append(([command.group(1), *(_QUOTED.findall(args.group(1)) if args else [])], n))
|
|
82
|
+
facts.strings.update(_STRING.findall(line))
|
|
83
|
+
# a base URL line is an LLM client only in a file that imports an LLM SDK; it belongs to that SDK
|
|
84
|
+
llm_sdk = next((pkg for pkg, _ in facts.imports if (s := services.by_sdk(pkg)) and s["kind"] == "llm"), None)
|
|
85
|
+
facts.clients = [LLMClient(line=c.line, sdk=llm_sdk, base_url=c.base_url) for c in facts.clients] if llm_sdk else []
|
|
86
|
+
return facts
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""Facts from a Python file through the stdlib `ast` (D21). Nothing is imported or run."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
|
|
7
|
+
from flowforge.scan.facts import FileFacts, LLMClient, Ref, UrlHit
|
|
8
|
+
from flowforge.scan.urls import clean_url
|
|
9
|
+
|
|
10
|
+
HTTP_CALLS = frozenset({"get", "post", "put", "patch", "delete", "head", "options", "request", "stream", "urlopen",
|
|
11
|
+
"Request", "fetch"})
|
|
12
|
+
# verbs that dicts and caches have too: an HTTP call only on something that looks like an HTTP client
|
|
13
|
+
AMBIGUOUS_CALLS = frozenset({"get", "delete", "head", "options"})
|
|
14
|
+
HTTP_RECEIVERS = ("client", "session", "http", "requests", "httpx", "aiohttp", "api", "urllib")
|
|
15
|
+
BASE_URL_KEYS = frozenset({"base_url", "api_base", "endpoint", "host"})
|
|
16
|
+
URL_KEYS = frozenset({"url", "uri", "base_url", "api_base", "api_url", "endpoint"})
|
|
17
|
+
MAX_STRING = 120
|
|
18
|
+
SUBPROCESS_CALLS = frozenset({"run", "Popen", "call", "check_call", "check_output"})
|
|
19
|
+
INTERPRETERS = frozenset({"python", "python3", "node", "bash", "sh", "ruby", "perl", "deno", "bun", "php", "pwsh"})
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _parents(tree: ast.AST) -> dict[ast.AST, ast.AST]:
|
|
23
|
+
return {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _callee(call: ast.Call) -> str | None:
|
|
27
|
+
f = call.func
|
|
28
|
+
return f.attr if isinstance(f, ast.Attribute) else f.id if isinstance(f, ast.Name) else None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _call_strings(call: ast.AST | None) -> tuple[str, ...]:
|
|
32
|
+
if not isinstance(call, ast.Call):
|
|
33
|
+
return ()
|
|
34
|
+
return tuple(a.value for a in call.args if isinstance(a, ast.Constant) and isinstance(a.value, str))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _receiver(call: ast.Call) -> str:
|
|
38
|
+
"""The lower-cased name the method is called on: `self.client.get` → "client", `requests.get` → "requests"."""
|
|
39
|
+
f = call.func
|
|
40
|
+
owner = f.value if isinstance(f, ast.Attribute) else None
|
|
41
|
+
if isinstance(owner, ast.Attribute):
|
|
42
|
+
return owner.attr.lower()
|
|
43
|
+
return owner.id.lower() if isinstance(owner, ast.Name) else ""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _is_http_call(call: ast.Call) -> bool:
|
|
47
|
+
name = _callee(call)
|
|
48
|
+
if name not in HTTP_CALLS:
|
|
49
|
+
return False
|
|
50
|
+
return name not in AMBIGUOUS_CALLS or any(hint in _receiver(call) for hint in HTTP_RECEIVERS)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _context(node: ast.AST, parents: dict[ast.AST, ast.AST]) -> tuple[bool, tuple[str, ...]]:
|
|
54
|
+
"""(is it where a URL goes, the enclosing call's string arguments).
|
|
55
|
+
|
|
56
|
+
Recognised: the first argument of an HTTP call, a url=/base_url= keyword, or the value of a "url" key in a
|
|
57
|
+
dict (directly or as a call argument: call("fetch", "fetch", {"url": ...})).
|
|
58
|
+
"""
|
|
59
|
+
parent = parents.get(node)
|
|
60
|
+
if isinstance(parent, ast.Call):
|
|
61
|
+
return bool(parent.args) and parent.args[0] is node and _is_http_call(parent), _call_strings(parent)
|
|
62
|
+
if isinstance(parent, ast.keyword):
|
|
63
|
+
return parent.arg in URL_KEYS, _call_strings(parents.get(parent))
|
|
64
|
+
if isinstance(parent, ast.Dict) and node in parent.values:
|
|
65
|
+
key = parent.keys[parent.values.index(node)]
|
|
66
|
+
is_url = isinstance(key, ast.Constant) and key.value in URL_KEYS
|
|
67
|
+
return is_url, _call_strings(parents.get(parent))
|
|
68
|
+
return False, ()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _ref(node: ast.AST | None) -> Ref:
|
|
72
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
73
|
+
return Ref(literal=node.value)
|
|
74
|
+
if isinstance(node, ast.Name):
|
|
75
|
+
return Ref(name=node.id)
|
|
76
|
+
if isinstance(node, ast.Attribute): # config.BASE_URL
|
|
77
|
+
return Ref(name=node.attr)
|
|
78
|
+
return Ref()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _env_read(call: ast.Call) -> tuple[str | None, ast.AST | None]:
|
|
82
|
+
"""(name, default) for os.getenv("X", d) / os.environ.get("X", d); (None, None) for any other call."""
|
|
83
|
+
f = call.func
|
|
84
|
+
is_getenv = isinstance(f, ast.Attribute) and f.attr == "getenv" or isinstance(f, ast.Name) and f.id == "getenv"
|
|
85
|
+
is_environ_get = isinstance(f, ast.Attribute) and f.attr == "get" and isinstance(f.value, ast.Attribute) \
|
|
86
|
+
and f.value.attr == "environ"
|
|
87
|
+
if not (is_getenv or is_environ_get) or not call.args:
|
|
88
|
+
return None, None
|
|
89
|
+
first = call.args[0]
|
|
90
|
+
if not (isinstance(first, ast.Constant) and isinstance(first.value, str)):
|
|
91
|
+
return None, None
|
|
92
|
+
default = call.args[1] if len(call.args) > 1 else next((k.value for k in call.keywords if k.arg == "default"), None)
|
|
93
|
+
return first.value, default
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _imports(tree: ast.AST, facts: FileFacts) -> dict[str, str]:
|
|
97
|
+
"""Record imported packages; return local name -> package for the names a client class can come from."""
|
|
98
|
+
aliases: dict[str, str] = {}
|
|
99
|
+
for node in ast.walk(tree):
|
|
100
|
+
if isinstance(node, ast.Import):
|
|
101
|
+
for a in node.names:
|
|
102
|
+
facts.imports.append((a.name, node.lineno))
|
|
103
|
+
aliases[a.asname or a.name.split(".")[0]] = a.name if a.asname else a.name.split(".")[0]
|
|
104
|
+
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
|
|
105
|
+
facts.imports.append((node.module, node.lineno))
|
|
106
|
+
for a in node.names:
|
|
107
|
+
aliases[a.asname or a.name] = node.module
|
|
108
|
+
facts.imports.append((f"{node.module}.{a.name}", node.lineno)) # from google import genai
|
|
109
|
+
return aliases
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def extract(path: str, text: str) -> FileFacts:
|
|
113
|
+
facts = FileFacts(path=path, language="python")
|
|
114
|
+
tree = ast.parse(text)
|
|
115
|
+
parents = _parents(tree)
|
|
116
|
+
aliases = _imports(tree, facts)
|
|
117
|
+
for stmt in tree.body: # module-level NAME = "..." (or os.getenv(..., "...")) constants
|
|
118
|
+
if isinstance(stmt, ast.Assign) and len(stmt.targets) == 1 and isinstance(stmt.targets[0], ast.Name):
|
|
119
|
+
value = stmt.value
|
|
120
|
+
if isinstance(value, ast.Call):
|
|
121
|
+
value = _env_read(value)[1]
|
|
122
|
+
if isinstance(value, ast.Constant) and isinstance(value.value, str):
|
|
123
|
+
facts.constants[stmt.targets[0].id] = (value.value, stmt.lineno)
|
|
124
|
+
for node in ast.walk(tree):
|
|
125
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
126
|
+
if len(node.value) <= MAX_STRING:
|
|
127
|
+
facts.strings.add(node.value)
|
|
128
|
+
if isinstance(parents.get(node), ast.JoinedStr):
|
|
129
|
+
continue # f-string pieces: only a leading URL counts, handled with the f-string below
|
|
130
|
+
url = clean_url(node.value)
|
|
131
|
+
if url:
|
|
132
|
+
confirmed, call = _context(node, parents)
|
|
133
|
+
facts.urls.append(UrlHit(line=node.lineno, url=url, confirmed=confirmed, call_strings=call))
|
|
134
|
+
elif isinstance(node, ast.JoinedStr) and node.values:
|
|
135
|
+
first = node.values[0]
|
|
136
|
+
confirmed, call = _context(node, parents)
|
|
137
|
+
if isinstance(first, ast.Constant) and isinstance(first.value, str) and (url := clean_url(first.value)):
|
|
138
|
+
facts.urls.append(UrlHit(line=node.lineno, url=url, confirmed=confirmed, call_strings=call))
|
|
139
|
+
elif isinstance(first, ast.FormattedValue) and isinstance(first.value, ast.Name): # f"{BASE}/x"
|
|
140
|
+
facts.urls.append(UrlHit(line=node.lineno, name=first.value.id, confirmed=confirmed,
|
|
141
|
+
call_strings=call))
|
|
142
|
+
elif isinstance(node, ast.Name) and (where := _context(node, parents))[0]:
|
|
143
|
+
facts.urls.append(UrlHit(line=node.lineno, name=node.id, confirmed=True, call_strings=where[1]))
|
|
144
|
+
elif isinstance(node, ast.Subscript) and isinstance(node.value, ast.Attribute) \
|
|
145
|
+
and node.value.attr == "environ" and isinstance(node.slice, ast.Constant) \
|
|
146
|
+
and isinstance(node.slice.value, str):
|
|
147
|
+
_env(facts, node.slice.value, node, parents)
|
|
148
|
+
elif isinstance(node, ast.Call):
|
|
149
|
+
_call(node, aliases, facts, parents)
|
|
150
|
+
return facts
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _env(facts: FileFacts, name: str, node: ast.AST, parents: dict[ast.AST, ast.AST]) -> None:
|
|
154
|
+
"""An env var name belongs to every line of its statement, so a call split over lines keeps it (D21)."""
|
|
155
|
+
stmt = node
|
|
156
|
+
while stmt in parents and not isinstance(stmt, ast.stmt):
|
|
157
|
+
stmt = parents[stmt]
|
|
158
|
+
first = getattr(stmt, "lineno", getattr(node, "lineno", 1))
|
|
159
|
+
last = getattr(stmt, "end_lineno", None) or first
|
|
160
|
+
facts.env.extend((name, line) for line in range(first, last + 1))
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _strings_in_order(node: ast.AST) -> list[str]:
|
|
164
|
+
"""String literals inside an expression, left to right: str(HERE / "tools" / "x.py") → ["tools", "x.py"]."""
|
|
165
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
166
|
+
return [node.value]
|
|
167
|
+
return [s for child in ast.iter_child_nodes(node) for s in _strings_in_order(child)]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _program(parts: list[ast.AST]) -> list[str] | None:
|
|
171
|
+
"""[program] or [interpreter, script] from an argv given as AST nodes; None when it isn't literal."""
|
|
172
|
+
if not parts:
|
|
173
|
+
return None
|
|
174
|
+
first = parts[0]
|
|
175
|
+
if isinstance(first, ast.Attribute) and first.attr == "executable": # sys.executable
|
|
176
|
+
program = "python"
|
|
177
|
+
elif isinstance(first, ast.Constant) and isinstance(first.value, str) and first.value.strip():
|
|
178
|
+
program = first.value.split()[0]
|
|
179
|
+
else:
|
|
180
|
+
return None
|
|
181
|
+
if program.rsplit("/", 1)[-1] in INTERPRETERS and len(parts) > 1:
|
|
182
|
+
script = "/".join(_strings_in_order(parts[1]))
|
|
183
|
+
if "." in script.rsplit("/", 1)[-1]:
|
|
184
|
+
return [program, script]
|
|
185
|
+
return [program]
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _subprocess(node: ast.Call, aliases: dict[str, str]) -> list[str] | None:
|
|
189
|
+
f = node.func
|
|
190
|
+
owner = f.value.id if isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name) else None
|
|
191
|
+
name = _callee(node)
|
|
192
|
+
if name in SUBPROCESS_CALLS and (owner == "subprocess" or (owner is None and aliases.get(name) == "subprocess")):
|
|
193
|
+
arg = node.args[0] if node.args else next((k.value for k in node.keywords if k.arg == "args"), None)
|
|
194
|
+
if isinstance(arg, (ast.List, ast.Tuple)):
|
|
195
|
+
return _program(list(arg.elts))
|
|
196
|
+
return _program([arg]) if arg is not None else None
|
|
197
|
+
if owner == "os" and name in ("system", "popen") and node.args:
|
|
198
|
+
return _program([node.args[0]])
|
|
199
|
+
if name == "create_subprocess_exec":
|
|
200
|
+
return _program(list(node.args))
|
|
201
|
+
if name == "create_subprocess_shell" and node.args:
|
|
202
|
+
return _program([node.args[0]])
|
|
203
|
+
return None
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _stdio_server(node: ast.Call) -> list[str] | None:
|
|
207
|
+
"""StdioServerParameters(command="uvx", args=["mcp-server-git", …]) with literal values: the argv."""
|
|
208
|
+
if _callee(node) != "StdioServerParameters":
|
|
209
|
+
return None
|
|
210
|
+
kw = {k.arg: k.value for k in node.keywords}
|
|
211
|
+
command, args = kw.get("command"), kw.get("args")
|
|
212
|
+
if not (isinstance(command, ast.Constant) and isinstance(command.value, str)):
|
|
213
|
+
return None
|
|
214
|
+
rest = [a.value for a in args.elts if isinstance(a, ast.Constant) and isinstance(a.value, str)] \
|
|
215
|
+
if isinstance(args, (ast.List, ast.Tuple)) else []
|
|
216
|
+
return [command.value, *rest]
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _call(node: ast.Call, aliases: dict[str, str], facts: FileFacts, parents: dict[ast.AST, ast.AST]) -> None:
|
|
220
|
+
facts.call_strings.update(_call_strings(node))
|
|
221
|
+
name, _ = _env_read(node)
|
|
222
|
+
if name:
|
|
223
|
+
_env(facts, name, node, parents)
|
|
224
|
+
return
|
|
225
|
+
server = _stdio_server(node)
|
|
226
|
+
if server:
|
|
227
|
+
facts.stdio.append((server, node.lineno))
|
|
228
|
+
return
|
|
229
|
+
command = _subprocess(node, aliases)
|
|
230
|
+
if command:
|
|
231
|
+
facts.commands.append((command, node.lineno))
|
|
232
|
+
return
|
|
233
|
+
f = node.func
|
|
234
|
+
sdk = None
|
|
235
|
+
if isinstance(f, ast.Name) and f.id[:1].isupper(): # OpenAI(...) from openai
|
|
236
|
+
sdk = aliases.get(f.id)
|
|
237
|
+
elif isinstance(f, ast.Attribute) and f.attr[:1].isupper() and isinstance(f.value, ast.Name):
|
|
238
|
+
sdk = aliases.get(f.value.id) # anthropic.Anthropic(...)
|
|
239
|
+
if sdk is not None:
|
|
240
|
+
base = next((k.value for k in node.keywords if k.arg in BASE_URL_KEYS), None)
|
|
241
|
+
facts.clients.append(LLMClient(line=node.lineno, sdk=sdk, base_url=_ref(base)))
|
|
242
|
+
model = next((k.value for k in node.keywords if k.arg == "model"), None)
|
|
243
|
+
if model is not None:
|
|
244
|
+
ref = _ref(model)
|
|
245
|
+
if ref.literal or ref.name:
|
|
246
|
+
facts.models.append((ref, node.lineno))
|