flowforge-map 0.1.0__py3-none-any.whl

flowforge/nodes/local_node.py

@@ -0,0 +1,164 @@
+"""Local command step (D13). The connector fixes the program and the folder; the step adds arguments.
+
+params: args (list[str], appended to the connector's command), stdin? (str),
+        subdir? (str, must stay inside the connector's cwd), ok_exit_codes? (default [0])
+output: {"kind": "text", "stdout": str, "stderr": str, "exit_code": int}
+
+Safety rules, all enforced here:
+  - no shell: the program starts from an argv list; step values are separate arguments,
+    never pasted into a command string. .bat/.cmd are refused, because Windows runs them
+    through cmd.exe, which would bring shell parsing back
+  - subdir is resolved (`..`, symlinks) and must stay inside cwd
+  - the environment is explicit: connector env + resolved env_refs + the minimum the OS
+    needs to start a program; nothing else is inherited
+  - on timeout or cancellation (the executor's wait_for) the process is killed and reaped
+  - stdout/stderr are capped at max_output_bytes; the rest is read and discarded
+An exit code outside ok_exit_codes is a permanent error. Never cached across runs by default.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import shutil
+import sys
+from pathlib import Path, PurePosixPath, PureWindowsPath
+from typing import Any
+
+from flowforge.connectors.models import LocalConnector
+from flowforge.connectors.secrets import SecretRefError, resolve_env
+from flowforge.nodes.base import Node, NodeError
+
+# What a process needs to start at all; everything else must be configured explicitly.
+_OS_ENV = ("PATH", "SYSTEMROOT", "TEMP", "TMP") if sys.platform == "win32" else ("PATH",)
+_SHELL_SCRIPTS = {".bat", ".cmd"}
+_CHUNK = 64 * 1024
+
+
+class LocalNode(Node):
+    type = "local"
+
+    def __init__(self, connector: LocalConnector) -> None:
+        self.connector = connector
+        self.connector_id = connector.id
+        self.rate_limit_key = connector.id
+        self.fallback = connector.fallback
+
+    @classmethod
+    def from_connector(cls, connector: LocalConnector) -> LocalNode:
+        return cls(connector)
+
+    @property
+    def _label(self) -> str:
+        return f"local '{self.connector.id}'"
+
+    def _env(self) -> dict[str, str]:
+        conn = self.connector.connection
+        env = {name: os.environ[name] for name in _OS_ENV if name in os.environ}
+        try:
+            env.update(resolve_env(conn.env, conn.env_refs))
+        except SecretRefError as exc:
+            raise NodeError(f"{self._label}: {exc}") from exc
+        return env
+
+    def _workdir(self, subdir: Any) -> Path:
+        base = Path(self.connector.connection.cwd).resolve()
+        if not base.is_dir():
+            raise NodeError(f"{self._label}: working folder {base} does not exist")
+        if subdir is None:
+            return base
+        if not isinstance(subdir, str):
+            raise NodeError(f"{self._label}: params.subdir must be a string")
+        # reject absolute or drive paths of either platform before joining
+        win, posix = PureWindowsPath(subdir), PurePosixPath(subdir)
+        if win.is_absolute() or win.drive or win.root or posix.is_absolute():
+            raise NodeError(f"{self._label}: subdir '{subdir}' is outside the connector's folder")
+        target = (base / subdir).resolve()
+        if not target.is_relative_to(base):
+            raise NodeError(f"{self._label}: subdir '{subdir}' is outside the connector's folder")
+        if not target.is_dir():
+            raise NodeError(f"{self._label}: subdir '{subdir}' does not exist")
+        return target
+
+    def _program(self, env: dict[str, str]) -> str:
+        name = self.connector.connection.command[0]
+        if os.path.dirname(name):
+            program = name if Path(name).is_file() else None
+        else:
+            program = shutil.which(name, path=env.get("PATH", ""))
+        if program is None:
+            raise NodeError(f"{self._label}: program '{name}' not found")
+        if Path(program).suffix.lower() in _SHELL_SCRIPTS:
+            raise NodeError(f"{self._label}: '{name}' is a batch script, which Windows runs through cmd.exe "
+                            "(shell parsing); point the connector at the real program instead")
+        return program
+
+    async def run(self, params: dict[str, Any]) -> Any:
+        args = params.get("args", [])
+        if not isinstance(args, list) or not all(isinstance(a, str) for a in args):
+            raise NodeError(f"{self._label}: params.args must be a list of strings")
+        stdin = params.get("stdin")
+        if stdin is not None and not isinstance(stdin, str):
+            raise NodeError(f"{self._label}: params.stdin must be a string")
+        ok_codes = params.get("ok_exit_codes", [0])
+        cwd = self._workdir(params.get("subdir"))
+        env = self._env()
+        program = self._program(env)
+        cap = self.connector.connection.max_output_bytes
+
+        try:
+            proc = await asyncio.create_subprocess_exec(
+                program, *self.connector.connection.command[1:], *args,
+                cwd=cwd, env=env,
+                stdin=asyncio.subprocess.PIPE if stdin is not None else asyncio.subprocess.DEVNULL,
+                stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
+            )
+        except OSError as exc:
+            raise NodeError(f"{self._label}: could not start '{program}': {exc}") from exc
+
+        try:
+            (out, out_cut), (err, err_cut), _ = await asyncio.gather(
+                _read_capped(proc.stdout, cap), _read_capped(proc.stderr, cap), _feed(proc, stdin)
+            )
+            code = await proc.wait()
+        except BaseException:  # timeout/cancellation from the executor, or anything else
+            if proc.returncode is None:
+                proc.kill()
+                await proc.wait()
+            raise
+
+        stdout, stderr = _text(out, out_cut, cap), _text(err, err_cut, cap)
+        if code not in ok_codes:
+            raise NodeError(f"{self._label}: exit code {code}: {stderr[-500:].strip()}")
+        return {"kind": "text", "stdout": stdout, "stderr": stderr, "exit_code": code}
+
+
+async def _read_capped(stream: asyncio.StreamReader | None, cap: int) -> tuple[bytes, bool]:
+    """Keep the first `cap` bytes; keep reading (and discarding) so the child never blocks."""
+    kept = bytearray()
+    truncated = False
+    if stream is None:
+        return b"", False
+    while chunk := await stream.read(_CHUNK):
+        room = cap - len(kept)
+        if len(chunk) > room:
+            truncated = True
+        if room > 0:
+            kept += chunk[:room]
+    return bytes(kept), truncated
+
+
+async def _feed(proc: asyncio.subprocess.Process, data: str | None) -> None:
+    if data is None or proc.stdin is None:
+        return
+    try:
+        proc.stdin.write(data.encode())
+        await proc.stdin.drain()
+        proc.stdin.close()
+    except (BrokenPipeError, ConnectionResetError):
+        pass  # the program exited without reading all of it
+
+
+def _text(data: bytes, truncated: bool, cap: int) -> str:
+    text = data.decode("utf-8", errors="replace")
+    return text + f"\n[output truncated at {cap} bytes]" if truncated else text

flowforge/nodes/mcp_node.py

@@ -0,0 +1,243 @@
+"""MCP tool call with the official `mcp` SDK (D9, D12).
+
+params: server  — a command list for a stdio server (default ["uvx", "mcp-server-fetch"])
+                  or the name of a server registered with MCPNode(servers={...});
+                  not allowed on a connector node, whose connector fixes the server
+        tool    — e.g. "fetch"
+        arguments (dict)
+output: {"content": str (text blocks joined), "structured": dict | None,
+         "blocks": [...]  — only when the result has non-text blocks (D12)}
+
+One connection per server is opened on first use and reused for the whole run
+(spawning a process per call is slow). Its tool list is discovered once and every call
+is checked against it — unknown tool, or arguments that fail the tool's JSON Schema —
+before anything is sent (D12). A tool that reports is_error fails the step.
+Not cached across runs unless the step sets cache: true (tools can have side effects).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import hashlib
+import mimetypes
+import re
+from typing import Any
+
+import jsonschema
+from mcp import Client, MCPError, StdioServerParameters
+
+from flowforge.connectors.models import MCPConnector
+from flowforge.connectors.secrets import SecretRefError, resolve_env
+from flowforge.home import flowforge_home
+from flowforge.nodes.base import Node, NodeError, TransientNodeError, current_run_id
+
+DEFAULT_SERVER = ["uvx", "mcp-server-fetch"]
+INLINE_LIMIT_BYTES = 64 * 1024  # larger binary blocks go to an artifact file (D12)
+_RUN_ID = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
+
+
+class _Connection:
+    """Owns one MCP client inside a dedicated task.
+
+    anyio cancel scopes must be entered and exited by the same task, so the client's
+    context manager lives in `_own()` rather than in whichever step happened to open it.
+    Steps call tools on the shared client from their own tasks.
+    """
+
+    def __init__(self, server: Any) -> None:
+        self.server = server
+        self.client: Client | None = None
+        self.tools: list[dict[str, Any]] | None = None  # discovered once per connection
+        self._ready = asyncio.Event()
+        self._closing = asyncio.Event()
+        self._error: BaseException | None = None
+        self._task = asyncio.create_task(self._own())
+
+    async def _own(self) -> None:
+        try:
+            # stdio servers like mcp-server-fetch predate `server/discover`; skip the probe
+            mode = "legacy" if isinstance(self.server, StdioServerParameters) else "auto"
+            async with Client(self.server, cache=None, mode=mode) as client:
+                self.client = client
+                self._ready.set()
+                await self._closing.wait()
+        except Exception as exc:
+            self._error = exc
+        finally:
+            self.client = None
+            self._ready.set()
+
+    @property
+    def alive(self) -> bool:
+        return not self._task.done()
+
+    async def wait_ready(self) -> Client:
+        await self._ready.wait()
+        if self.client is None:
+            raise TransientNodeError(f"could not connect to MCP server: {self._error!r}")
+        return self.client
+
+    async def close(self) -> None:
+        self._closing.set()
+        await self._task
+
+
+def check_call(tools: list[dict[str, Any]], tool: str, arguments: Any) -> str | None:
+    """Why calling `tool` with `arguments` would be wrong, or None if it's fine (D12).
+    `arguments=None` checks the tool name only (used when arguments hold templates)."""
+    by_name = {t["name"]: t for t in tools}
+    if tool not in by_name:
+        return f"Unknown tool '{tool}'; this server has: {', '.join(sorted(by_name)) or 'no tools'}"
+    if arguments is None:
+        return None
+    schema = by_name[tool].get("input_schema") or {}
+    errors = sorted(jsonschema.Draft202012Validator(schema).iter_errors(arguments), key=lambda e: list(e.path))
+    if errors:
+        first = errors[0]
+        where = "/".join(map(str, first.path))
+        at = f" (at '{where}')" if where else ""
+        return f"arguments for tool '{tool}' don't match its schema: {first.message}{at}"
+    return None
+
+
+class MCPNode(Node):
+    type = "mcp"
+
+    def __init__(self, servers: dict[str, Any] | None = None, default_server: Any = None) -> None:
+        self.servers = servers or {}
+        self.default_server = default_server  # used when params.server is absent; None = DEFAULT_SERVER
+        self.connector: MCPConnector | None = None
+        self._connections: dict[Any, _Connection] = {}
+        self._lock = asyncio.Lock()
+
+    @classmethod
+    def from_connector(cls, connector: MCPConnector) -> MCPNode:
+        node = cls()
+        node.connector = connector
+        node.connector_id = connector.id
+        node.fallback = connector.fallback
+        node.rate_limit_key = connector.id
+        return node
+
+    def _connector_server(self) -> StdioServerParameters:
+        """Server parameters for the connector; secret env values are resolved here, at spawn (D12)."""
+        assert self.connector is not None
+        conn = self.connector.connection
+        try:
+            env = resolve_env(conn.env, conn.env_refs)
+        except SecretRefError as exc:
+            raise NodeError(f"MCP connector '{self.connector.id}': {exc}") from exc
+        return StdioServerParameters(command=conn.command, args=list(conn.args), env=env or None)
+
+    def _resolve_server(self, spec: Any) -> tuple[Any, Any]:
+        """(connection key, what to pass to mcp.Client)."""
+        if self.connector is not None:
+            if spec is not None:
+                raise NodeError(f"connector '{self.connector.id}' fixes the server; remove params.server")
+            return ("connector", self.connector.id), self._connector_server()
+        if spec is None:
+            if self.default_server is None:
+                spec = DEFAULT_SERVER
+            elif isinstance(self.default_server, list):
+                spec = self.default_server
+            else:
+                return "__default__", self.default_server
+        if isinstance(spec, str):
+            if spec not in self.servers:
+                raise NodeError(f"unknown MCP server '{spec}'")
+            return spec, self.servers[spec]
+        if not spec or not all(isinstance(part, str) for part in spec):
+            raise NodeError("params.server must be a server name or a command list")
+        return tuple(spec), StdioServerParameters(command=spec[0], args=list(spec[1:]))
+
+    async def _connection(self, spec: Any) -> tuple[_Connection, Client]:
+        key, server = self._resolve_server(spec)
+        async with self._lock:
+            conn = self._connections.get(key)
+            if conn is None or not conn.alive:
+                conn = self._connections[key] = _Connection(server)
+        return conn, await conn.wait_ready()
+
+    async def aclose(self) -> None:
+        for conn in self._connections.values():
+            await conn.close()
+        self._connections.clear()
+
+    async def list_tools(self, server: Any = None) -> list[dict[str, Any]]:
+        """Every tool the server offers: name, description, JSON input schema (D12)."""
+        conn, client = await self._connection(server)
+        if conn.tools is None:
+            tools: list[dict[str, Any]] = []
+            cursor: str | None = None
+            try:
+                while True:
+                    page = await client.list_tools(cursor=cursor)
+                    tools += [{"name": t.name, "description": t.description or "", "input_schema": t.input_schema}
+                              for t in page.tools]
+                    if not (cursor := page.next_cursor):
+                        break
+            except MCPError as exc:
+                raise NodeError(f"MCP error listing tools: {exc}") from exc
+            except (OSError, EOFError) as exc:
+                raise TransientNodeError(f"MCP connection lost: {exc}") from exc
+            conn.tools = tools
+        return conn.tools
+
+    async def run(self, params: dict[str, Any]) -> Any:
+        if "tool" not in params:
+            raise NodeError("mcp step needs params.tool")
+        server = params.get("server")
+        arguments = params.get("arguments", {})
+        problem = check_call(await self.list_tools(server), params["tool"], arguments)
+        if problem:
+            raise NodeError(problem)
+        _, client = await self._connection(server)
+        try:
+            result = await client.call_tool(params["tool"], arguments)
+        except MCPError as exc:
+            raise NodeError(f"MCP error: {exc}") from exc
+        except (OSError, EOFError) as exc:
+            raise TransientNodeError(f"MCP connection lost: {exc}") from exc
+
+        text = "\n".join(block.text for block in result.content if getattr(block, "type", None) == "text")
+        if result.is_error:
+            raise NodeError(f"MCP tool '{params['tool']}' failed: {text}")
+        output: dict[str, Any] = {"content": text, "structured": result.structured_content}
+        blocks = [b for b in map(_convert_block, result.content) if b is not None]
+        if blocks:
+            output["blocks"] = blocks
+        return output
+
+
+# --- non-text result blocks (D12) ----------------------------------------------------------------
+
+def _convert_block(block: Any) -> dict[str, Any] | None:
+    kind = getattr(block, "type", None)
+    if kind == "text":
+        return None
+    if kind in ("image", "audio"):
+        return _binary(kind, block.mime_type, base64.b64decode(block.data))
+    if kind == "resource_link":
+        return {"kind": "resource", "uri": str(block.uri), "name": block.name}
+    if kind == "resource":
+        res = block.resource
+        if getattr(res, "text", None) is not None:
+            return {"kind": "text", "uri": str(res.uri), "text": res.text}
+        mime = res.mime_type or "application/octet-stream"
+        major = mime.split("/", 1)[0]
+        return _binary(major if major in ("image", "audio", "video") else "file", mime, base64.b64decode(res.blob))
+    return {"kind": "unknown", "type": str(kind)}
+
+
+def _binary(kind: str, mime: str | None, data: bytes) -> dict[str, Any]:
+    mime = mime or "application/octet-stream"
+    if len(data) <= INLINE_LIMIT_BYTES:
+        return {"kind": kind, "mime": mime, "data_b64": base64.b64encode(data).decode()}
+    run_id = current_run_id.get()
+    folder = flowforge_home() / "artifacts" / (run_id if run_id and _RUN_ID.match(run_id) else "adhoc")
+    folder.mkdir(parents=True, exist_ok=True)
+    path = folder / f"{hashlib.sha256(data).hexdigest()}{mimetypes.guess_extension(mime) or '.bin'}"
+    if not path.exists():
+        path.write_bytes(data)
+    return {"kind": kind, "mime": mime, "path": str(path), "bytes": len(data)}

flowforge/nodes/mock_node.py

@@ -0,0 +1,40 @@
+"""Sleep-based node for tests and the simulated benchmark track (D6). Never touches the network.
+
+params:
+  duration_ms: how long to "work"
+  output:      value to return (default: {"step_done": true})
+  fail:        "transient" | "permanent" to simulate errors
+  fail_times:  how many calls fail before succeeding (default: always)
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from flowforge.nodes.base import Node, NodeError, TransientNodeError
+
+
+class MockNode(Node):
+    type = "mock"
+
+    def __init__(self) -> None:
+        self.calls = 0
+        self._failures: dict[str, int] = {}
+
+    async def run(self, params: dict[str, Any]) -> Any:
+        self.calls += 1
+        await asyncio.sleep(params.get("duration_ms", 100) / 1000)
+
+        fail = params.get("fail")
+        if fail:
+            key = repr(sorted(params.items()))
+            done = self._failures.get(key, 0)
+            limit = params.get("fail_times")
+            if limit is None or done < limit:
+                self._failures[key] = done + 1
+                if fail == "transient":
+                    raise TransientNodeError("simulated transient failure")
+                raise NodeError("simulated permanent failure")
+
+        return params.get("output", {"step_done": True})

flowforge/scan/__init__.py

@@ -0,0 +1,6 @@
+"""The repo scanner (D21): `scan(root) -> ScanResult`. Plain code, read-only, no network and no LLM (D20)."""
+
+from flowforge.scan.models import Dependency, ScanResult
+from flowforge.scan.scanner import scan
+
+__all__ = ["Dependency", "ScanResult", "scan"]

flowforge/scan/facts.py

@@ -0,0 +1,47 @@
+"""What one file shows, before the scanner merges files into dependencies (D21). Internal to `scan/`."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class UrlHit:
+    """A URL literal, or a constant's name, used where a URL goes. `confirmed`: a recognised call or key."""
+
+    line: int
+    url: str | None = None      # scheme://host[:port]/path, query dropped; None while `name` is unresolved
+    name: str | None = None     # a module-level constant holding the URL (resolved across files)
+    confirmed: bool = False
+    call_strings: tuple[str, ...] = ()   # the enclosing call's string arguments: call("fetch", ...) → via
+
+
+@dataclass
+class Ref:
+    """A value given either as a literal or as the name of a constant (resolved across files)."""
+
+    literal: str | None = None
+    name: str | None = None
+
+
+@dataclass
+class LLMClient:
+    line: int
+    sdk: str                    # the imported package the client class comes from
+    base_url: Ref = field(default_factory=Ref)
+
+
+@dataclass
+class FileFacts:
+    path: str
+    language: str
+    imports: list[tuple[str, int]] = field(default_factory=list)          # (package, line)
+    urls: list[UrlHit] = field(default_factory=list)
+    clients: list[LLMClient] = field(default_factory=list)
+    models: list[tuple[Ref, int]] = field(default_factory=list)           # model= values
+    env: list[tuple[str, int]] = field(default_factory=list)              # env var names read, never values
+    commands: list[tuple[list[str], int]] = field(default_factory=list)   # [program] or [interpreter, script]
+    stdio: list[tuple[list[str], int]] = field(default_factory=list)      # MCP servers started from code: argv
+    constants: dict[str, tuple[str, int]] = field(default_factory=dict)   # NAME -> (string value, line)
+    strings: set[str] = field(default_factory=set)       # short string literals: does the code name ".mcp.json"?
+    call_strings: set[str] = field(default_factory=set)  # string arguments of calls: does it name an MCP server?

flowforge/scan/files.py

@@ -0,0 +1,112 @@
+"""Which files the scan reads (D21, D22): code and known config files, never anything vendored, built or ignored."""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from flowforge.scan.manifests import is_manifest
+from flowforge.scan.mcp_config import is_config
+
+# never walked into, and not counted: other people's code and build output
+VENDOR_DIRS = frozenset({".git", "node_modules", ".venv", "venv", "build", "dist", ".dart_tool", "__pycache__",
+                         "vendor"})
+# skipped on purpose, and counted as "ignored": their URLs are fixtures, not calls (D21, D23)
+TEST_DIRS = frozenset({"tests", "test", "__tests__", "fixtures", "mocks", "__mocks__", "fakes"})
+TEST_FILE = re.compile(r"^(?:fake|mock|dummy)_|^test_.*\.py$|_test\.(?:py|dart)$|\.(?:test|spec)\.[^.]+$")
+DOC_SUFFIXES = frozenset({".md", ".txt", ".rst"})
+MAX_BYTES = 1_000_000
+BINARY_PROBE = 8192
+LANGUAGES = {".py": "python", ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript",
+             ".ts": "typescript", ".tsx": "typescript", ".dart": "dart"}
+
+
+@dataclass
+class Listing:
+    files: list[str] = field(default_factory=list)          # paths relative to the root, "/"-separated
+    skipped: dict[str, int] = field(default_factory=dict)   # why -> how many: ignored, binary, too_big, outside…
+
+    def skip(self, why: str) -> None:
+        self.skipped[why] = self.skipped.get(why, 0) + 1
+
+
+def language(rel: str) -> str | None:
+    return LANGUAGES.get(Path(rel).suffix.lower())
+
+
+def classify(rel: str) -> str | None:
+    """"read", "ignored" (code or docs skipped on purpose: tests, fake data, docs, your tools' hidden folders) or None
+    (not something the scan looks at, such as images or lock files).
+
+    Hidden folders (.claude, .cursor, .github …) hold your tools, not your app: only their MCP configs are read.
+    """
+    path = Path(rel)
+    relevant = language(rel) is not None or is_config(rel) or is_manifest(rel)
+    if any(part.startswith(".") for part in path.parent.parts):
+        return "read" if is_config(rel) else "ignored" if relevant else None
+    if any(part in TEST_DIRS for part in path.parent.parts) or TEST_FILE.search(path.name):
+        return "ignored" if relevant else None
+    if path.suffix.lower() in DOC_SUFFIXES and not is_manifest(rel):
+        return "ignored"
+    return "read" if relevant else None
+
+
+def _git_files(root: Path) -> list[str] | None:
+    """Files git tracks or doesn't ignore, relative to `root`; None when `root` isn't in a git repo.
+    A local command only: `ls-files` never touches the network."""
+    try:
+        done = subprocess.run(["git", "-C", str(root), "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
+                              capture_output=True, timeout=20, check=False,
+                              env={**os.environ, "GIT_TERMINAL_PROMPT": "0", "GIT_OPTIONAL_LOCKS": "0"})
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if done.returncode != 0:
+        return None
+    return sorted({p for p in done.stdout.decode("utf-8", "replace").split("\0") if p})
+
+
+def _walk(root: Path) -> list[str]:
+    found = []
+    for folder, dirs, names in os.walk(root):
+        dirs[:] = sorted(d for d in dirs if d not in VENDOR_DIRS)
+        found += [(Path(folder) / name).relative_to(root).as_posix() for name in sorted(names)]
+    return found
+
+
+def _is_binary(path: Path) -> bool:
+    with path.open("rb") as handle:
+        return b"\0" in handle.read(BINARY_PROBE)
+
+
+def list_files(root: Path) -> Listing:
+    out = Listing()
+    root = root.resolve()
+    candidates = _git_files(root)
+    if candidates is None:
+        candidates = _walk(root)
+    for rel in candidates:
+        if any(part in VENDOR_DIRS for part in Path(rel).parent.parts):
+            continue
+        kind = classify(rel)
+        if kind is None:
+            continue
+        if kind == "ignored":
+            out.skip("ignored")
+            continue
+        path = root / rel
+        if path.is_symlink() and not path.resolve().is_relative_to(root):
+            out.skip("outside")
+            continue
+        if not path.is_file():
+            continue
+        if path.stat().st_size > MAX_BYTES:
+            out.skip("too_big")
+            continue
+        if _is_binary(path):
+            out.skip("binary")
+            continue
+        out.files.append(rel)
+    return out

flowforge/scan/manifests.py

@@ -0,0 +1,65 @@
+"""Installed packages from dependency files (D21): requirements*.txt, pyproject.toml, package.json, pubspec.yaml.
+
+Only package names and their line are kept. An installed service SDK the code never imports becomes an
+unconfirmed hint on the map.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import tomllib
+from pathlib import PurePosixPath
+
+_REQUIREMENT = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._\-]*)")
+_PUBSPEC_DEP = re.compile(r"^  ([A-Za-z0-9_]+)\s*:")
+
+
+def is_manifest(rel: str) -> bool:
+    name = PurePosixPath(rel).name
+    return name in ("pyproject.toml", "package.json", "pubspec.yaml") or \
+        (name.startswith("requirements") and name.endswith(".txt"))
+
+
+def _line_of(lines: list[str], package: str) -> int:
+    pattern = re.compile(rf"""(^|[\s"'\[]){re.escape(package)}([\s"'=<>~!:\[;,]|$)""")
+    return next((i for i, raw in enumerate(lines, 1) if pattern.search(raw)), 1)
+
+
+def parse(rel: str, text: str) -> list[tuple[str, int]]:
+    """(package, line) for every dependency the file lists."""
+    name = PurePosixPath(rel).name
+    lines = text.splitlines()
+    if name.endswith(".txt"):
+        out = []
+        for i, raw in enumerate(lines, 1):
+            raw = raw.split("#")[0]
+            if raw.strip().startswith("-"):
+                continue
+            if m := _REQUIREMENT.match(raw):
+                out.append((m.group(1), i))
+        return out
+    if name == "pubspec.yaml":
+        out, inside = [], False
+        for i, raw in enumerate(lines, 1):
+            if raw and not raw[0].isspace():
+                inside = raw.rstrip().rstrip(":") in ("dependencies", "dev_dependencies")
+            elif inside and (m := _PUBSPEC_DEP.match(raw)) and m.group(1) != "flutter":
+                out.append((m.group(1), i))
+        return out
+    try:
+        data = json.loads(text) if name == "package.json" else tomllib.loads(text)
+    except ValueError:
+        return []
+    names: list[str] = []
+    if name == "package.json":
+        for key in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
+            names += list(data.get(key, {}) or {})
+    else:
+        project = data.get("project", {})
+        specs = list(project.get("dependencies", []))
+        for group in project.get("optional-dependencies", {}).values():
+            specs += group
+        names += [m.group(1) for spec in specs if (m := _REQUIREMENT.match(spec))]
+        names += [n for n in data.get("tool", {}).get("poetry", {}).get("dependencies", {}) if n != "python"]
+    return [(n, _line_of(lines, n)) for n in dict.fromkeys(names)]