stackthread 0.1.0.dev2__py3-none-any.whl

stackthread/lang/js/worker.py

@@ -0,0 +1,186 @@
+"""The node process that runs the bundled extractor. One long-lived worker, JSON lines over stdin and stdout."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import subprocess
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+from stackthread import logs
+
+MIN_NODE = (22, 18)
+_log = logs.get("worker")
+
+
+class NodeUnavailable(RuntimeError):
+    pass
+
+
+def extractor_path() -> Path | None:
+    """The bundled extractor: shipped in the wheel, or built in a source checkout, or set by STACKTHREAD_EXTRACTOR."""
+    override = os.environ.get("STACKTHREAD_EXTRACTOR")
+    if override:
+        return Path(override)
+    here = Path(__file__).resolve()
+    shipped = here.parents[2] / "_vendor" / "extract.mjs"
+    if shipped.is_file():
+        return shipped
+    for parent in here.parents:
+        candidate = parent / "js" / "packages" / "extract" / "dist" / "extract.mjs"
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def node_version(node: str) -> tuple[int, int] | None:
+    try:
+        out = subprocess.run([node, "--version"], capture_output=True, text=True, timeout=10, check=False).stdout
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    m = re.match(r"v(\d+)\.(\d+)", out.strip())
+    return (int(m.group(1)), int(m.group(2))) if m else None
+
+
+def find_node() -> str:
+    node = os.environ.get("STACKTHREAD_NODE") or shutil.which("node")
+    if not node:
+        raise NodeUnavailable("node is not on PATH; frontend sources need Node.js 22.18 or later")
+    version = node_version(node)
+    if version is None:
+        raise NodeUnavailable(f"could not run {node} --version")
+    if version < MIN_NODE:
+        raise NodeUnavailable(f"node {version[0]}.{version[1]} found; stackthread needs 22.18 or later")
+    return node
+
+
+class JsWorker:
+    def __init__(self) -> None:
+        self._proc: subprocess.Popen[bytes] | None = None
+        self._lock = threading.Lock()
+        self._next = 0
+
+    def _ensure(self) -> subprocess.Popen[bytes]:
+        if self._proc is not None and self._proc.poll() is None:
+            return self._proc
+        script = extractor_path()
+        if script is None:
+            raise NodeUnavailable(
+                "the JavaScript extractor is missing; reinstall stackthread or run `npm run build` in js/"
+            )
+        node = find_node()
+        self._proc = subprocess.Popen(
+            [node, str(script), "--worker"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            bufsize=0,
+        )
+        _log.info(
+            "started the JavaScript reader (node %s, pid %d)",
+            ".".join(map(str, node_version(node) or ())),
+            self._proc.pid,
+            extra=logs.ctx(node=node, script=str(script), pid=self._proc.pid),
+        )
+        threading.Thread(target=_drain, args=(self._proc,), name="stackthread-node-stderr", daemon=True).start()
+        return self._proc
+
+    def extract(
+        self, items: list[tuple[Path, str]], options: dict[str, Any] | None = None
+    ) -> dict[str, dict[str, Any]]:
+        """Facts for many files at once. `items` are (absolute path, project-relative file). Pipelined: requests
+        are written by a helper thread while replies are read, so large batches never deadlock on pipe buffers."""
+        if not items:
+            return {}
+        started = time.perf_counter()
+        with self._lock:
+            proc = self._ensure()
+            assert proc.stdin is not None and proc.stdout is not None
+            ids: dict[int, str] = {}
+            lines = []
+            for path, file in items:
+                self._next += 1
+                ids[self._next] = file
+                lines.append(json.dumps({"id": self._next, "path": str(path), "file": file, "options": options or {}}))
+            payload = ("\n".join(lines) + "\n").encode("utf-8")
+            stdin = proc.stdin
+
+            def write() -> None:
+                try:
+                    stdin.write(payload)
+                    stdin.flush()
+                except OSError:
+                    pass
+
+            writer = threading.Thread(target=write, daemon=True)
+            writer.start()
+            out: dict[str, dict[str, Any]] = {}
+            for _ in ids:
+                raw = proc.stdout.readline()
+                if not raw:
+                    code = proc.poll()
+                    self._proc = None
+                    _log.error(
+                        "the JavaScript reader stopped unexpectedly (exit code %s)", code, extra=logs.ctx(code=code)
+                    )
+                    raise NodeUnavailable("the extractor process stopped unexpectedly")
+                reply = json.loads(raw)
+                file = ids.get(reply.get("id"), "")
+                if "facts" in reply:
+                    out[file] = reply["facts"]
+                else:
+                    _log.warning("could not read %s: %s", file, reply.get("error"), extra=logs.ctx(file=file))
+                    out[file] = _error_facts(file, reply.get("error", "unknown error"))
+            writer.join(timeout=5)
+            _log.debug(
+                "read %d files with node in %.0f ms",
+                len(items),
+                (time.perf_counter() - started) * 1000,
+                extra=logs.ctx(files=len(items)),
+            )
+            return out
+
+    def close(self) -> None:
+        with self._lock:
+            if self._proc is not None:
+                try:
+                    if self._proc.stdin:
+                        self._proc.stdin.close()
+                    self._proc.wait(timeout=3)
+                except (OSError, subprocess.TimeoutExpired):
+                    self._proc.kill()
+                self._proc = None
+
+
+def _drain(proc: subprocess.Popen[bytes]) -> None:
+    """Node's own error output goes to the log, line by line, instead of being thrown away."""
+    if proc.stderr is None:
+        return
+    for raw in iter(proc.stderr.readline, b""):
+        line = raw.decode("utf-8", errors="replace").rstrip()
+        if line:
+            _log.warning("node: %s", line[:2000])
+
+
+def _error_facts(file: str, message: str) -> dict[str, Any]:
+    return {
+        "schema": 1,
+        "lang": "js",
+        "file": file,
+        "lines": 0,
+        "imports": [],
+        "exports": [],
+        "functions": [],
+        "elements": [],
+        "http": [],
+        "clients": [],
+        "routes": [],
+        "consts": {},
+        "dynamicImports": [],
+        "errors": [{"line": 0, "message": message}],
+    }

stackthread/lang/python/__init__.py

@@ -0,0 +1 @@
+"""Python: facts from the stdlib `ast`, symbol resolution, and framework detectors."""

stackthread/lang/python/build.py

@@ -0,0 +1,249 @@
+"""Python facts -> graph: modules, definitions, imports, inheritance, calls, uses, then framework and ORM detectors."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from stackthread.config import SourceConfig
+from stackthread.lang.python.resolve import PyIndex, Ref
+from stackthread.model import Edge, Finding, Graph, Node, Site
+
+MAX_OUTSIDE = 40
+
+
+def module_id(source: str, module: str) -> str:
+    return f"py:{source}:{module}"
+
+
+def def_id(source: str, key: str) -> str:
+    return f"py:{source}:{key}"
+
+
+def is_test_file(file: str) -> bool:
+    parts = file.split("/")
+    name = parts[-1]
+    return (
+        "tests" in parts[:-1]
+        or "test" in parts[:-1]
+        or name.startswith("test_")
+        or name.endswith("_test.py")
+        or name == "conftest.py"
+    )
+
+
+class PyBuild:
+    """Builds one Python source into the graph. Detectors use `node_of`, `ref_node` and the index."""
+
+    def __init__(self, graph: Graph, source: SourceConfig, facts: list[dict[str, Any]]) -> None:
+        self.graph = graph
+        self.source = source
+        self.idx = PyIndex(facts)
+        self.module_files = {m: f["file"] for m, f in self.idx.modules.items()}
+
+    # ---- ids ----
+
+    def node_of(self, key: str) -> str:
+        return def_id(self.source.name, key)
+
+    def ref_node(self, ref: Ref | None) -> str | None:
+        if ref is None:
+            return None
+        if ref.kind == "def" and ref.key in self.idx.defs:
+            return self.node_of(ref.key)
+        if ref.kind == "module" and ref.key in self.idx.modules:
+            return module_id(self.source.name, ref.key)
+        return None
+
+    def site(self, module: str, line: int) -> Site:
+        return Site(self.module_files[module], line)
+
+    # ---- build ----
+
+    def run(self) -> None:
+        g, src = self.graph, self.source.name
+        for module, f in self.idx.modules.items():
+            tags = ["test"] if is_test_file(f["file"]) else []
+            g.add_node(
+                Node(
+                    id=module_id(src, module),
+                    kind="module",
+                    name=module.rsplit(".", 1)[-1],
+                    qualname=module,
+                    lang="python",
+                    source=src,
+                    file=f["file"],
+                    line=1,
+                    end=f["lines"],
+                    layer="server",
+                    doc=f["doc"],
+                    tags=tags + (["package"] if f["package"] else []),
+                    attrs={"errors": f["errors"]} if f["errors"] else {},
+                )
+            )
+            for err in f["errors"]:
+                g.finding(Finding("parse-error", "error", err["message"], f["file"], err["line"]))
+        for key, d in self.idx.defs.items():
+            module = d["module"]
+            tags = ["test"] if is_test_file(d["file"]) else []
+            if d.get("async"):
+                tags.append("async")
+            g.add_node(
+                Node(
+                    id=self.node_of(key),
+                    kind=d["kind"],
+                    name=d["name"],
+                    qualname=f"{module}.{d['q']}",
+                    lang="python",
+                    source=src,
+                    file=d["file"],
+                    line=d["line"],
+                    end=d["end"],
+                    col=d["col"],
+                    layer="server",
+                    doc=d["doc"],
+                    tags=tags,
+                    attrs={
+                        k: v
+                        for k, v in {
+                            "args": [a["name"] for a in d.get("args", [])],
+                            "decorators": [x["f"] for x in d.get("decorators", [])],
+                            "bases": d.get("bases", []),
+                        }.items()
+                        if v
+                    },
+                )
+            )
+            parent = self.node_of(f"{module}:{d['parent']}") if d["parent"] else module_id(src, module)
+            g.add_edge(Edge(parent, self.node_of(key), "contains"))
+        self._imports()
+        self._inheritance()
+        self._calls()
+        from stackthread.lang.python import frameworks, orm
+
+        frameworks.detect(self)
+        orm.detect(self)
+
+    def _imports(self) -> None:
+        src = self.source.name
+        for module, f in self.idx.modules.items():
+            outside: list[str] = []
+            for imp in f["imports"]:
+                target = imp["module"]
+                name = imp["name"]
+                candidates = [f"{target}.{name}"] if name and name != "*" else []
+                candidates.append(target)
+                hit = next((c for c in candidates if c in self.idx.modules), None)
+                if hit and hit != module:
+                    self.graph.add_edge(
+                        Edge(
+                            module_id(src, module), module_id(src, hit), "imports", site=self.site(module, imp["line"])
+                        )
+                    )
+                elif not hit and target:
+                    root = target.split(".")[0]
+                    if root not in outside:
+                        outside.append(root)
+            if outside:
+                self.graph.nodes[module_id(src, module)].attrs["packages"] = outside
+
+    def _inheritance(self) -> None:
+        for key, d in self.idx.defs.items():
+            if d["kind"] != "class":
+                continue
+            for ref in self.idx.bases(key):
+                target = self.ref_node(ref)
+                if target:
+                    self.graph.add_edge(
+                        Edge(self.node_of(key), target, "inherits", site=self.site(d["module"], d["line"]))
+                    )
+
+    def _calls(self) -> None:
+        g = self.graph
+        for key, d in self.idx.defs.items():
+            node_id = self.node_of(key)
+            module = d["module"]
+            outside: list[str] = []
+            for call in d.get("calls", []):
+                ref, confidence = self.idx.resolve_call(d, call["f"])
+                target = self.ref_node(ref)
+                if target:
+                    g.add_edge(
+                        Edge(
+                            node_id,
+                            target,
+                            "calls",
+                            confidence,  # type: ignore[arg-type]
+                            site=self.site(module, call["line"]),
+                            label="raises" if call.get("raise") else "",
+                        )
+                    )
+                elif ref is not None and ref.kind == "external":
+                    if ref.key not in outside:
+                        outside.append(ref.key)
+                elif call["f"] not in outside and not call["f"].startswith("?"):
+                    outside.append(call["f"])
+                for arg in [*call.get("args", []), *call.get("kw", {}).values()]:
+                    self._passes(d, arg, call["line"])
+            self._uses(d)
+            if outside:
+                g.nodes[node_id].attrs["outside"] = outside[:MAX_OUTSIDE]
+
+    def _passes(self, d: dict[str, Any], arg: dict[str, Any] | None, line: int) -> None:
+        """A project function passed as an argument (a callback, a task, a signal handler) is used here."""
+        if not arg or arg.get("t") != "name":
+            return
+        ref = self.idx.resolve_dotted(d["module"], arg["v"], d)
+        target = self.ref_node(ref)
+        if target and ref and ref.kind == "def" and self.idx.defs[ref.key]["kind"] in ("function", "method"):
+            self.graph.add_edge(
+                Edge(self.node_of(d["key"]), target, "uses", site=self.site(d["module"], line), label="passes")
+            )
+
+    def _uses(self, d: dict[str, Any]) -> None:
+        g, module, node_id = self.graph, d["module"], self.node_of(d["key"])
+        site = self.site(module, d["line"])
+        for arg in d.get("args", []):
+            for name in arg.get("ann", []):
+                target = self.ref_node(self.idx.resolve_dotted(module, name))
+                if target and g.nodes[target].kind in ("class", "model"):
+                    g.add_edge(Edge(node_id, target, "uses", site=site, label="type"))
+            for value in (arg.get("default"), arg.get("ann_value")):
+                for call in _calls_in(value):
+                    if call.get("args") and call["args"][0] and call["args"][0].get("t") == "name":
+                        target = self.ref_node(self.idx.resolve_dotted(module, call["args"][0]["v"]))
+                        if target:
+                            label = call["f"].rsplit(".", 1)[-1]
+                            g.add_edge(Edge(node_id, target, "uses", site=site, label=label))
+        for name in d.get("returns", []):
+            target = self.ref_node(self.idx.resolve_dotted(module, name))
+            if target and g.nodes[target].kind in ("class", "model"):
+                g.add_edge(Edge(node_id, target, "uses", site=site, label="returns"))
+        for deco in d.get("decorators", []):
+            target = self.ref_node(self.idx.resolve_dotted(module, deco["f"]))
+            if target:
+                g.add_edge(Edge(node_id, target, "uses", site=self.site(module, deco["line"]), label="decorator"))
+
+
+def _calls_in(value: dict[str, Any] | None) -> list[dict[str, Any]]:
+    """Every call inside a summarised value, outermost first."""
+    if not value:
+        return []
+    out: list[dict[str, Any]] = []
+    t = value.get("t")
+    if t == "call":
+        out.append(value)
+        for a in [*value.get("args", []), *value.get("kw", {}).values()]:
+            out += _calls_in(a)
+    elif t == "list":
+        for v in value["v"]:
+            out += _calls_in(v)
+    elif t == "dict":
+        for v in value["v"].values():
+            out += _calls_in(v)
+    return out
+
+
+def build(graph: Graph, source: SourceConfig, facts: list[dict[str, Any]]) -> PyBuild:
+    builder = PyBuild(graph, source, facts)
+    builder.run()
+    return builder