entrygraph 0.1.0__py3-none-any.whl

entrygraph/fs/walker.py

@@ -0,0 +1,167 @@
+"""Repository file walk with pruning, .gitignore support, and content gates.
+
+Order of gates (cheapest first):
+1. hard-pruned directory names (checked before pathspec — pruning node_modules
+   at the directory level is the single biggest walk speedup),
+2. .gitignore rules (root + nested, via pathspec),
+3. per-file gates: size cap, NUL-byte binary sniff, minified-JS heuristics.
+
+Every skipped-but-recognized file is reported with a reason so "why isn't my
+file indexed" is always answerable.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+import pathspec
+
+from entrygraph.fs.lang import RepoLanguageProfile, detect_language
+
+PRUNED_DIRS = frozenset(
+    {
+        ".git", ".hg", ".svn",
+        "node_modules", "bower_components",
+        ".venv", "venv", ".tox", "__pycache__", ".mypy_cache", ".ruff_cache",
+        ".pytest_cache", "site-packages", ".eggs",
+        "vendor", "third_party",
+        "dist", "build", "target", "out",
+        ".next", ".nuxt", ".gradle", ".idea", ".vscode",
+        "Pods", "DerivedData",
+    }
+)
+
+MAX_FILE_BYTES = 2 * 1024 * 1024  # 2 MiB
+_MINIFIED_SUFFIXES = (".min.js", ".min.css", ".bundle.js", ".map")
+
+
+@dataclass(slots=True)
+class WalkedFile:
+    path: str  # repo-relative, posix separators
+    abs_path: str
+    language: str | None
+    size_bytes: int
+    mtime_ns: int
+    skip_reason: str | None = None  # too_large | binary | minified | None
+
+
+def _load_gitignore(root: Path) -> pathspec.GitIgnoreSpec | None:
+    patterns: list[str] = []
+    for ignore_file in sorted(root.rglob(".gitignore")):
+        try:
+            rel_dir = ignore_file.parent.relative_to(root).as_posix()
+        except ValueError:  # pragma: no cover - symlink escape
+            continue
+        prefix = "" if rel_dir == "." else rel_dir + "/"
+        try:
+            lines = ignore_file.read_text(encoding="utf-8", errors="replace").splitlines()
+        except OSError:
+            continue
+        for line in lines:
+            stripped = line.strip()
+            if not stripped or stripped.startswith("#"):
+                continue
+            if prefix:
+                # scope nested .gitignore patterns to their directory
+                negate = stripped.startswith("!")
+                body = stripped[1:] if negate else stripped
+                anchored = body.lstrip("/")
+                scoped = f"{prefix}{anchored}" if body.startswith("/") else f"{prefix}**/{anchored}"
+                patterns.append(("!" if negate else "") + scoped)
+            else:
+                patterns.append(stripped)
+    if not patterns:
+        return None
+    return pathspec.GitIgnoreSpec.from_lines(patterns)
+
+
+def _content_gate(abs_path: str, language: str | None, size_bytes: int) -> str | None:
+    """Return a skip reason, or None if the file should be parsed."""
+    if size_bytes > MAX_FILE_BYTES:
+        return "too_large"
+    name = os.path.basename(abs_path)
+    if name.endswith(_MINIFIED_SUFFIXES):
+        return "minified"
+    try:
+        with open(abs_path, "rb") as fh:
+            head = fh.read(8192)
+    except OSError:
+        return "unreadable"
+    if b"\x00" in head:
+        return "binary"
+    if language in ("javascript", "typescript", "tsx"):
+        lines = head.splitlines()
+        if any(len(line) > 5000 for line in lines):
+            return "minified"
+        if size_bytes > 50 * 1024 and lines and (len(head) / max(len(lines), 1)) > 250:
+            return "minified"
+    return None
+
+
+def walk_repo(root: str | Path) -> tuple[list[WalkedFile], RepoLanguageProfile]:
+    """Walk a repository, returning candidate files and a language profile.
+
+    Files in unrecognized languages are omitted; recognized-but-gated files are
+    included with ``skip_reason`` set so they can be recorded in the DB.
+    """
+    root = Path(root).resolve()
+    spec = _load_gitignore(root)
+    profile = RepoLanguageProfile()
+    results: list[WalkedFile] = []
+
+    stack = [str(root)]
+    while stack:
+        current = stack.pop()
+        try:
+            entries = list(os.scandir(current))
+        except OSError:
+            continue
+        for entry in entries:
+            name = entry.name
+            try:
+                is_dir = entry.is_dir(follow_symlinks=False)
+            except OSError:
+                continue
+            if is_dir:
+                if name in PRUNED_DIRS or name.startswith(".git"):
+                    continue
+                rel = os.path.relpath(entry.path, root).replace(os.sep, "/")
+                if spec and spec.match_file(rel + "/"):
+                    continue
+                stack.append(entry.path)
+                continue
+            if not entry.is_file(follow_symlinks=False):
+                continue
+            rel = os.path.relpath(entry.path, root).replace(os.sep, "/")
+            if spec and spec.match_file(rel):
+                continue
+
+            language = detect_language(rel)
+            if language is None:
+                # one cheap read for shebang sniffing of extensionless files
+                if "." not in name:
+                    try:
+                        with open(entry.path, "rb") as fh:
+                            language = detect_language(rel, fh.readline(256))
+                    except OSError:
+                        continue
+                if language is None:
+                    continue
+
+            stat = entry.stat(follow_symlinks=False)
+            profile.add(language, stat.st_size)
+            results.append(
+                WalkedFile(
+                    path=rel,
+                    abs_path=entry.path,
+                    language=language,
+                    size_bytes=stat.st_size,
+                    mtime_ns=stat.st_mtime_ns,
+                    skip_reason=_content_gate(entry.path, language, stat.st_size),
+                )
+            )
+
+    results.sort(key=lambda f: f.path)
+    return results, profile

entrygraph/graph/adjacency.py

@@ -0,0 +1,146 @@
+"""In-memory adjacency over the edges table — the primary reachability engine.
+
+One indexed scan loads all resolved edges of the requested kinds into forward
+and reverse adjacency dicts; every subsequent traversal is pure-Python BFS/DFS.
+The cache is keyed by (edge kinds, index generation) and dropped on re-index.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from entrygraph.db.models import Edge
+from entrygraph.kinds import EdgeKind
+
+_MAX_DFS_VISITS = 200_000  # hard bound on path-enumeration work
+
+
+@dataclass(frozen=True, slots=True)
+class Hop:
+    dst: int
+    kind: str
+    line: int
+    confidence: int
+    edge_id: int = 0
+    via: str | None = None
+
+
+class AdjacencyCache:
+    def __init__(self, generation: int, kinds: frozenset[str]) -> None:
+        self.generation = generation
+        self.kinds = kinds
+        self.forward: dict[int, list[Hop]] = {}
+        self.reverse: dict[int, list[Hop]] = {}
+
+    @classmethod
+    def build(cls, session: Session, generation: int, kinds: frozenset[str],
+              min_confidence: int = 0, include_cha: bool = True) -> "AdjacencyCache":
+        cache = cls(generation, kinds)
+        conditions = [
+            Edge.kind.in_([EdgeKind(k) for k in kinds]),
+            Edge.dst_symbol_id.is_not(None),
+            Edge.confidence >= min_confidence,
+        ]
+        if not include_cha:  # class-hierarchy candidates are opt-in speculative edges
+            conditions.append((Edge.via.is_(None)) | (Edge.via != "cha"))
+        stmt = select(
+            Edge.src_symbol_id, Edge.dst_symbol_id, Edge.kind, Edge.line,
+            Edge.confidence, Edge.id, Edge.via,
+        ).where(*conditions)
+        for src, dst, kind, line, confidence, edge_id, via in session.execute(stmt):
+            cache.forward.setdefault(src, []).append(
+                Hop(dst, kind.value, line, confidence, edge_id, via))
+            cache.reverse.setdefault(dst, []).append(
+                Hop(src, kind.value, line, confidence, edge_id, via))
+        for adjacency in (cache.forward, cache.reverse):
+            for hops in adjacency.values():
+                hops.sort(key=lambda h: (h.dst, h.line))
+        return cache
+
+    # ---------------- traversals ----------------
+
+    def neighborhood(self, starts: set[int], depth: int, direction: str) -> set[int]:
+        """All nodes within `depth` hops (excluding the starts themselves)."""
+        adjacency = self.forward if direction == "out" else self.reverse
+        seen = set(starts)
+        frontier = set(starts)
+        found: set[int] = set()
+        for _ in range(depth):
+            next_frontier: set[int] = set()
+            for node in frontier:
+                for hop in adjacency.get(node, ()):
+                    if hop.dst not in seen:
+                        seen.add(hop.dst)
+                        next_frontier.add(hop.dst)
+                        found.add(hop.dst)
+            if not next_frontier:
+                break
+            frontier = next_frontier
+        return found
+
+    def reachable(self, sources: set[int], sinks: set[int], max_depth: int) -> bool:
+        if sources & sinks:
+            return True
+        seen = set(sources)
+        frontier = deque((s, 0) for s in sources)
+        while frontier:
+            node, depth = frontier.popleft()
+            if depth >= max_depth:
+                continue
+            for hop in self.forward.get(node, ()):
+                if hop.dst in sinks:
+                    return True
+                if hop.dst not in seen:
+                    seen.add(hop.dst)
+                    frontier.append((hop.dst, depth + 1))
+        return False
+
+    def paths(
+        self,
+        sources: set[int],
+        sinks: set[int],
+        max_depth: int = 25,
+        max_paths: int = 10,
+    ) -> list[list[tuple[int, Hop | None]]]:
+        """Enumerate simple paths as [(symbol_id, hop_into_it | None), ...].
+
+        DFS with an on-path visited set; neighbor order is deterministic.
+        Results are sorted shortest-first. A global visit budget bounds work on
+        pathological graphs; enumeration also stops at max_paths.
+        """
+        results: list[list[tuple[int, Hop | None]]] = []
+        budget = _MAX_DFS_VISITS
+
+        for source in sorted(sources):
+            if budget <= 0 or len(results) >= max_paths:
+                break
+            stack: list[tuple[int, Hop | None]] = [(source, None)]
+            on_path = {source}
+
+            def dfs(node: int, depth: int) -> None:
+                nonlocal budget
+                if budget <= 0 or len(results) >= max_paths:
+                    return
+                budget -= 1
+                if node in sinks:
+                    results.append(list(stack))
+                    return
+                if depth >= max_depth:
+                    return
+                for hop in self.forward.get(node, ()):
+                    if hop.dst in on_path:
+                        continue
+                    stack.append((hop.dst, hop))
+                    on_path.add(hop.dst)
+                    dfs(hop.dst, depth + 1)
+                    on_path.discard(hop.dst)
+                    stack.pop()
+
+            dfs(source, 0)
+
+        results.sort(key=lambda p: (len(p), [n for n, _ in p]))
+        return results[:max_paths]

entrygraph/graph/cte.py

@@ -0,0 +1,123 @@
+"""Recursive-CTE reachability engine — the SQL-side fallback.
+
+Same paths()/reachable() contract as AdjacencyCache, but the traversal runs
+inside SQLite via a recursive CTE instead of loading adjacency into memory.
+Path reconstruction and cycle-guarding use an encoded path string, which is why
+this is the fallback (it does more work per row); the memory engine is primary.
+The two are kept behaviorally identical by the parametrized reachability tests.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy import bindparam, text
+from sqlalchemy.orm import Session
+
+from entrygraph.graph.adjacency import Hop
+from entrygraph.kinds import EdgeKind
+
+_SEP = ","
+
+
+class CteEngine:
+    def __init__(self, session: Session, kinds: frozenset[str], min_confidence: int = 0,
+                 include_cha: bool = True) -> None:
+        self.session = session
+        self.kind_values = [EdgeKind(k).value for k in kinds]
+        self.min_confidence = min_confidence
+        self.include_cha = include_cha
+
+    def reachable(self, sources: set[int], sinks: set[int], max_depth: int) -> bool:
+        if sources & sinks:
+            return True
+        return bool(self.paths(sources, sinks, max_depth=max_depth, max_paths=1))
+
+    def paths(
+        self, sources: set[int], sinks: set[int], max_depth: int = 25, max_paths: int = 10
+    ) -> list[list[tuple[int, Hop | None]]]:
+        results: list[list[tuple[int, Hop | None]]] = []
+        # a source that is itself a sink is a length-1 path (matches memory engine)
+        for src in sorted(sources & sinks):
+            results.append([(src, None)])
+
+        rows = self.session.execute(
+            text(
+                """
+                WITH RECURSIVE walk(node, nodes, lines, kinds, ids, confs, depth) AS (
+                    SELECT e.dst_symbol_id,
+                           :sep || e.src_symbol_id || :sep || e.dst_symbol_id || :sep,
+                           :sep || e.line || :sep,
+                           :sep || e.kind || :sep,
+                           :sep || e.id || :sep,
+                           :sep || e.confidence || :sep,
+                           1
+                    FROM edges e
+                    WHERE e.src_symbol_id IN :sources
+                      AND e.dst_symbol_id IS NOT NULL
+                      AND e.kind IN :kinds
+                      AND e.confidence >= :minconf
+                      AND (:include_cha OR e.via IS NULL OR e.via != 'cha')
+                    UNION ALL
+                    SELECT e.dst_symbol_id,
+                           w.nodes || e.dst_symbol_id || :sep,
+                           w.lines || e.line || :sep,
+                           w.kinds || e.kind || :sep,
+                           w.ids || e.id || :sep,
+                           w.confs || e.confidence || :sep,
+                           w.depth + 1
+                    FROM walk w
+                    JOIN edges e ON e.src_symbol_id = w.node
+                    WHERE w.depth < :max_depth
+                      AND e.dst_symbol_id IS NOT NULL
+                      AND e.kind IN :kinds
+                      AND e.confidence >= :minconf
+                      AND (:include_cha OR e.via IS NULL OR e.via != 'cha')
+                      AND w.nodes NOT LIKE '%' || :sep || e.dst_symbol_id || :sep || '%'
+                )
+                SELECT nodes, lines, kinds, ids, confs, depth FROM walk
+                WHERE node IN :sinks
+                ORDER BY depth
+                """
+            ).bindparams(
+                bindparam("sources", expanding=True),
+                bindparam("sinks", expanding=True),
+                bindparam("kinds", expanding=True),
+            ),
+            {
+                "sources": list(sources),
+                "sinks": list(sinks),
+                "kinds": self.kind_values,
+                "minconf": self.min_confidence,
+                "include_cha": 1 if self.include_cha else 0,
+                "max_depth": max_depth,
+                "sep": _SEP,
+            },
+        ).all()
+
+        for nodes_str, lines_str, kinds_str, ids_str, confs_str, _depth in rows:
+            if len(results) >= max_paths:
+                break
+            path = self._decode(nodes_str, lines_str, kinds_str, ids_str, confs_str)
+            if path is not None:
+                results.append(path)
+
+        results.sort(key=lambda p: (len(p), [n for n, _ in p]))
+        return results[:max_paths]
+
+    @staticmethod
+    def _decode(
+        nodes_str, lines_str, kinds_str, ids_str, confs_str
+    ) -> list[tuple[int, Hop | None]] | None:
+        node_ids = [int(x) for x in nodes_str.strip(_SEP).split(_SEP) if x]
+        lines = [int(x) for x in lines_str.strip(_SEP).split(_SEP) if x]
+        kinds = [x for x in kinds_str.strip(_SEP).split(_SEP) if x]
+        edge_ids = [int(x) for x in ids_str.strip(_SEP).split(_SEP) if x]
+        confs = [int(x) for x in confs_str.strip(_SEP).split(_SEP) if x]
+        if len(lines) != len(node_ids) - 1:
+            return None
+        path: list[tuple[int, Hop | None]] = [(node_ids[0], None)]
+        for i in range(1, len(node_ids)):
+            path.append(
+                (node_ids[i], Hop(node_ids[i], kinds[i - 1], lines[i - 1],
+                                  confs[i - 1], edge_ids[i - 1]))
+            )
+        return path

entrygraph/graph/scoring.py

@@ -0,0 +1,101 @@
+"""Query-time path risk scoring (heuristic taint tier).
+
+Everything here runs at query time, never at index time, so weights can be
+retuned without re-indexing. A path's risk combines: the terminal sink's
+severity, the weakest edge confidence along the path, path length, whether any
+hop is speculative (CHA/dynamic/callback), whether a sanitizer intervened,
+whether the sink was called with only constant arguments, and whether the
+source's user-controlled parameters are known.
+"""
+
+from __future__ import annotations
+
+import re
+
+from entrygraph.kinds import Confidence
+
+_SEVERITY_BASE = {"critical": 1.0, "high": 0.85, "medium": 0.6, "low": 0.35}
+_DEFAULT_SEVERITY = 0.6
+
+_CONFIDENCE_WEIGHT = {
+    int(Confidence.EXACT): 1.0,
+    int(Confidence.IMPORT): 0.95,
+    int(Confidence.FUZZY): 0.7,
+    int(Confidence.UNRESOLVED): 0.5,
+}
+
+_SPECULATIVE_VIA = {"cha", "dynamic"}
+_LENGTH_DECAY = 0.97
+
+# A literal-only argument preview: strings, numbers, bools, None, kwarg names,
+# and bracketed literal collections. Anything with an identifier/operator that
+# could carry a variable makes it non-constant.
+_CONST_TOKEN = re.compile(
+    r"""^(
+        \s | , | = | \( | \) | \[ | \] | \{ | \} | : |
+        '[^']*' | "[^"]*" | `[^`]*` |
+        \d[\d_.eExXaAbBcCdDfF]* |
+        True|False|None|null|true|false|nil |
+        [A-Za-z_]\w*\s*=          # kwarg name before '='
+    )*$""",
+    re.VERBOSE,
+)
+
+
+def is_constant_args(arg_preview: str | None) -> bool:
+    """True if the sink was called with only literal/constant arguments.
+
+    Conservative: an empty/None preview is constant (no args); a preview that was
+    truncated at the 80-char cap (trailing ellipsis) returns False because we
+    can't see the whole argument list.
+    """
+    if not arg_preview:
+        return True
+    text = arg_preview.strip()
+    if text.endswith("…") or text.endswith("..."):
+        return False
+    inner = text
+    if inner.startswith("(") and inner.endswith(")"):
+        inner = inner[1:-1]
+    if not inner.strip():
+        return True
+    return bool(_CONST_TOKEN.match(text))
+
+
+def confidence_factor(confidences: list[int]) -> float:
+    return min((_CONFIDENCE_WEIGHT.get(c, 0.5) for c in confidences), default=1.0)
+
+
+def score_path(
+    *,
+    hop_confidences: list[int],
+    hop_vias: list[str | None],
+    sink_severity: str | None,
+    sanitized_effect: str | None,  # "neutralizes" | "reduces" | None
+    constant_args: bool,
+    source_tainted: bool,
+) -> float:
+    """Return a risk score in [0, 1]; higher = riskier. See module docstring."""
+    severity_base = _SEVERITY_BASE.get(sink_severity or "", _DEFAULT_SEVERITY)
+    conf = confidence_factor(hop_confidences)
+    hops = max(len(hop_confidences), 1)
+    length_decay = _LENGTH_DECAY ** (hops - 1)
+    speculative = 0.85 if any(v in _SPECULATIVE_VIA for v in hop_vias) else 1.0
+    if sanitized_effect == "neutralizes":
+        sanitizer_factor = 0.0
+    elif sanitized_effect == "reduces":
+        sanitizer_factor = 0.3
+    else:
+        sanitizer_factor = 1.0
+    const_factor = 0.4 if constant_args else 1.0
+    source_factor = 1.0 if source_tainted else 0.9
+    risk = (
+        severity_base
+        * conf
+        * length_decay
+        * speculative
+        * sanitizer_factor
+        * const_factor
+        * source_factor
+    )
+    return round(risk, 4)

entrygraph/kinds.py

@@ -0,0 +1,51 @@
+"""Shared enums used by the ORM models, the extraction IR, and the public API.
+
+Kept dependency-free so worker processes can unpickle IR without importing
+SQLAlchemy machinery.
+"""
+
+from __future__ import annotations
+
+import enum
+
+
+class SymbolKind(enum.Enum):
+    MODULE = "module"
+    CLASS = "class"
+    FUNCTION = "function"
+    METHOD = "method"
+    VARIABLE = "variable"
+    CONSTANT = "constant"
+    INTERFACE = "interface"
+    STRUCT = "struct"
+    PROPERTY = "property"
+    FIELD = "field"
+    EXTERNAL = "external"
+
+
+class EdgeKind(enum.Enum):
+    CALLS = "calls"
+    IMPORTS = "imports"
+    INHERITS = "inherits"
+    IMPLEMENTS = "implements"
+    REFERENCES = "references"
+    PASSED_AS_CALLBACK = "callback"  # function name handed to another call as an argument
+
+
+class EntrypointKind(enum.Enum):
+    HTTP_ROUTE = "http_route"
+    CLI_COMMAND = "cli_command"
+    MAIN = "main"
+    TASK = "task"
+    LAMBDA_HANDLER = "lambda_handler"
+    EVENT_HANDLER = "event_handler"
+    MIDDLEWARE = "middleware"  # request/response interceptor (before_request, app.use, ...)
+
+
+class Confidence(enum.IntEnum):
+    """How a reference was bound to its target symbol."""
+
+    UNRESOLVED = 0  # kept with its textual target only
+    FUZZY = 1  # unique-name match, no import evidence
+    IMPORT = 2  # via the file's import map (project or external)
+    EXACT = 3  # same-scope / same-module / known-class-method

entrygraph/parsing/parsers.py

@@ -0,0 +1,49 @@
+"""Parser construction over tree-sitter-language-pack.
+
+We build tree_sitter.Parser objects from the pack's Language handles rather
+than using tree_sitter_language_pack.get_parser(), whose bundled Parser class
+is incompatible with the py-tree-sitter 0.25 API surface we use.
+
+Parsers and Language handles are cached per process; they are created lazily
+inside worker processes and must never be pickled.
+"""
+
+from __future__ import annotations
+
+from functools import cache
+
+from tree_sitter import Language, Parser
+
+# our language ids -> tree-sitter-language-pack grammar names
+_GRAMMAR_NAMES = {
+    "python": "python",
+    "javascript": "javascript",
+    "typescript": "typescript",
+    "tsx": "tsx",
+    "go": "go",
+    "java": "java",
+    "ruby": "ruby",
+    "csharp": "csharp",
+    "php": "php",
+    "rust": "rust",
+}
+
+
+def supported(lang_id: str) -> bool:
+    return lang_id in _GRAMMAR_NAMES
+
+
+@cache
+def language(lang_id: str) -> Language:
+    from tree_sitter_language_pack import get_language
+
+    return get_language(_GRAMMAR_NAMES[lang_id])
+
+
+@cache
+def parser(lang_id: str) -> Parser:
+    return Parser(language(lang_id))
+
+
+def parse(lang_id: str, source: bytes):
+    return parser(lang_id).parse(source)

entrygraph/parsing/queries.py

@@ -0,0 +1,39 @@
+"""Compiled tree-sitter query cache.
+
+The single seam between entrygraph and the py-tree-sitter query API — if the
+API shifts again (as it did at 0.24/0.25), this is the only file to update.
+`.scm` sources ship as package data under entrygraph/queries/<lang>/.
+"""
+
+from __future__ import annotations
+
+from functools import cache
+from importlib.resources import files as resource_files
+
+from tree_sitter import Node, Query, QueryCursor
+
+from entrygraph.parsing.parsers import language
+
+
+@cache
+def load_query_for(grammar_lang: str, source_lang: str, name: str) -> Query:
+    """Compile a `.scm` source (shipped under queries/<source_lang>/) against a
+    possibly-different grammar.
+
+    A tree-sitter Query is bound to one Language and only matches trees from that
+    same grammar. TypeScript/TSX are supersets of JavaScript with the same node
+    names for the constructs we harvest, so their extractor reuses the JavaScript
+    query sources compiled against the TS/TSX grammars — without this, a `.ts`
+    tree queried with JS-compiled queries silently matches nothing.
+    """
+    source = (resource_files("entrygraph") / "queries" / source_lang / f"{name}.scm").read_text()
+    return Query(language(grammar_lang), source)
+
+
+@cache
+def load_query(lang_id: str, name: str) -> Query:
+    return load_query_for(lang_id, lang_id, name)
+
+
+def captures(query: Query, node: Node) -> dict[str, list[Node]]:
+    return QueryCursor(query).captures(node)