codebase-explorer 2.2.0__py3-none-any.whl

cbe/graph_backends/__init__.py

@@ -0,0 +1,43 @@
+"""Backend registry for the public ``cbe-graph/1`` graph analyzer.
+
+Each backend wraps one upstream open-source analyzer and returns a fragment in
+the standard node/edge shape.  ``cbe.graph`` merges fragments, owns the schema
+and never re-implements language analysis.
+"""
+
+from __future__ import annotations
+
+from ._paths import canonical_repo_path
+from .imports import (
+    DEPCRUISE_LANGUAGES,
+    build_with_depcruise,
+    build_with_grimp,
+    find_depcruise,
+)
+from .joern import (
+    FRONTEND_NAME,
+    JOERN_BYTECODE_FRONTENDS,
+    JOERN_LANGUAGE_FLAGS,
+    JoernError,
+    build_with_joern,
+    joern_version,
+)
+from .toolchain import ToolchainError, find_java, find_joern, setup_hint
+
+__all__ = [
+    "DEPCRUISE_LANGUAGES",
+    "FRONTEND_NAME",
+    "JOERN_BYTECODE_FRONTENDS",
+    "JOERN_LANGUAGE_FLAGS",
+    "JoernError",
+    "ToolchainError",
+    "build_with_depcruise",
+    "build_with_grimp",
+    "build_with_joern",
+    "canonical_repo_path",
+    "find_depcruise",
+    "find_java",
+    "find_joern",
+    "joern_version",
+    "setup_hint",
+]

cbe/graph_backends/_calls.py

@@ -0,0 +1,262 @@
+"""One call-resolution policy for normalized edges and capability counts."""
+from __future__ import annotations
+
+import ast
+import builtins
+from pathlib import PurePosixPath
+import posixpath
+import re
+
+
+_DOTTED_TYPE = re.compile(r"(?:[A-Za-z_]\w*\.)+([A-Za-z_]\w*)")
+
+
+def csharp_signature_key(full: str) -> str:
+    """One key for both spellings of a C# method's signature.
+
+    Joern's C# frontend writes a repository type with its namespace in the method's own full
+    name (``N.Guard.NotNull:T(N.IGuardClause,T)``) but without it in the full name of a call
+    to that method (``N.Guard.NotNull:T(IGuardClause,T)``). The method's qualified name before
+    the colon is kept as it is; inside the signature every dotted type name, generic
+    arguments included, is cut to its last component.
+    """
+    head, colon, signature = full.partition(":")
+    return head + colon + _DOTTED_TYPE.sub(r"\1", signature) if colon else full
+
+
+_UNKNOWN_NAMES = ("<unresolvedNamespace>", "<unresolvedName>", "<unresolvedSignature>",
+                  "<unknownFullName>", "<empty>")
+
+
+class CallResolver:
+    def __init__(self, methods: list[dict], files=None, types=None, go_modules=None,
+                 javascript_packages=None):
+        self.methods = {int(m["nodeId"]): m for m in methods if m.get("nodeId") is not None}
+        self.method_names = {m.get("name") for m in methods}
+        self.files = set(files) if files is not None else {m.get("file", "") for m in methods if m.get("file")}
+        self.diagnostics: list[dict] = []
+        self.go_modules = go_modules or {}
+        self.javascript_packages = set(javascript_packages or ())
+        self.python_modules = set()
+        self.python_top_modules = set()
+        for path in self.files:
+            if path.endswith(".py"):
+                module = path[:-3].removesuffix("/__init__").removeprefix("src/")
+                self.python_modules.add(PurePosixPath(path).stem)
+                top = module.split("/", 1)[0]
+                if top:
+                    self.python_top_modules.add(top)
+                    self.python_modules.add(top)
+        self.by_file: dict[str, dict[str, list[int]]] = {}
+        self.by_pair: dict[str, list[int]] = {}
+        self.by_full: dict[str, list[int]] = {}
+        self.by_csharp_key: dict[str, list[int]] = {}
+        self.by_csharp_name: dict[str, list[int]] = {}
+        self.internal_prefixes = set()
+        self.classes = {"Self"}
+        method_names = {m.get("fullName") for m in methods}
+        # pysrc2cpg also emits TYPE_DECL for every function. Those entries
+        # have the exact identity of a method, unlike their owning class.
+        self.types = {t["fullName"]: t for t in (types or []) if t.get("file") in self.files
+                      and not (str(t.get("file", "")).endswith(".py") and t["fullName"] in method_names)}
+        self.types_by_name = {}
+        for full, entry in self.types.items():
+            name = entry.get("name") or full.split(".")[-1]
+            self.internal_prefixes.add(full)
+            if "." in full and not str(entry.get("file", "")).endswith(".py"):
+                self.internal_prefixes.add(full.rsplit(".", 1)[0])
+            self.classes.add(name)
+            self.types_by_name.setdefault(name, []).append(full)
+        for native, method in self.methods.items():
+            full = method.get("fullName", "")
+            owner = full.rsplit(".", 1)[0]
+            if (owner and owner != full and "<clinit>" not in full
+                    and not str(method.get("file", "")).endswith(".py")):
+                self.internal_prefixes.add(owner)
+            self.by_full.setdefault(full, []).append(native)
+            if str(method.get("file", "")).endswith(".cs") and ":" in full:
+                self.by_csharp_key.setdefault(csharp_signature_key(full), []).append(native)
+                self.by_csharp_name.setdefault(full.partition(":")[0], []).append(native)
+            by_file = self.by_file.setdefault(method.get("file", ""), {})
+            by_file.setdefault(full.partition(":")[2], []).append(native)
+            member = full.rsplit(":", 1)[-1]
+            if member != full.partition(":")[2]:
+                by_file.setdefault(member, []).append(native)
+            # Python frontend: path.py:<module>.Class.method. Do not match
+            # a bare method name, which confuses unrelated receiver types.
+            name = full.split(":")[-1]
+            parts = name.split(".")
+            if len(parts) >= 2 and not any(p.startswith("<") for p in parts[-2:]):
+                self.by_pair.setdefault(".".join(parts[-2:]), []).append(native)
+                if (str(method.get("file", "")).endswith(".py") and ":<module>." in full
+                        and full.rsplit(".", 1)[0] not in method_names):
+                    self.classes.add(parts[-2])
+
+    def inherited_targets(self, receiver, name, call):
+        if receiver in {"Self", "te.Self"}:
+            native_caller = call.get("callerId")
+            caller = self.methods.get(int(native_caller), {}) if native_caller is not None else {}
+            owners = [caller.get("fullName", "").rsplit(".", 1)[0]]
+        else:
+            owners = self.types_by_name.get(receiver, [])
+        if len(owners) != 1:
+            return []
+        visited = set()
+        def search(owner):
+            if owner in visited:
+                return []
+            visited.add(owner)
+            direct = self.by_full.get(owner + "." + name, [])
+            if direct:
+                return direct
+            targets = []
+            for base in self.types.get(owner, {}).get("inherits", []):
+                # Only traverse exact analyzer type identities. A short-name
+                # collision is insufficient evidence of an inheritance edge.
+                targets.extend(search(base))
+            return targets
+        return list(dict.fromkeys(search(owners[0])))
+
+    def resolve(self, call: dict) -> tuple[str, list[int], list[str]]:
+        result = self._resolve(call)
+        status, _, external_names = result
+        # Enforce Python's external identity contract at the single exit,
+        # including future policy branches that return an external target.
+        if (status == "external" and str(call.get("callerFile", "")).endswith(".py")
+                and (not external_names or any(".py:" not in name and not name.startswith("__builtin")
+                                               for name in external_names))):
+            self.diagnostics.append({
+                "path": call.get("callerFile"), "line": call.get("line"),
+                "code": "python_external_name_unverified", "analyzer_id": call.get("nodeId"),
+                "message": f"Python external target lacks a library identity: {external_names!r}",
+            })
+            return "unresolved", [], []
+        return result
+
+    def _resolve(self, call: dict) -> tuple[str, list[int], list[str]]:
+        callees = call.get("callees") or []
+        internal = list(dict.fromkeys(int(c["nodeId"]) for c in callees
+                                     if not c.get("external") and c.get("nodeId") is not None
+                                     and int(c["nodeId"]) in self.methods))
+        full = call.get("methodFullName") or ""
+        if internal:
+            # Name-only binding with an unknown receiver is not a resolution.
+            if full == "<unknownFullName>":
+                return "ambiguous", internal, []
+            return ("resolved" if len(internal) == 1 else "ambiguous", internal, [])
+        # Python's frontend leaves builtin exception constructors without any
+        # callee and with an unknown full name. A known builtin expression is
+        # stronger evidence than that missing analyzer name.
+        if not callees and str(call.get("callerFile", "")).endswith(".py"):
+            try:
+                expression = ast.parse(call.get("code", ""), mode="eval").body
+            except SyntaxError:
+                expression = None
+            if isinstance(expression, ast.Call) and isinstance(expression.func, ast.Name):
+                name = expression.func.id
+                if callable(getattr(builtins, name, None)):
+                    return "external", [], ["__builtin." + name]
+        if any(marker in full for marker in _UNKNOWN_NAMES) or any(
+                any(marker in str(c.get("fullName") or "") for marker in _UNKNOWN_NAMES) for c in callees):
+            return "unresolved", [], []
+        relative = self._relative_typescript_target(call, full)
+        if relative is not None:
+            return relative
+        # A leading // is a relative import whose package prefix was lost.
+        # Search only package ancestors; an unrelated basename is not evidence.
+        if full.startswith("//") and ".py:<module>." in full and str(call.get("callerFile", "")).endswith(".py"):
+            relative, _, member = full[2:].partition(":")
+            caller = PurePosixPath(call["callerFile"])
+            candidates = {str(parent / relative) for parent in caller.parents} & self.files
+            if len(candidates) == 1:
+                matches = self.by_file.get(next(iter(candidates)), {}).get(member, [])
+                if len(matches) == 1:
+                    return "resolved", matches, []
+            return "unresolved", [], []
+        exact = self.by_full.get(full, [])
+        if exact:
+            return ("resolved" if len(exact) == 1 else "ambiguous", exact, [])
+        if str(call.get("callerFile", "")).endswith(".cs") and ":" in full:
+            matches = self.by_csharp_key.get(csharp_signature_key(full), [])
+            if matches:
+                # The frontend's signature for the call is not reliable enough to pick an
+                # overload (it types -1L and 0.0 as Int32), so an overloaded method is a
+                # candidate set, not a resolution.
+                overloads = self.by_csharp_name.get(full.partition(":")[0], [])
+                if len(matches) == 1 and len(overloads) == 1:
+                    return "resolved", matches, []
+                return "ambiguous", sorted(set(matches) | set(overloads)), []
+        # A repository-qualified miss is analyzer guesswork, not a known
+        # external target. Never salvage its last two components.
+        if self._repository_qualified(full):
+            return "unresolved", [], []
+        if re.fullmatch(r"[A-Za-z_]\w*\.[A-Za-z_]\w*", full):
+            matches = self.by_pair.get(full, [])
+            if matches:
+                return ("resolved" if len(matches) == 1 else "ambiguous", matches, [])
+        receiver, _, name = full.rpartition(".")
+        if ((".py:" not in full and full.split(".", 1)[0] in self.classes)
+                or full.startswith("te.Self.")):
+            matches = self.inherited_targets(receiver, name, call)
+            if matches:
+                return ("resolved" if len(matches) == 1 else "ambiguous", matches, [])
+            return "unresolved", [], []
+        if (".py:" not in full and not full.startswith("__builtin")
+                and ("<returnValue>" in full or "<member>(" in full
+                     or (str(call.get("callerFile", "")).endswith(".py")
+                         and full.split(".", 1)[0] in self.python_modules))):
+            return "unresolved", [], []
+        external = list(dict.fromkeys(c.get("fullName") or call.get("name", "") for c in callees if c.get("external")))
+        if (full.startswith("ANY.") and str(call.get("callerFile", "")).endswith(
+                (".cc", ".cpp", ".cxx", ".h", ".hpp", ".hh", ".go"))):
+            return "unresolved", [], []
+        if str(call.get("callerFile", "")).endswith((".cc", ".cpp", ".cxx", ".h", ".hpp", ".hh")):
+            if (not full.startswith(("std.", "__gnu"))
+                    and re.fullmatch(r"(?:[A-Za-z_]\w*\.)*([A-Za-z_]\w*)\.\1(?::.*)?", full)):
+                return "unresolved", [], []
+            if full.split(":", 1)[0] in self.method_names:
+                return "unresolved", [], []
+        if any(self._repository_qualified(name) for name in external):
+            return "unresolved", [], []
+        # An internal callee missing from the exported method set is unknown.
+        return ("external", [], external) if external else ("unresolved", [], [])
+
+    def _repository_qualified(self, full: str) -> bool:
+        if any(full.startswith(path + ":") for path in self.files):
+            return True
+        if ".py:" in full:
+            module = full.split(".py:", 1)[0]
+            if any(module == top or module.startswith(top + "/") for top in self.python_top_modules):
+                return True
+        if any(full == module or full.startswith(module + ".") or full.startswith(module + "/")
+               for module in self.go_modules):
+            return True
+        if any(full == package or full.startswith(package + ":") for package in self.javascript_packages):
+            return True
+        return any(prefix and full.startswith(prefix + ".") for prefix in self.internal_prefixes)
+
+    def _relative_typescript_target(self, call: dict, full: str):
+        path = str(call.get("callerFile", ""))
+        if not path.endswith((".ts", ".tsx", ".mts", ".cts")) or ":" not in full:
+            return None
+        relative, _, member = full.partition(":")
+        if not relative.endswith((".js", ".jsx", ".mjs", ".cjs")):
+            return None
+        stem = posixpath.normpath(posixpath.join(posixpath.dirname(path), relative))
+        if stem == ".." or stem.startswith("../"):
+            return "unresolved", [], []
+        replacements = {".js": (".ts", ".tsx"), ".jsx": (".tsx",),
+                        ".mjs": (".mts",), ".cjs": (".cts",)}
+        suffix = next(suffix for suffix in replacements if stem.endswith(suffix))
+        base = stem[:-len(suffix)]
+        candidates = [base + ext for ext in replacements[suffix]]
+        candidates += [base + "/index" + ext for ext in (".ts", ".tsx")]
+        candidates = [candidate for candidate in candidates if candidate in self.files]
+        matches = list(dict.fromkeys(
+            native for candidate in candidates
+            for native in self.by_file.get(candidate, {}).get(member, [])))
+        if len(matches) == 1:
+            return "resolved", matches, []
+        if len(matches) > 1:
+            return "ambiguous", matches, []
+        return ("unresolved", [], []) if candidates else None

cbe/graph_backends/_paths.py

@@ -0,0 +1,42 @@
+"""Canonical repo-relative path handling shared by the graph layer.
+
+``lstrip("./")`` is wrong: it turns ``../foo`` into ``foo`` and ``.hidden``
+into ``hidden``.  A query or manifest path must be a normalized POSIX path
+inside the repository, or be rejected.
+"""
+
+from __future__ import annotations
+
+import re
+
+_WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:")
+
+
+def canonical_repo_path(raw: object) -> str | None:
+    """Return a canonical repo-relative POSIX path, or ``None`` if unsafe.
+
+    ``./a/b`` -> ``a/b``; ``a/../b`` -> ``b``; ``../a`` -> ``None`` (escapes
+    the root); absolute paths and Windows drive paths -> ``None``.  Dot-files
+    such as ``.gitignore`` are preserved.
+    """
+
+    if raw is None:
+        return None
+    text = str(raw).replace("\\", "/").strip()
+    if not text:
+        return None
+    if text.startswith("/") or _WINDOWS_DRIVE.match(text):
+        return None
+    parts: list[str] = []
+    for part in text.split("/"):
+        if part in ("", "."):
+            continue
+        if part == "..":
+            if not parts:
+                return None
+            parts.pop()
+            continue
+        parts.append(part)
+    if not parts:
+        return None
+    return "/".join(parts)

cbe/graph_backends/identity.py

@@ -0,0 +1,93 @@
+"""Repository file membership and canonical graph identities.
+
+All backends and the merge boundary use the same excluded-directory policy.
+A file identity can only be constructed for an inventoried repository file.
+"""
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+
+from cbe.grammars import DEFAULT_EXCLUDE_DIR_NAMES
+from ._paths import canonical_repo_path
+
+
+def repository_files(repo: Path) -> frozenset[str]:
+    repo = repo.resolve()
+    files = set()
+    for directory, dirs, names in os.walk(repo, followlinks=False):
+        dirs[:] = sorted(d for d in dirs if d not in DEFAULT_EXCLUDE_DIR_NAMES
+                         and not (Path(directory) / d).is_symlink())
+        for name in names:
+            full = Path(directory) / name
+            if full.is_file() and not full.is_symlink():
+                files.add(full.relative_to(repo).as_posix())
+    return frozenset(files)
+
+
+def file_id(path: str | None, files) -> str | None:
+    return "file:" + path if path and path in files and canonical_repo_path(path) == path else None
+
+
+def external_id(name: str) -> str:
+    return "ext:" + name
+
+
+def validate_identities(nodes, edges, files):
+    """Validate backend identities; import resolution belongs to the resolver."""
+    aliases, invalid, diagnostics, result = {}, set(), [], []
+    for node in nodes:
+        native = node["id"]
+        if native.startswith("file:"):
+            canonical = file_id(node.get("path"), files)
+            if canonical is None:
+                invalid.add(native)
+                diagnostics.append(dict(path=node.get("path"), code="invalid_file_identity",
+                                        message=f"discarded non-repository file identity {native}"))
+                continue
+            aliases[native] = canonical
+            node = {**node, "id": canonical}
+        result.append(node)
+    unique = {n["id"]: n for n in result}
+    repaired = []
+    include_evidence = {}
+    for edge in edges:
+        if edge.get("kind") != "import" or edge.get("status") != "resolved":
+            continue
+        key = (edge.get("source"), edge.get("target"))
+        evidence = include_evidence.setdefault(key, {"angle": set(), "quoted": set()})
+        for expression in edge.get("expressions") or [edge.get("expression") or ""]:
+            match = re.fullmatch(r'#\s*include\s*([<"])([^>"]+)[>"]\s*', expression)
+            if match:
+                delimiter, token = match.groups()
+                if delimiter == '<' and '/' not in token and not edge.get("resolution_basis"):
+                    evidence["angle"].add(token)
+                elif delimiter == '"':
+                    evidence["quoted"].add(token)
+    for (source, target), evidence in include_evidence.items():
+        if evidence["angle"] and evidence["quoted"]:
+            diagnostics.append(dict(path=source, code="external_header_collision",
+                message=f"external header and repository header share target {target}",
+                angle_headers=sorted(evidence["angle"]), quoted_headers=sorted(evidence["quoted"])))
+    for edge in edges:
+        edge = dict(edge)
+        degraded = False
+        for endpoint in ("source", "target"):
+            value = edge.get(endpoint)
+            if value in invalid:
+                edge[endpoint] = None
+                degraded = True
+            elif value in aliases:
+                edge[endpoint] = aliases[value]
+                if endpoint == "target" and edge.get("kind") == "import":
+                    edge["status"] = "resolved"
+        if degraded:
+            edge["status"] = "unresolved"
+        if (edge.get("kind") == "import" and edge.get("source") is not None
+                and edge.get("source") == edge.get("target")):
+            diagnostics.append(dict(path=edge.get("path"), code="self_import_suppressed",
+                message=f"discarded self import {edge['source']}", expression=edge.get("expression")))
+            continue
+        repaired.append(edge)
+    return list(unique.values()), repaired, diagnostics