coretrace-python-analyzer 0.1.0__py3-none-any.whl

coretrace_python/interprocedural/callgraph.py

@@ -0,0 +1,291 @@
+"""Intra-module call graph (architecture §20).
+
+Every call site of every analysable function resolves, through the SSA form, to a
+``KnownFunction`` defined at module level, an ``ExternalSymbol`` reached through imports
+or builtins, or ``UnknownTarget`` (parameters, attributes, methods) until type inference
+and framework models narrow it down.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import ClassVar
+
+from coretrace_python.analysis import Analysis, AnalysisContext, AnyAnalysis
+from coretrace_python.cfg import CFGError
+from coretrace_python.hir import nodes
+from coretrace_python.ir.lowering import LoweringError, analyzable_functions, qualified_name
+from coretrace_python.ir.model import (
+    Call,
+    FunctionIR,
+    GetAttr,
+    GetItem,
+    Global,
+    MakeFunction,
+    Symbol,
+    Value,
+    WithEnter,
+)
+from coretrace_python.ir.ssa import SSAAnalysis
+from coretrace_python.semantic.scopes import BindingKind, ScopeAnalysis, ScopeTable
+from coretrace_python.semantic.symbols import SymbolAnalysis, SymbolId, SymbolTable
+from coretrace_python.source import SourceSpan
+
+
+@dataclass(frozen=True)
+class KnownFunction:
+    name: str
+
+
+@dataclass(frozen=True)
+class ExternalSymbol:
+    symbol: SymbolId
+
+
+@dataclass(frozen=True)
+class UnknownTarget:
+    pass
+
+
+Target = KnownFunction | ExternalSymbol | UnknownTarget
+
+
+@dataclass(frozen=True)
+class CallSite:
+    caller: str
+    location: SourceSpan
+    target: Target
+    arguments: int
+    keywords: int
+
+
+class CallGraph:
+    def __init__(
+        self,
+        definitions: Mapping[str, nodes.Function],
+        sites: Mapping[str, tuple[CallSite, ...]],
+        unsupported: frozenset[str],
+        symbols: Mapping[str, Mapping[Value, SymbolId]] | None = None,
+    ) -> None:
+        self._symbols = {name: MappingProxyType(dict(found)) for name, found in (symbols or {}).items()}
+        self.definitions: Mapping[str, nodes.Function] = MappingProxyType(dict(definitions))
+        # A graph rebuilt from cached call sites has sites but no definitions.
+        self.functions = tuple(dict.fromkeys((*definitions, *sites)))
+        self._names = {function.span: name for name, function in definitions.items()}
+        self.unsupported = unsupported
+        self._sites = MappingProxyType(dict(sites))
+        self._targets = {
+            (site.caller, site.location): site.target for found in sites.values() for site in found
+        }
+        callers: dict[str, set[str]] = {name: set() for name in definitions}
+        for found in sites.values():
+            for site in found:
+                if isinstance(site.target, KnownFunction):
+                    callers.setdefault(site.target.name, set()).add(site.caller)
+        self._callers = {name: frozenset(found) for name, found in callers.items()}
+
+    def name_of(self, function: nodes.Function) -> str:
+        return self._names[function.span]
+
+    def symbols(self, name: str) -> Mapping[Value, SymbolId]:
+        """Values of ``name`` that denote a symbol, including derived call-chain symbols."""
+
+        return self._symbols.get(name, MappingProxyType({}))
+
+    def sites(self, caller: str) -> tuple[CallSite, ...]:
+        return self._sites.get(caller, ())
+
+    def target_at(self, caller: str, location: SourceSpan) -> Target:
+        return self._targets.get((caller, location), UnknownTarget())
+
+    def callees(self, caller: str) -> frozenset[str]:
+        return frozenset(
+            site.target.name for site in self.sites(caller) if isinstance(site.target, KnownFunction)
+        )
+
+    def callers(self, name: str) -> frozenset[str]:
+        return self._callers.get(name, frozenset())
+
+
+def derive_symbols(
+    function: FunctionIR, initial: Mapping[Value, SymbolId] | None = None
+) -> dict[Value, SymbolId]:
+    """Symbols of values: ``Symbol`` results, attributes and items of symbol values,
+    results of calling a symbol (``sqlite3.connect(p)`` denotes ``python.sqlite3.connect``)
+    and the values a ``with`` on such a result binds. Known functions derive nothing;
+    parameters only through ``initial``, their annotated classes."""
+
+    symbols: dict[Value, SymbolId] = dict(initial or {})
+    changed = True
+    while changed:
+        changed = False
+        for block in function.blocks:
+            for instruction in block.instructions:
+                if instruction.result is None or instruction.result in symbols:
+                    continue
+                symbol: SymbolId | None = None
+                if isinstance(instruction, Symbol):
+                    symbol = instruction.symbol_id
+                elif isinstance(instruction, GetAttr) and instruction.object in symbols:
+                    symbol = symbols[instruction.object].attribute(instruction.attribute)
+                elif isinstance(instruction, GetItem) and instruction.object in symbols:
+                    # An item of a symbol-denoted container (``request.files['f']``)
+                    # carries the container's symbol, so its methods resolve.
+                    symbol = symbols[instruction.object]
+                elif isinstance(instruction, Call | WithEnter):
+                    origin = (
+                        instruction.callee if isinstance(instruction, Call) else instruction.context
+                    )
+                    symbol = symbols.get(origin)
+                if symbol is not None:
+                    symbols[instruction.result] = symbol
+                    changed = True
+    return symbols
+
+
+def resolve_targets(
+    function: FunctionIR,
+    scopes: ScopeTable,
+    known: frozenset[str],
+    parameters: Mapping[Value, SymbolId] | None = None,
+    nested: frozenset[str] = frozenset(),
+    classes: Mapping[str, frozenset[str]] | None = None,
+    owner: str | None = None,
+    typed: Mapping[Value, str] | None = None,
+) -> tuple[dict[Value, Target], dict[Value, SymbolId]]:
+    """Map every callee value of ``function`` to its target, and every symbol value.
+    ``nested`` names the functions defined inside this one (``outer.inner``);
+    ``classes`` maps the module's classes to their methods, ``owner`` is the class of a
+    method and ``typed`` the parameters annotated with a module class. ``App(x)`` is a
+    call to ``App.__init__``, ``app.run()`` and ``self.run()`` calls to ``App.run``."""
+
+    module = scopes.module_scope
+    symbols = derive_symbols(function, parameters)
+    targets: dict[Value, Target] = {
+        value: ExternalSymbol(symbol) for value, symbol in symbols.items()
+    }
+    classes = classes or {}
+    instance_of: dict[Value, str] = dict(typed or {})
+    if owner is not None and function.parameters:
+        instance_of[function.parameters[0]] = owner
+    defs = {i.result: i for block in function.blocks for i in block.instructions if i.result}
+    for block in function.blocks:
+        for instruction in block.instructions:
+            if isinstance(instruction, MakeFunction):
+                qualified = f"{function.name}.{instruction.name}"
+                if qualified in nested:
+                    targets[instruction.result] = KnownFunction(qualified)
+            elif isinstance(instruction, Global):
+                binding = module.bindings.get(instruction.name)
+                if binding is None:
+                    continue
+                if binding.kind is BindingKind.FUNCTION and instruction.name in known:
+                    targets[instruction.result] = KnownFunction(instruction.name)
+                elif binding.kind is BindingKind.CLASS and "__init__" in classes.get(instruction.name, ()):
+                    targets[instruction.result] = KnownFunction(f"{instruction.name}.__init__")
+            elif isinstance(instruction, Call):
+                callee = defs.get(instruction.callee)
+                if isinstance(callee, Global) and callee.name in classes:
+                    instance_of[instruction.result] = callee.name
+            elif isinstance(instruction, GetAttr) and instruction.object in instance_of:
+                method = f"{instance_of[instruction.object]}.{instruction.attribute}"
+                if method in nested:
+                    targets[instruction.result] = KnownFunction(method)
+    return targets, symbols
+
+
+def _annotated(
+    function: nodes.Function, ssa: FunctionIR, scopes: ScopeTable, table: SymbolTable
+) -> dict[Value, SymbolId]:
+    """Parameters annotated with a resolvable class denote that class (``db: Session``)."""
+
+    scope = scopes.scope_for(function)
+    enclosing = scope.parent if scope.parent is not None else scope.id
+    found: dict[Value, SymbolId] = {}
+    # Captured variables follow the explicit parameters and carry no annotation.
+    for value, parameter in zip(ssa.parameters, function.parameters, strict=False):
+        if parameter.annotation is None:
+            continue
+        symbol = table.resolve_expression(enclosing, parameter.annotation)
+        if symbol is not None:
+            found[value] = symbol
+    return found
+
+
+def _typed_with_module_classes(
+    function: nodes.Function, ssa: FunctionIR, classes: Mapping[str, frozenset[str]]
+) -> dict[Value, str]:
+    """Parameters annotated with a class of this module (``a: App``)."""
+
+    found: dict[Value, str] = {}
+    for value, parameter in zip(ssa.parameters, function.parameters, strict=False):
+        annotation = parameter.annotation
+        if isinstance(annotation, nodes.Name) and annotation.identifier in classes:
+            found[value] = annotation.identifier
+    return found
+
+
+class CallGraphAnalysis(Analysis[CallGraph]):
+    name: ClassVar[str] = "interprocedural.callgraph"
+    requires: ClassVar[frozenset[AnyAnalysis]] = frozenset({SSAAnalysis, ScopeAnalysis, SymbolAnalysis})
+
+    @classmethod
+    def compute(cls, ctx: AnalysisContext) -> CallGraph:
+        scopes = ctx.get(ScopeAnalysis)
+        table = ctx.get(SymbolAnalysis)
+        definitions: dict[str, nodes.Function] = {}
+        for function in analyzable_functions(ctx.module):
+            # A property and its setter, or a redefinition, share a qualified name; each
+            # definition still needs a name of its own.
+            base = unique = qualified_name(scopes, function)
+            ordinal = 2
+            while unique in definitions:
+                unique = f"{base}__{ordinal}"
+                ordinal += 1
+            definitions[unique] = function
+        known = frozenset(
+            name for name, function in definitions.items() if function in ctx.module.body
+        )
+        classes = {
+            statement.name: frozenset(m.name for m in statement.body if isinstance(m, nodes.Function))
+            for statement in ctx.module.body
+            if isinstance(statement, nodes.Class)
+        }
+        sites: dict[str, tuple[CallSite, ...]] = {}
+        symbols: dict[str, Mapping[Value, SymbolId]] = {}
+        unsupported: set[str] = set()
+        for name, function in definitions.items():
+            try:
+                ssa = ctx.get(SSAAnalysis, function)
+            except (LoweringError, CFGError):
+                unsupported.add(name)
+                sites[name] = ()
+                continue
+            owner = name.rsplit(".", 1)[0] if "." in name and name.rsplit(".", 1)[0] in classes else None
+            targets, symbols[name] = resolve_targets(
+                ssa,
+                scopes,
+                known,
+                _annotated(function, ssa, scopes, table),
+                frozenset(definitions),
+                classes,
+                owner,
+                _typed_with_module_classes(function, ssa, classes),
+            )
+            found: list[CallSite] = []
+            for block in ssa.blocks:
+                for instruction in block.instructions:
+                    if isinstance(instruction, Call):
+                        found.append(
+                            CallSite(
+                                name,
+                                instruction.location,
+                                targets.get(instruction.callee, UnknownTarget()),
+                                len(instruction.arguments),
+                                len(instruction.keywords),
+                            )
+                        )
+            sites[name] = tuple(found)
+        return CallGraph(definitions, sites, frozenset(unsupported), symbols)

coretrace_python/interprocedural/modulegraph.py

@@ -0,0 +1,218 @@
+"""Module graph and project-wide summary index (architecture §21).
+
+A project is a directory of Python files. Each file is one module named after its
+packages (``app/helpers.py`` is ``app.helpers``); the graph records which project modules
+each module imports. Functions defined in the project get project symbols
+(``python.app.helpers.run``) whose summaries live in a ``SummaryIndex`` that the engine
+provides to every module's manager, so calls into other files are analysed through
+summaries rather than by retaining every module's PyIR.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from types import MappingProxyType
+
+from coretrace_python.hir import nodes
+from coretrace_python.interprocedural.summaries import ProjectSummaries, SummaryIndex
+from coretrace_python.semantic.imports import ImportTable
+from coretrace_python.semantic.symbols import SymbolId
+from coretrace_python.source import SourceFile, SourceManager
+
+IGNORED_DIRECTORIES = frozenset({"__pycache__", "node_modules", "venv", "build", "dist", "site-packages"})
+
+
+def discover_files(root: Path, pattern: str = "*") -> tuple[Path, ...]:
+    """Every file matching ``pattern`` under ``root``, skipping hidden and tooling
+    directories and virtual environments, recognised by the ``pyvenv.cfg`` at their root
+    whatever their name."""
+
+    found: list[Path] = []
+    environments: dict[Path, bool] = {}
+    for path in sorted(root.rglob(pattern)):
+        if not path.is_file():
+            continue
+        relative = path.relative_to(root)
+        if any(p.startswith(".") or p in IGNORED_DIRECTORIES for p in relative.parts[:-1]):
+            continue
+        if any(_is_environment(root / Path(*relative.parts[:depth]), environments) for depth in range(1, len(relative.parts))):
+            continue
+        found.append(path)
+    return tuple(found)
+
+
+def discover_sources(
+    root: Path, manager: SourceManager, errors: list[tuple[Path, str]] | None = None
+) -> tuple[SourceFile, ...]:
+    """Load every ``.py`` file under ``root`` (see ``discover_files``). A file that cannot
+    be read or decoded is skipped and reported in ``errors``: Python could not import it
+    either."""
+
+    found: list[SourceFile] = []
+    for path in discover_files(root, "*.py"):
+        try:
+            found.append(manager.load_file(path))
+        except (OSError, UnicodeDecodeError) as error:
+            if errors is None:
+                raise
+            errors.append((path, str(error)))
+    return tuple(found)
+
+
+def _is_environment(directory: Path, known: dict[Path, bool]) -> bool:
+    if directory not in known:
+        known[directory] = (directory / "pyvenv.cfg").is_file()
+    return known[directory]
+
+
+def project_symbol(module_name: str, qualified_name: str) -> SymbolId:
+    """The canonical symbol of a project function. A module whose name is not an
+    identifier cannot be imported by that name, but its functions still need stable
+    symbols: invalid characters become underscores."""
+
+    module = ".".join(_component(part) for part in module_name.split("."))
+    return SymbolId(f"python.{module}.{qualified_name}")
+
+
+def _component(part: str) -> str:
+    if part.isidentifier():
+        return part
+    cleaned = "".join(c if (c.isalnum() or c == "_") else "_" for c in part)
+    return cleaned if cleaned.isidentifier() else f"_{cleaned}"
+
+
+@dataclass(frozen=True)
+class ModuleGraph:
+    _sources: Mapping[str, SourceFile]
+    _imports: Mapping[str, frozenset[str]]
+
+    @property
+    def modules(self) -> tuple[str, ...]:
+        return tuple(sorted(self._sources))
+
+    def source(self, name: str) -> SourceFile:
+        return self._sources[name]
+
+    def imports(self, name: str) -> frozenset[str]:
+        return self._imports.get(name, frozenset())
+
+    def importers(self, name: str) -> frozenset[str]:
+        return frozenset(m for m, imported in self._imports.items() if name in imported)
+
+    def schedule(self) -> tuple[tuple[frozenset[str], ...], ...]:
+        """Strongly connected components in waves: every component of a wave imports,
+        outside itself, only components of earlier waves, so one wave can be analysed
+        in parallel and a component's imports are final when it starts (§29)."""
+
+        names = frozenset(self._sources)
+        edges = {name: sorted(m for m in self.imports(name) if m in names) for name in names}
+        component_of: dict[str, int] = {}
+        components: list[frozenset[str]] = []
+        index: dict[str, int] = {}
+        low: dict[str, int] = {}
+        stack: list[str] = []
+
+        def visit(name: str) -> None:
+            index[name] = low[name] = len(index)
+            stack.append(name)
+            for imported in edges[name]:
+                if imported not in index:
+                    visit(imported)
+                    low[name] = min(low[name], low[imported])
+                elif imported in stack:
+                    low[name] = min(low[name], index[imported])
+            if low[name] == index[name]:
+                members: list[str] = []
+                while True:
+                    member = stack.pop()
+                    members.append(member)
+                    if member == name:
+                        break
+                for member in members:
+                    component_of[member] = len(components)
+                components.append(frozenset(members))
+
+        for name in sorted(names):
+            if name not in index:
+                visit(name)
+
+        depth: dict[int, int] = {}
+
+        def level(component: int) -> int:
+            if component not in depth:
+                below = {
+                    component_of[imported]
+                    for member in components[component]
+                    for imported in edges[member]
+                    if component_of[imported] != component
+                }
+                depth[component] = 1 + max((level(c) for c in below), default=-1)
+            return depth[component]
+
+        waves: dict[int, list[frozenset[str]]] = {}
+        for number in range(len(components)):
+            waves.setdefault(level(number), []).append(components[number])
+        return tuple(
+            tuple(sorted(waves[wave], key=min)) for wave in sorted(waves)
+        )
+
+
+def build_module_graph(
+    sources: Mapping[str, SourceFile],
+    modules: Mapping[str, nodes.Module],
+    imports: Mapping[str, ImportTable],
+) -> ModuleGraph:
+    """Edges from each module to the project modules it imports, in any scope."""
+
+    names = frozenset(sources)
+    edges: dict[str, frozenset[str]] = {}
+    for name, module in modules.items():
+        candidates: set[str] = set()
+        dotted_tops: set[str] = set()
+        for statement in _statements(module.body):
+            if isinstance(statement, nodes.Import):
+                for alias in statement.names:
+                    candidates.add(alias.name)
+                    if "." in alias.name:
+                        dotted_tops.add(alias.name.partition(".")[0])
+        for symbol in imports[name].all_symbols():
+            path = symbol.canonical_name.removeprefix("python.")
+            # ``import app.config`` binds ``app``; the statement already named the module.
+            if path not in dotted_tops:
+                candidates.add(path)
+        # The most specific project module each import names; ``app.helpers`` implies
+        # the ``app`` package, which is not an edge worth recording.
+        found = {
+            next((m for m in _prefixes(candidate) if m in names), None) for candidate in candidates
+        }
+        edges[name] = frozenset(m for m in found if m is not None and m != name)
+    return ModuleGraph(MappingProxyType(dict(sources)), MappingProxyType(edges))
+
+
+def _prefixes(dotted: str) -> list[str]:
+    parts = dotted.split(".")
+    return [".".join(parts[:length]) for length in range(len(parts), 0, -1)]
+
+
+def _statements(body: Iterable[nodes.Statement]) -> Iterable[nodes.Statement]:
+    for statement in body:
+        yield statement
+        for attribute in ("body", "orelse", "finalbody"):
+            nested = getattr(statement, attribute, None)
+            if isinstance(nested, tuple):
+                yield from _statements(nested)
+        if isinstance(statement, nodes.Try):
+            for handler in statement.handlers:
+                yield from _statements(handler.body)
+
+__all__ = [
+    "IGNORED_DIRECTORIES",
+    "ModuleGraph",
+    "ProjectSummaries",
+    "SummaryIndex",
+    "build_module_graph",
+    "discover_sources",
+    "project_symbol",
+]