codegraph-ir 0.1.0__py3-none-any.whl

cgir/report/stats.py

@@ -0,0 +1,121 @@
+"""Codebase structure report over ComponentSpecs.
+
+``compute_stats`` is a pure function from ``list[ComponentSpec]`` to a
+JSON-able dict — it drives both the ``cgir stats`` text output and its
+``--json`` mode. Purity buckets follow the scoring rubric in
+:mod:`cgir.analyses.purity`: 1.0 pure, 0.7 effect-tainted (calls into
+effectful code only), everything below direct-impure.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from cgir.analyses.effects import IMPURE_EFFECT_TAGS
+from cgir.ir.component_spec import ComponentSpec
+
+TOP_N = 10
+_IMPURE = set(IMPURE_EFFECT_TAGS)
+
+
+def compute_stats(specs: list[ComponentSpec], top_n: int = TOP_N) -> dict[str, Any]:
+    known_ids = {s.id for s in specs}
+    files = {s.trace[0].rsplit(":", 1)[0] for s in specs if s.trace}
+
+    kinds = Counter(s.kind.value for s in specs)
+    effects = Counter(tag for s in specs for tag in s.effects)
+
+    internal_callers: Counter[str] = Counter()
+    external_callers: Counter[str] = Counter()
+    constructed: Counter[str] = Counter()
+    for s in specs:
+        for callee in set(s.calls):
+            if callee in known_ids:
+                internal_callers[callee] += 1
+            else:
+                external_callers[callee] += 1
+        for type_name in set(s.constructs):
+            constructed[type_name] += 1
+
+    pure = sum(1 for s in specs if s.purity == 1.0)
+    tainted = sum(1 for s in specs if s.purity == 0.7)
+    impure = len(specs) - pure - tainted
+    mean = sum(s.purity or 0.0 for s in specs) / len(specs) if specs else 0.0
+
+    fan_out = sorted(specs, key=lambda s: (-len(s.calls), s.id))
+
+    return {
+        "total": len(specs),
+        "files": len(files),
+        "kinds": dict(kinds),
+        "purity": {"mean": mean, "pure": pure, "tainted": tainted, "impure": impure},
+        "effects": dict(effects),
+        "most_called": [
+            {"id": callee, "callers": n} for callee, n in internal_callers.most_common(top_n)
+        ],
+        "top_fan_out": [{"id": s.id, "calls": len(s.calls)} for s in fan_out[:top_n] if s.calls],
+        "external_calls": [
+            {"id": callee, "callers": n} for callee, n in external_callers.most_common(top_n)
+        ],
+        "top_constructed": [
+            {"id": type_name, "constructors": n} for type_name, n in constructed.most_common(top_n)
+        ],
+        "entrypoints": [
+            {"id": s.id, "entrypoint": s.entrypoint}
+            for s in sorted(specs, key=lambda s: (s.entrypoint or "", s.id))
+            if s.entrypoint
+        ],
+        "untested_effectful": [
+            {"id": s.id, "effects": s.effects}
+            for s in sorted(specs, key=lambda s: s.id)
+            if _IMPURE & set(s.effects) and not s.covered_by
+        ],
+    }
+
+
+def render_text(stats: dict[str, Any]) -> str:
+    """Terminal-friendly rendering of :func:`compute_stats` output."""
+    lines: list[str] = []
+    lines.append(f"Components: {stats['total']}  (files: {stats['files']})")
+
+    if stats["kinds"]:
+        kinds = " · ".join(f"{k} {n}" for k, n in sorted(stats["kinds"].items()))
+        lines.append(f"Kinds:      {kinds}")
+
+    purity = stats["purity"]
+    lines.append(
+        f"Purity:     mean {purity['mean']:.2f} · pure {purity['pure']}"
+        f" · tainted {purity['tainted']} · impure {purity['impure']}"
+    )
+
+    if stats["effects"]:
+        effects = " · ".join(f"{k} {n}" for k, n in sorted(stats["effects"].items()))
+        lines.append(f"Effects:    {effects}")
+
+    if stats["entrypoints"]:
+        lines.append("Entrypoints:")
+        width = max(len(e["entrypoint"]) for e in stats["entrypoints"])
+        for entry in stats["entrypoints"]:
+            lines.append(f"  {entry['entrypoint']:<{width}}  {entry['id']}")
+
+    if stats["untested_effectful"]:
+        lines.append(f"Untested effectful ({len(stats['untested_effectful'])}):")
+        for entry in stats["untested_effectful"][:TOP_N]:
+            lines.append(f"  [{','.join(entry['effects'])}]  {entry['id']}")
+
+    for title, key, count_key in (
+        ("Most called", "most_called", "callers"),
+        ("Top fan-out", "top_fan_out", "calls"),
+        ("Constructed types", "top_constructed", "constructors"),
+        ("External calls", "external_calls", "callers"),
+    ):
+        entries = stats[key]
+        if not entries:
+            continue
+        lines.append(f"{title}:")
+        width = len(str(entries[0][count_key]))
+        for entry in entries:
+            lines.append(f"  {entry[count_key]:>{width}}x {entry['id']}")
+
+    return "\n".join(lines) + "\n"

cgir/slicing/__init__.py

@@ -0,0 +1,5 @@
+"""Component slicing: RepoGraph → ComponentSpec[]."""
+
+from cgir.slicing.slicer import slice_components
+
+__all__ = ["slice_components"]

cgir/slicing/slicer.py

@@ -0,0 +1,174 @@
+"""Slice each Function/Method node into a ComponentSpec."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from cgir.analyses.effects import IMPURE_EFFECT_TAGS, TRANSITIVE_TAG
+from cgir.analyses.entrypoints import detect as detect_entrypoint
+from cgir.analyses.purity import PLACEHOLDER_SCORE
+from cgir.ir.component_spec import ComponentKind, ComponentSpec
+from cgir.ir.edges import EdgeKind
+from cgir.ir.graph import RepoGraph
+from cgir.ir.nodes import Node, NodeKind
+from cgir.languages import adapter_for_extension
+
+
+def _node_language(node: Node, fallback: str) -> str:
+    """The component's language, from its file's adapter (multi-language repos)."""
+    if node.path:
+        adapter = adapter_for_extension(Path(node.path).suffix)
+        if adapter is not None:
+            return adapter.name
+    return fallback
+
+
+def slice_components(
+    graph: RepoGraph,
+    effects: dict[str, list[str]] | None = None,
+    purity_scores: dict[str, float] | None = None,
+    language: str = "python",
+) -> list[ComponentSpec]:
+    effects = effects or {}
+    purity_scores = purity_scores or {}
+    specs: list[ComponentSpec] = []
+
+    for node in graph.nodes():
+        if node.kind not in {NodeKind.Function, NodeKind.Method}:
+            continue
+
+        qual = str(node.attrs.get("qualname") or node.name)
+        inputs = [param.name for param in graph.children(node.id, NodeKind.Parameter)]
+        calls, constructs = _split_callees(graph, node.id)
+        node_effects = effects.get(node.id, [])
+        purity_score = purity_scores.get(node.id, PLACEHOLDER_SCORE)
+        mutates_state = _has_mutations(graph, node.id)
+        kind = _classify(node_effects, purity_score, mutates_state)
+
+        trace = []
+        if node.path is not None and node.start_line is not None:
+            trace.append(f"{node.path}:{node.start_line}")
+
+        attrs = node.attrs or {}
+        signature = attrs.get("signature")
+        returns = attrs.get("returns")
+        decorators = attrs.get("decorators")
+        doc = attrs.get("doc")
+        raises = attrs.get("raises")
+        covered_by = _covering_tests(graph, node.id)
+
+        specs.append(
+            ComponentSpec(
+                id=qual,
+                kind=kind,
+                inputs=inputs,
+                outputs=[returns] if isinstance(returns, str) and returns else [],
+                effects=list(node_effects),
+                calls=calls,
+                constructs=constructs,
+                trace=trace,
+                language=_node_language(node, language),
+                signature=signature if isinstance(signature, str) else None,
+                entrypoint=detect_entrypoint(
+                    decorators if isinstance(decorators, list) else [], node.name
+                ),
+                doc=doc if isinstance(doc, str) and doc else None,
+                raises=list(raises) if isinstance(raises, list) else [],
+                covered_by=covered_by,
+                purity=purity_score,
+            )
+        )
+
+    specs.sort(key=lambda s: s.id)
+    return specs
+
+
+def _covering_tests(graph: RepoGraph, func_id: str) -> list[str]:
+    """Test components that call this one (via resolved CALLS edges)."""
+    tests: set[str] = set()
+    for edge in graph.in_edges(func_id, EdgeKind.CALLS):
+        caller = graph.get_node(edge.src)
+        if _is_test_node(caller):
+            tests.add(_callee_label(graph, caller.id))
+    return sorted(tests)
+
+
+def _is_test_node(node: Node) -> bool:
+    """Heuristic: pytest-style test functions in test files."""
+    if node.name.startswith("test_"):
+        return True
+    if node.path:
+        parts = node.path.replace("\\", "/").split("/")
+        stem = parts[-1].removesuffix(".py")
+        return "tests" in parts[:-1] or stem.startswith("test_") or stem.endswith("_test")
+    return False
+
+
+def _split_callees(graph: RepoGraph, func_id: str) -> tuple[list[str], list[str]]:
+    """Partition CALLS edges into (calls, constructs).
+
+    A call whose target is a Class node is a construction: it resolves to
+    the class's ``__init__`` component when one is defined, else the class
+    qualname lands in ``constructs`` (dataclass / ORM style).
+    """
+    calls: set[str] = set()
+    constructs: set[str] = set()
+    for edge in graph.out_edges(func_id, EdgeKind.CALLS):
+        callee = graph.get_node(edge.dst)
+        if callee.kind == NodeKind.Class:
+            init = next(
+                (m for m in graph.children(callee.id, NodeKind.Method) if m.name == "__init__"),
+                None,
+            )
+            if init is not None:
+                calls.add(_callee_label(graph, init.id))
+            else:
+                constructs.add(_callee_label(graph, callee.id))
+        else:
+            calls.add(_callee_label(graph, edge.dst))
+    return sorted(calls), sorted(constructs)
+
+
+def _callee_label(graph: RepoGraph, node_id: str) -> str:
+    node = graph.get_node(node_id)
+    qual = node.attrs.get("qualname") if node.attrs else None
+    return str(qual) if isinstance(qual, str) else node.name
+
+
+def _has_mutations(graph: RepoGraph, func_id: str) -> bool:
+    """True if any CFG child mutates state *observable by the caller*.
+
+    Covers attribute/subscript assignment LHS (``self.x = v``, ``xs[0] = v``,
+    ``self.total += n``) on ``Assignment`` nodes and bare mutator method
+    calls (``xs.append(x)``) on ``Statement`` nodes — both populated by
+    :mod:`cgir.analyses.cfg` via ``attrs["mutates"]``.
+
+    Mutating an object the function *itself created* (a local) is invisible
+    to callers and stays pure: a mutated base name only counts if it is a
+    parameter, ``self``, or a name never written locally (a global).
+    """
+    params = {p.name for p in graph.children(func_id, NodeKind.Parameter)}
+    local_writes: set[str] = set()
+    mutated: list[str] = []
+    for child in graph.children(func_id):
+        attrs = child.attrs or {}
+        writes = attrs.get("writes")
+        if isinstance(writes, list):
+            local_writes.update(writes)
+        mutates = attrs.get("mutates")
+        if isinstance(mutates, list):
+            mutated.extend(mutates)
+    return any(base in params or base not in local_writes for base in mutated)
+
+
+def _classify(effects: list[str], purity_score: float, mutates_state: bool) -> ComponentKind:
+    tags = set(effects)
+    if tags & IMPURE_EFFECT_TAGS:
+        return ComponentKind.effect_adapter
+    if TRANSITIVE_TAG in tags:
+        return ComponentKind.orchestrator
+    if mutates_state:
+        return ComponentKind.state_transformer
+    if purity_score == 1.0:
+        return ComponentKind.pure_function
+    return ComponentKind.unknown

cgir/sources/__init__.py

@@ -0,0 +1,6 @@
+"""GraphSource adapters: tree-sitter (working) + joern/codeql stubs."""
+
+from cgir.sources.base import GraphSource
+from cgir.sources.tree_sitter_source import TreeSitterSource
+
+__all__ = ["GraphSource", "TreeSitterSource"]

cgir/sources/base.py

@@ -0,0 +1,14 @@
+"""Abstract GraphSource — every backend normalizes into the same RepoGraph."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from cgir.ir.graph import RepoGraph
+
+
+class GraphSource(ABC):
+    @abstractmethod
+    def ingest(self, repo_path: Path) -> RepoGraph:
+        """Walk ``repo_path`` and produce a normalized RepoGraph."""

cgir/sources/codeql_source.py

@@ -0,0 +1,13 @@
+"""CodeQL adapter — milestone: P2-codeql-bridge."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from cgir.ir.graph import RepoGraph
+from cgir.sources.base import GraphSource
+
+
+class CodeQLSource(GraphSource):
+    def ingest(self, repo_path: Path) -> RepoGraph:
+        raise NotImplementedError("milestone: P2-codeql-bridge")

cgir/sources/joern_source.py

@@ -0,0 +1,13 @@
+"""Joern adapter — milestone: P2-joern-bridge."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from cgir.ir.graph import RepoGraph
+from cgir.sources.base import GraphSource
+
+
+class JoernSource(GraphSource):
+    def ingest(self, repo_path: Path) -> RepoGraph:
+        raise NotImplementedError("milestone: P2-joern-bridge")

cgir/sources/tree_sitter_source.py

@@ -0,0 +1,270 @@
+"""Generic tree-sitter ingester — language-neutral over a LanguageAdapter.
+
+Walks a repo, parses each source file with the active adapter, and emits
+the ``Repository → File → Module → (Class → Method | Function)``
+containment spine plus ``Parameter`` children, per-module ``Import``
+siblings, and module-level ``Variable`` nodes.
+
+All grammar-specific extraction (what a function/class/import looks like,
+signatures, docstrings, raised names, free names, relative-import
+resolution) comes from :meth:`LanguageAdapter.module_declarations` as
+normalized declarations; this module only builds graph nodes and edges.
+Symbol resolution and ``CALLS`` edges live in :mod:`cgir.analyses`.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+from tree_sitter import Node as TSNode
+
+from cgir.ir.edges import Edge, EdgeKind
+from cgir.ir.graph import RepoGraph
+from cgir.ir.nodes import Node, NodeKind
+from cgir.languages import ADAPTERS, LanguageAdapter, adapter_for_extension
+from cgir.languages.base import ClassDecl, FunctionDecl, ImportDecl, VariableDecl
+from cgir.languages.cache import parse_cached
+from cgir.sources.base import GraphSource
+
+DEFAULT_IGNORE_DIRS: frozenset[str] = frozenset(
+    {
+        # Virtual environments
+        "venv",
+        "env",
+        # Build / dist artefacts
+        "build",
+        "dist",
+        "target",
+        "out",
+        # Python caches
+        "__pycache__",
+        "site-packages",
+        # Tool caches (dot-prefixed names are also covered by the dot-prefix
+        # filter below; listed here for completeness when authors rename them)
+        ".tox",
+        ".pytest_cache",
+        ".mypy_cache",
+        ".ruff_cache",
+        # Other ecosystems sometimes vendored into repos
+        "node_modules",
+    }
+)
+
+
+class TreeSitterSource(GraphSource):
+    def __init__(
+        self,
+        ignore_dirs: Iterable[str] | None = None,
+        adapter: LanguageAdapter | None = None,
+    ) -> None:
+        # adapter=None → dispatch per file extension across all registered
+        # languages; a forced adapter restricts ingest to its extensions.
+        self._forced = adapter
+        self._ignore: frozenset[str] = DEFAULT_IGNORE_DIRS | frozenset(ignore_dirs or ())
+
+    def _extensions(self) -> tuple[str, ...]:
+        if self._forced is not None:
+            return self._forced.file_extensions
+        return tuple(ext for a in ADAPTERS.values() for ext in a.file_extensions)
+
+    def ingest(self, repo_path: Path) -> RepoGraph:
+        repo_path = repo_path.resolve()
+        graph = RepoGraph()
+        repo_id = f"repo:{repo_path.name}"
+        graph.add_node(
+            Node(id=repo_id, kind=NodeKind.Repository, name=repo_path.name, path=str(repo_path))
+        )
+        files: list[Path] = []
+        for ext in self._extensions():
+            files.extend(repo_path.rglob(f"*{ext}"))
+        for source_file in sorted(set(files)):
+            if self._should_skip(source_file.relative_to(repo_path)):
+                continue
+            adapter = self._forced or adapter_for_extension(source_file.suffix)
+            if adapter is not None:
+                self._ingest_file(graph, repo_path, repo_id, source_file, adapter)
+        return graph
+
+    def _should_skip(self, rel: Path) -> bool:
+        for part in rel.parts:
+            if part.startswith("."):
+                return True
+            if part in self._ignore:
+                return True
+        return False
+
+    def _ingest_file(
+        self,
+        graph: RepoGraph,
+        repo_path: Path,
+        repo_id: str,
+        file_path: Path,
+        adapter: LanguageAdapter,
+    ) -> None:
+        rel = file_path.relative_to(repo_path)
+        rel_str = str(rel)
+        module_name = ".".join(rel.with_suffix("").parts)
+        file_id = f"file:{rel_str}"
+        module_id = f"module:{module_name}"
+
+        source = file_path.read_bytes()
+        root = parse_cached(adapter, source)
+
+        graph.add_node(
+            Node(
+                id=file_id,
+                kind=NodeKind.File,
+                name=rel_str,
+                path=rel_str,
+                start_line=1,
+                end_line=root.end_point[0] + 1,
+            )
+        )
+        graph.add_edge(Edge(src=repo_id, dst=file_id, kind=EdgeKind.CONTAINS))
+
+        graph.add_node(
+            Node(
+                id=module_id,
+                kind=NodeKind.Module,
+                name=module_name,
+                path=rel_str,
+                start_line=1,
+                end_line=root.end_point[0] + 1,
+                attrs={"language": adapter.name},
+            )
+        )
+        graph.add_edge(Edge(src=file_id, dst=module_id, kind=EdgeKind.CONTAINS))
+
+        for decl in adapter.module_declarations(root, source, module_name, rel_str):
+            if isinstance(decl, FunctionDecl):
+                self._add_function(graph, module_id, module_name, rel_str, decl, is_method=False)
+            elif isinstance(decl, ClassDecl):
+                self._add_class(graph, module_id, module_name, rel_str, decl)
+            elif isinstance(decl, ImportDecl):
+                self._add_import(graph, module_id, rel_str, decl)
+            elif isinstance(decl, VariableDecl):
+                self._add_variable(graph, module_id, module_name, rel_str, decl)
+
+    def _add_function(
+        self,
+        graph: RepoGraph,
+        parent_id: str,
+        parent_qual: str,
+        rel_path: str,
+        decl: FunctionDecl,
+        is_method: bool,
+    ) -> None:
+        qual = f"{parent_qual}.{decl.name}"
+        kind = NodeKind.Method if is_method else NodeKind.Function
+        prefix = "method" if is_method else "func"
+        node_id = f"{prefix}:{qual}"
+        graph.add_node(
+            Node(
+                id=node_id,
+                kind=kind,
+                name=decl.name,
+                path=rel_path,
+                start_line=decl.node.start_point[0] + 1,
+                end_line=decl.node.end_point[0] + 1,
+                attrs={
+                    "qualname": qual,
+                    "signature": decl.signature,
+                    "returns": decl.returns,
+                    "decorators": list(decl.decorators),
+                    "doc": decl.doc,
+                    "raises": list(decl.raises),
+                    "free_names": list(decl.free_names),
+                },
+            )
+        )
+        graph.add_edge(Edge(src=parent_id, dst=node_id, kind=EdgeKind.CONTAINS))
+        for param in decl.params:
+            self._add_parameter(graph, node_id, qual, rel_path, param.name, param.node)
+
+    def _add_parameter(
+        self,
+        graph: RepoGraph,
+        func_id: str,
+        func_qual: str,
+        rel_path: str,
+        name: str,
+        ts_node: TSNode,
+    ) -> None:
+        param_id = f"param:{func_qual}.{name}"
+        graph.add_node(
+            Node(
+                id=param_id,
+                kind=NodeKind.Parameter,
+                name=name,
+                path=rel_path,
+                start_line=ts_node.start_point[0] + 1,
+                end_line=ts_node.end_point[0] + 1,
+            )
+        )
+        graph.add_edge(Edge(src=func_id, dst=param_id, kind=EdgeKind.CONTAINS))
+
+    def _add_class(
+        self,
+        graph: RepoGraph,
+        parent_id: str,
+        parent_qual: str,
+        rel_path: str,
+        decl: ClassDecl,
+    ) -> None:
+        qual = f"{parent_qual}.{decl.name}"
+        node_id = f"class:{qual}"
+        graph.add_node(
+            Node(
+                id=node_id,
+                kind=NodeKind.Class,
+                name=decl.name,
+                path=rel_path,
+                start_line=decl.node.start_point[0] + 1,
+                end_line=decl.node.end_point[0] + 1,
+                attrs={"qualname": qual, "fields": dict(decl.fields)},
+            )
+        )
+        graph.add_edge(Edge(src=parent_id, dst=node_id, kind=EdgeKind.CONTAINS))
+        for method in decl.methods:
+            self._add_function(graph, node_id, qual, rel_path, method, is_method=True)
+
+    def _add_import(
+        self, graph: RepoGraph, module_id: str, rel_path: str, decl: ImportDecl
+    ) -> None:
+        import_id = f"import:{module_id}::{decl.target}"
+        graph.add_node(
+            Node(
+                id=import_id,
+                kind=NodeKind.Import,
+                name=decl.target,
+                path=rel_path,
+                start_line=decl.node.start_point[0] + 1,
+                end_line=decl.node.end_point[0] + 1,
+                attrs={"target": decl.target, "alias": decl.alias},
+            )
+        )
+        graph.add_edge(Edge(src=module_id, dst=import_id, kind=EdgeKind.CONTAINS))
+        graph.add_edge(
+            Edge(src=module_id, dst=import_id, kind=EdgeKind.IMPORTS, attrs={"target": decl.target})
+        )
+
+    def _add_variable(
+        self, graph: RepoGraph, module_id: str, module_name: str, rel_path: str, decl: VariableDecl
+    ) -> None:
+        var_id = f"var:{module_name}.{decl.name}"
+        graph.add_node(
+            Node(
+                id=var_id,
+                kind=NodeKind.Variable,
+                name=decl.name,
+                path=rel_path,
+                start_line=decl.node.start_point[0] + 1,
+                end_line=decl.node.end_point[0] + 1,
+                attrs={"qualname": f"{module_name}.{decl.name}"},
+            )
+        )
+        graph.add_edge(Edge(src=module_id, dst=var_id, kind=EdgeKind.CONTAINS))
+
+
+__all__ = ["DEFAULT_IGNORE_DIRS", "TreeSitterSource"]

cgir/trace/__init__.py

@@ -0,0 +1,5 @@
+"""Trace map: source-loc ↔ ComponentSpec id."""
+
+from cgir.trace.trace_map import TraceMap, build_trace_map
+
+__all__ = ["TraceMap", "build_trace_map"]