code-graph-builder 0.2.0__py3-none-any.whl

code_graph_builder/parsers/factory.py

@@ -0,0 +1,119 @@
+"""Code Graph Builder - Processor Factory."""
+
+from pathlib import Path
+
+from ..constants import SupportedLanguage
+from ..services import IngestorProtocol
+from ..types import (
+    ASTCacheProtocol,
+    FunctionRegistryTrieProtocol,
+    LanguageQueries,
+    SimpleNameLookup,
+)
+from .call_processor import CallProcessor
+from .definition_processor import DefinitionProcessor
+from .import_processor import ImportProcessor
+from .structure_processor import StructureProcessor
+from .type_inference import TypeInferenceEngine
+
+
+class ProcessorFactory:
+    """Factory for creating and caching processors."""
+
+    def __init__(
+        self,
+        ingestor: IngestorProtocol,
+        repo_path: Path,
+        project_name: str,
+        queries: dict[SupportedLanguage, LanguageQueries],
+        function_registry: FunctionRegistryTrieProtocol,
+        simple_name_lookup: SimpleNameLookup,
+        ast_cache: ASTCacheProtocol,
+        unignore_paths: frozenset[str] | None = None,
+        exclude_paths: frozenset[str] | None = None,
+    ) -> None:
+        self.ingestor = ingestor
+        self.repo_path = repo_path
+        self.project_name = project_name
+        self.queries = queries
+        self.function_registry = function_registry
+        self.simple_name_lookup = simple_name_lookup
+        self.ast_cache = ast_cache
+        self.unignore_paths = unignore_paths
+        self.exclude_paths = exclude_paths
+
+        self.module_qn_to_file_path: dict[str, Path] = {}
+
+        self._import_processor: ImportProcessor | None = None
+        self._structure_processor: StructureProcessor | None = None
+        self._definition_processor: DefinitionProcessor | None = None
+        self._type_inference: TypeInferenceEngine | None = None
+        self._call_processor: CallProcessor | None = None
+
+    @property
+    def import_processor(self) -> ImportProcessor:
+        if self._import_processor is None:
+            self._import_processor = ImportProcessor(
+                repo_path=self.repo_path,
+                project_name=self.project_name,
+                ingestor=self.ingestor,
+                function_registry=self.function_registry,
+            )
+        return self._import_processor
+
+    @property
+    def structure_processor(self) -> StructureProcessor:
+        if self._structure_processor is None:
+            self._structure_processor = StructureProcessor(
+                ingestor=self.ingestor,
+                repo_path=self.repo_path,
+                project_name=self.project_name,
+                queries=self.queries,
+                unignore_paths=self.unignore_paths,
+                exclude_paths=self.exclude_paths,
+            )
+        return self._structure_processor
+
+    @property
+    def definition_processor(self) -> DefinitionProcessor:
+        if self._definition_processor is None:
+            self._definition_processor = DefinitionProcessor(
+                ingestor=self.ingestor,
+                repo_path=self.repo_path,
+                project_name=self.project_name,
+                function_registry=self.function_registry,
+                simple_name_lookup=self.simple_name_lookup,
+                import_processor=self.import_processor,
+                module_qn_to_file_path=self.module_qn_to_file_path,
+            )
+        return self._definition_processor
+
+    @property
+    def type_inference(self) -> TypeInferenceEngine:
+        if self._type_inference is None:
+            self._type_inference = TypeInferenceEngine(
+                import_processor=self.import_processor,
+                function_registry=self.function_registry,
+                repo_path=self.repo_path,
+                project_name=self.project_name,
+                ast_cache=self.ast_cache,
+                queries=self.queries,
+                module_qn_to_file_path=self.module_qn_to_file_path,
+                class_inheritance=self.definition_processor.class_inheritance,
+                simple_name_lookup=self.simple_name_lookup,
+            )
+        return self._type_inference
+
+    @property
+    def call_processor(self) -> CallProcessor:
+        if self._call_processor is None:
+            self._call_processor = CallProcessor(
+                ingestor=self.ingestor,
+                repo_path=self.repo_path,
+                project_name=self.project_name,
+                function_registry=self.function_registry,
+                import_processor=self.import_processor,
+                type_inference=self.type_inference,
+                class_inheritance=self.definition_processor.class_inheritance,
+            )
+        return self._call_processor

code_graph_builder/parsers/import_processor.py

@@ -0,0 +1,293 @@
+"""Code Graph Builder - Import Processor."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from loguru import logger
+from tree_sitter import Node, QueryCursor
+
+from .. import constants as cs
+from ..parsers.utils import safe_decode_text
+from ..services import IngestorProtocol
+from ..types import FunctionRegistryTrieProtocol
+
+if TYPE_CHECKING:
+    from ..types import LanguageQueries
+
+
+class ImportProcessor:
+    """Process import statements in source code."""
+
+    def __init__(
+        self,
+        repo_path: Path,
+        project_name: str,
+        ingestor: IngestorProtocol | None = None,
+        function_registry: FunctionRegistryTrieProtocol | None = None,
+    ) -> None:
+        self.repo_path = repo_path
+        self.project_name = project_name
+        self.ingestor = ingestor
+        self.function_registry = function_registry
+        self.import_mapping: dict[str, dict[str, str]] = {}
+
+    def parse_imports(
+        self,
+        root_node: Node,
+        module_qn: str,
+        language: cs.SupportedLanguage,
+        queries: dict[cs.SupportedLanguage, LanguageQueries],
+    ) -> None:
+        """Parse imports from a file."""
+        if language not in queries:
+            return
+
+        lang_queries = queries[language]
+        imports_query = lang_queries.get(cs.QUERY_IMPORTS)
+        if not imports_query:
+            return
+
+        self.import_mapping[module_qn] = {}
+
+        try:
+            cursor = QueryCursor(imports_query)
+            captures = cursor.captures(root_node)
+
+            match language:
+                case cs.SupportedLanguage.PYTHON:
+                    self._parse_python_imports(captures, module_qn)
+                case cs.SupportedLanguage.JS | cs.SupportedLanguage.TS:
+                    self._parse_js_ts_imports(captures, module_qn)
+                case cs.SupportedLanguage.JAVA:
+                    self._parse_java_imports(captures, module_qn)
+                case cs.SupportedLanguage.RUST:
+                    self._parse_rust_imports(captures, module_qn)
+                case cs.SupportedLanguage.GO:
+                    self._parse_go_imports(captures, module_qn)
+                case cs.SupportedLanguage.C | cs.SupportedLanguage.CPP:
+                    self._parse_c_cpp_imports(captures, module_qn)
+                case _:
+                    pass
+
+            logger.debug(f"Parsed {len(self.import_mapping[module_qn])} imports for {module_qn}")
+
+            if self.ingestor:
+                for alias, full_name in self.import_mapping[module_qn].items():
+                    self.ingestor.ensure_relationship_batch(
+                        (cs.NodeLabel.MODULE, cs.KEY_QUALIFIED_NAME, module_qn),
+                        cs.RelationshipType.IMPORTS,
+                        (cs.NodeLabel.MODULE, cs.KEY_QUALIFIED_NAME, full_name),
+                    )
+
+        except Exception as e:
+            logger.warning(f"Failed to parse imports for {module_qn}: {e}")
+
+    def _parse_python_imports(self, captures: dict, module_qn: str) -> None:
+        """Parse Python import statements."""
+        import_nodes = captures.get(cs.CAPTURE_IMPORT, [])
+        import_from_nodes = captures.get(cs.CAPTURE_IMPORT_FROM, [])
+
+        for node in import_nodes + import_from_nodes:
+            if not isinstance(node, Node):
+                continue
+
+            if node.type == "import_statement":
+                self._handle_python_import(node, module_qn)
+            elif node.type == "import_from_statement":
+                self._handle_python_from_import(node, module_qn)
+
+    def _handle_python_import(self, node: Node, module_qn: str) -> None:
+        """Handle 'import xxx' or 'import xxx as yyy'."""
+        for child in node.named_children:
+            if child.type == "dotted_name":
+                name = self._get_dotted_name(child)
+                if name:
+                    full_qn = f"{self.project_name}.{name.replace('.', cs.SEPARATOR_DOT)}"
+                    self.import_mapping[module_qn][name.split(cs.SEPARATOR_DOT)[0]] = full_qn
+
+    def _handle_python_from_import(self, node: Node, module_qn: str) -> None:
+        """Handle 'from xxx import yyy'."""
+        module_node = None
+        for child in node.children:
+            if child.type == "dotted_name":
+                module_node = child
+                break
+
+        if not module_node:
+            return
+
+        module_name = self._get_dotted_name(module_node)
+        if not module_name:
+            return
+
+        module_prefix = f"{self.project_name}.{module_name.replace('.', cs.SEPARATOR_DOT)}"
+
+        for child in node.named_children:
+            if child.type == "imported_name" or child.type == "identifier":
+                name = safe_decode_text(child)
+                if name:
+                    full_qn = f"{module_prefix}.{name}"
+                    self.import_mapping[module_qn][name] = full_qn
+
+    def _parse_js_ts_imports(self, captures: dict, module_qn: str) -> None:
+        """Parse JavaScript/TypeScript imports."""
+        import_nodes = captures.get(cs.CAPTURE_IMPORT, [])
+
+        for node in import_nodes:
+            if not isinstance(node, Node):
+                continue
+
+            if node.type == "import_statement":
+                self._handle_js_ts_import(node, module_qn)
+
+    def _handle_js_ts_import(self, node: Node, module_qn: str) -> None:
+        """Handle ES6 import statements."""
+        source_node = None
+        for child in node.children:
+            if child.type == "string":
+                source_node = child
+                break
+
+        if not source_node:
+            return
+
+        source = safe_decode_text(source_node)
+        if not source:
+            return
+
+        source = source.strip("'\"")
+
+        for child in node.named_children:
+            if child.type == "import_clause":
+                self._process_js_import_clause(child, source, module_qn)
+
+    def _process_js_import_clause(self, node: Node, source: str, module_qn: str) -> None:
+        """Process import clause (default, named, or namespace imports)."""
+        name = safe_decode_text(node)
+        if name:
+            self.import_mapping[module_qn][name] = source
+
+    def _parse_java_imports(self, captures: dict, module_qn: str) -> None:
+        """Parse Java imports."""
+        import_nodes = captures.get(cs.CAPTURE_IMPORT, [])
+
+        for node in import_nodes:
+            if not isinstance(node, Node):
+                continue
+
+            scoped_name = None
+            for child in node.named_children:
+                if child.type == "scoped_identifier":
+                    scoped_name = safe_decode_text(child)
+                    break
+                elif child.type == "identifier":
+                    scoped_name = safe_decode_text(child)
+
+            if scoped_name:
+                parts = scoped_name.split(cs.SEPARATOR_DOT)
+                if parts:
+                    self.import_mapping[module_qn][parts[-1]] = scoped_name.replace(
+                        cs.SEPARATOR_DOT, "."
+                    )
+
+    def _parse_rust_imports(self, captures: dict, module_qn: str) -> None:
+        """Parse Rust use statements."""
+        import_nodes = captures.get(cs.CAPTURE_IMPORT, [])
+
+        for node in import_nodes:
+            if not isinstance(node, Node):
+                continue
+
+            if node.type == "use_declaration":
+                self._handle_rust_use(node, module_qn)
+
+    def _handle_rust_use(self, node: Node, module_qn: str) -> None:
+        """Handle Rust use statements."""
+        for child in node.named_children:
+            if child.type == "scoped_use_list":
+                prefix = None
+                use_list = None
+                for c in child.children:
+                    if c.type == "identifier" or c.type == "scoped_identifier":
+                        prefix = safe_decode_text(c)
+                    elif c.type == "use_list":
+                        use_list = c
+
+                if prefix and use_list:
+                    for item in use_list.named_children:
+                        name = safe_decode_text(item)
+                        if name:
+                            full_qn = f"{prefix}::{name}"
+                            self.import_mapping[module_qn][name] = full_qn
+            elif child.type in ("scoped_identifier", "identifier"):
+                name = safe_decode_text(child)
+                if name:
+                    parts = name.split("::")
+                    if parts:
+                        self.import_mapping[module_qn][parts[-1]] = name
+
+    def _parse_go_imports(self, captures: dict, module_qn: str) -> None:
+        """Parse Go imports."""
+        import_nodes = captures.get(cs.CAPTURE_IMPORT, [])
+
+        for node in import_nodes:
+            if not isinstance(node, Node):
+                continue
+
+            if node.type == "import_declaration":
+                for child in node.named_children:
+                    if child.type == "import_spec":
+                        self._handle_go_import_spec(child, module_qn)
+                    elif child.type == "import_spec_list":
+                        for spec in child.named_children:
+                            if spec.type == "import_spec":
+                                self._handle_go_import_spec(spec, module_qn)
+
+    def _handle_go_import_spec(self, node: Node, module_qn: str) -> None:
+        """Handle Go import specification."""
+        alias = None
+        path = None
+
+        for child in node.named_children:
+            if child.type == "package_identifier":
+                alias = safe_decode_text(child)
+            elif child.type == "interpreted_string_literal":
+                path = safe_decode_text(child)
+
+        if path:
+            path = path.strip('"')
+            key = alias if alias else path.split("/")[-1]
+            self.import_mapping[module_qn][key] = path
+
+    def _parse_c_cpp_imports(self, captures: dict, module_qn: str) -> None:
+        """Parse C/C++ #include directives."""
+        import_nodes = captures.get(cs.CAPTURE_IMPORT, [])
+
+        for node in import_nodes:
+            if not isinstance(node, Node):
+                continue
+
+            if node.type == "preproc_include":
+                for child in node.named_children:
+                    if child.type in ("string_literal", "system_lib_string"):
+                        header = safe_decode_text(child)
+                        if header:
+                            header = header.strip('"<>')
+                            key = header.replace(".", "_")
+                            self.import_mapping[module_qn][key] = header
+
+    def _get_dotted_name(self, node: Node) -> str | None:
+        """Get dotted name from a node."""
+        parts = []
+        for child in node.children:
+            if child.type == "identifier":
+                name = safe_decode_text(child)
+                if name:
+                    parts.append(name)
+        return cs.SEPARATOR_DOT.join(parts) if parts else None
+
+    def get_import_mapping(self, module_qn: str) -> dict[str, str]:
+        """Get import mapping for a module."""
+        return self.import_mapping.get(module_qn, {})

code_graph_builder/parsers/structure_processor.py

@@ -0,0 +1,145 @@
+"""Code Graph Builder - Structure Processor."""
+
+from pathlib import Path
+
+from loguru import logger
+
+from .. import constants as cs
+from ..services import IngestorProtocol
+from ..types import LanguageQueries, NodeIdentifier
+from ..utils.path_utils import should_skip_path
+
+
+class StructureProcessor:
+    """Processor for identifying project structure (packages, folders, files)."""
+
+    def __init__(
+        self,
+        ingestor: IngestorProtocol,
+        repo_path: Path,
+        project_name: str,
+        queries: dict[cs.SupportedLanguage, LanguageQueries],
+        unignore_paths: frozenset[str] | None = None,
+        exclude_paths: frozenset[str] | None = None,
+    ):
+        self.ingestor = ingestor
+        self.repo_path = repo_path
+        self.project_name = project_name
+        self.queries = queries
+        self.structural_elements: dict[Path, str | None] = {}
+        self.unignore_paths = unignore_paths
+        self.exclude_paths = exclude_paths
+
+    def _get_parent_identifier(
+        self, parent_rel_path: Path, parent_container_qn: str | None
+    ) -> NodeIdentifier:
+        """Get parent node identifier for relationship creation."""
+        if parent_rel_path == Path(cs.PATH_CURRENT_DIR):
+            return (cs.NodeLabel.PROJECT, cs.KEY_NAME, self.project_name)
+        if parent_container_qn:
+            return (cs.NodeLabel.PACKAGE, cs.KEY_QUALIFIED_NAME, parent_container_qn)
+        return (cs.NodeLabel.FOLDER, cs.KEY_PATH, parent_rel_path.as_posix())
+
+    def identify_structure(self) -> None:
+        """Identify project structure: packages and folders."""
+        # Create project node first
+        self.ingestor.ensure_node_batch(
+            cs.NodeLabel.PROJECT,
+            {
+                cs.KEY_NAME: self.project_name,
+                cs.KEY_QUALIFIED_NAME: self.project_name,
+                cs.KEY_PATH: str(self.repo_path),
+            },
+        )
+        logger.info(f"Created Project node: {self.project_name}")
+
+        directories = {self.repo_path}
+        for path in self.repo_path.rglob(cs.GLOB_ALL):
+            if path.is_dir() and not should_skip_path(
+                path,
+                self.repo_path,
+                exclude_paths=self.exclude_paths,
+                unignore_paths=self.unignore_paths,
+            ):
+                directories.add(path)
+
+        for root in sorted(directories):
+            relative_root = root.relative_to(self.repo_path)
+
+            parent_rel_path = relative_root.parent
+            parent_container_qn = self.structural_elements.get(parent_rel_path)
+
+            is_package = False
+            package_indicators: set[str] = set()
+
+            for lang_queries in self.queries.values():
+                lang_config = lang_queries[cs.QUERY_CONFIG]
+                package_indicators.update(lang_config.package_indicators)
+
+            for indicator in package_indicators:
+                if (root / indicator).exists():
+                    is_package = True
+                    break
+
+            if is_package:
+                package_qn = cs.SEPARATOR_DOT.join(
+                    [self.project_name] + list(relative_root.parts)
+                )
+                self.structural_elements[relative_root] = package_qn
+                logger.info(f"Identified Package: {package_qn}")
+                self.ingestor.ensure_node_batch(
+                    cs.NodeLabel.PACKAGE,
+                    {
+                        cs.KEY_QUALIFIED_NAME: package_qn,
+                        cs.KEY_NAME: root.name,
+                        cs.KEY_PATH: relative_root.as_posix(),
+                    },
+                )
+                parent_identifier = self._get_parent_identifier(
+                    parent_rel_path, parent_container_qn
+                )
+                self.ingestor.ensure_relationship_batch(
+                    parent_identifier,
+                    cs.RelationshipType.CONTAINS_PACKAGE,
+                    (cs.NodeLabel.PACKAGE, cs.KEY_QUALIFIED_NAME, package_qn),
+                )
+            elif root != self.repo_path:
+                self.structural_elements[relative_root] = None
+                logger.info(f"Identified Folder: {relative_root}")
+                self.ingestor.ensure_node_batch(
+                    cs.NodeLabel.FOLDER,
+                    {cs.KEY_PATH: relative_root.as_posix(), cs.KEY_NAME: root.name},
+                )
+                parent_identifier = self._get_parent_identifier(
+                    parent_rel_path, parent_container_qn
+                )
+                self.ingestor.ensure_relationship_batch(
+                    parent_identifier,
+                    cs.RelationshipType.CONTAINS_FOLDER,
+                    (cs.NodeLabel.FOLDER, cs.KEY_PATH, relative_root.as_posix()),
+                )
+
+    def process_generic_file(self, file_path: Path, file_name: str) -> None:
+        """Process a generic file node."""
+        relative_filepath = file_path.relative_to(self.repo_path).as_posix()
+        relative_root = file_path.parent.relative_to(self.repo_path)
+
+        parent_container_qn = self.structural_elements.get(relative_root)
+        parent_identifier = self._get_parent_identifier(
+            relative_root, parent_container_qn
+        )
+
+        self.ingestor.ensure_node_batch(
+            cs.NodeLabel.FILE,
+            {
+                cs.KEY_PATH: relative_filepath,
+                cs.KEY_NAME: file_name,
+                cs.KEY_EXTENSION: file_path.suffix,
+            },
+        )
+
+        self.ingestor.ensure_relationship_batch(
+            parent_identifier,
+            cs.RelationshipType.CONTAINS_FILE,
+            (cs.NodeLabel.FILE, cs.KEY_PATH, relative_filepath),
+        )

code_graph_builder/parsers/type_inference.py

@@ -0,0 +1,143 @@
+"""Code Graph Builder - Type Inference."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from tree_sitter import Node
+
+from .. import constants as cs
+from ..parsers.utils import safe_decode_text
+from ..types import ASTCacheProtocol, FunctionRegistryTrieProtocol, SimpleNameLookup
+
+if TYPE_CHECKING:
+    from ..types import LanguageQueries
+    from .import_processor import ImportProcessor
+
+
+class TypeInferenceEngine:
+    """Infer types from source code."""
+
+    def __init__(
+        self,
+        import_processor: ImportProcessor,
+        function_registry: FunctionRegistryTrieProtocol,
+        repo_path: Path,
+        project_name: str,
+        ast_cache: ASTCacheProtocol,
+        queries: dict[cs.SupportedLanguage, LanguageQueries],
+        module_qn_to_file_path: dict[str, Path],
+        class_inheritance: dict[str, list[str]],
+        simple_name_lookup: SimpleNameLookup,
+    ):
+        self.import_processor = import_processor
+        self.function_registry = function_registry
+        self.repo_path = repo_path
+        self.project_name = project_name
+        self.ast_cache = ast_cache
+        self.queries = queries
+        self.module_qn_to_file_path = module_qn_to_file_path
+        self.class_inheritance = class_inheritance
+        self.simple_name_lookup = simple_name_lookup
+        self._variable_types: dict[str, dict[str, str]] = {}
+
+    def infer_variable_type(
+        self,
+        var_name: str,
+        scope_qn: str,
+        local_node: Node | None = None,
+    ) -> str | None:
+        """Infer the type of a variable in a given scope."""
+        # Check if we have cached type info
+        if scope_qn in self._variable_types:
+            if var_name in self._variable_types[scope_qn]:
+                return self._variable_types[scope_qn][var_name]
+
+        # Try to infer from local node
+        if local_node:
+            inferred = self._infer_from_node(var_name, local_node)
+            if inferred:
+                if scope_qn not in self._variable_types:
+                    self._variable_types[scope_qn] = {}
+                self._variable_types[scope_qn][var_name] = inferred
+                return inferred
+
+        return None
+
+    def _infer_from_node(self, var_name: str, node: Node) -> str | None:
+        """Try to infer type from AST node."""
+        # Look for variable declaration
+        for child in node.children:
+            if child.type in (
+                "variable_declarator",
+                "variable_declaration",
+                "lexical_declaration",
+            ):
+                type_hint = self._get_type_from_declaration(child, var_name)
+                if type_hint:
+                    return type_hint
+        return None
+
+    def _get_type_from_declaration(self, node: Node, var_name: str) -> str | None:
+        """Extract type from a variable declaration."""
+        name_node = node.child_by_field_name(cs.FIELD_NAME)
+        if name_node:
+            name = safe_decode_text(name_node)
+            if name == var_name:
+                # Check for type annotation
+                type_node = node.child_by_field_name(cs.FIELD_TYPE)
+                if type_node:
+                    return safe_decode_text(type_node)
+
+                # Check for initialization value
+                value_node = node.child_by_field_name(cs.FIELD_VALUE)
+                if value_node:
+                    return self._infer_from_value(value_node)
+
+        return None
+
+    def _infer_from_value(self, node: Node) -> str | None:
+        """Infer type from a value node."""
+        type_mapping = {
+            "string": "str",
+            "string_literal": "str",
+            "integer": "int",
+            "integer_literal": "int",
+            "float": "float",
+            "floating_point_literal": "float",
+            "true": "bool",
+            "false": "bool",
+            "boolean_literal": "bool",
+            "list": "list",
+            "list_literal": "list",
+            "dictionary": "dict",
+            "dict_literal": "dict",
+            "tuple": "tuple",
+            "call_expression": None,  # Would need to resolve the call
+        }
+
+        return type_mapping.get(node.type)
+
+    def get_class_for_variable(
+        self,
+        var_name: str,
+        scope_qn: str,
+        module_qn: str,
+    ) -> str | None:
+        """Get the class type for a variable."""
+        var_type = self.infer_variable_type(var_name, scope_qn)
+        if not var_type:
+            return None
+
+        # Check if it's a class from imports
+        import_map = self.import_processor.get_import_mapping(module_qn)
+        if var_type in import_map:
+            return import_map[var_type]
+
+        # Check if it's a local class
+        class_qn = f"{module_qn}.{var_type}"
+        if class_qn in self.class_inheritance:
+            return class_qn
+
+        return var_type