kodit 0.4.3__py3-none-any.whl → 0.5.1__py3-none-any.whl

kodit/infrastructure/slicing/slicer.py

@@ -8,14 +8,19 @@ from collections import defaultdict
 from collections.abc import Generator
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, ClassVar
+from typing import Any
 
 import structlog
 from tree_sitter import Node, Parser, Tree
-from tree_sitter_language_pack import get_language
 
-from kodit.domain.entities import File, Snippet
+from kodit.domain.entities.git import GitFile, SnippetV2
 from kodit.domain.value_objects import LanguageMapping
+from kodit.infrastructure.slicing.ast_analyzer import (
+    ASTAnalyzer,
+    FunctionDefinition,
+    LanguageConfig,
+    ParsedFile,
+)
 
 
 @dataclass
@@ -43,105 +48,6 @@ class AnalyzerState:
     )
 
 
-class LanguageConfig:
-    """Language-specific configuration."""
-
-    CONFIGS: ClassVar[dict[str, dict[str, Any]]] = {
-        "python": {
-            "function_nodes": ["function_definition"],
-            "method_nodes": [],
-            "call_node": "call",
-            "import_nodes": ["import_statement", "import_from_statement"],
-            "extension": ".py",
-            "name_field": None,  # Use identifier child
-        },
-        "java": {
-            "function_nodes": ["method_declaration"],
-            "method_nodes": [],
-            "call_node": "method_invocation",
-            "import_nodes": ["import_declaration"],
-            "extension": ".java",
-            "name_field": None,
-        },
-        "c": {
-            "function_nodes": ["function_definition"],
-            "method_nodes": [],
-            "call_node": "call_expression",
-            "import_nodes": ["preproc_include"],
-            "extension": ".c",
-            "name_field": "declarator",
-        },
-        "cpp": {
-            "function_nodes": ["function_definition"],
-            "method_nodes": [],
-            "call_node": "call_expression",
-            "import_nodes": ["preproc_include", "using_declaration"],
-            "extension": ".cpp",
-            "name_field": "declarator",
-        },
-        "rust": {
-            "function_nodes": ["function_item"],
-            "method_nodes": [],
-            "call_node": "call_expression",
-            "import_nodes": ["use_declaration", "extern_crate_declaration"],
-            "extension": ".rs",
-            "name_field": "name",
-        },
-        "go": {
-            "function_nodes": ["function_declaration"],
-            "method_nodes": ["method_declaration"],
-            "call_node": "call_expression",
-            "import_nodes": ["import_declaration"],
-            "extension": ".go",
-            "name_field": None,
-        },
-        "javascript": {
-            "function_nodes": [
-                "function_declaration",
-                "function_expression",
-                "arrow_function",
-            ],
-            "method_nodes": [],
-            "call_node": "call_expression",
-            "import_nodes": ["import_statement", "import_declaration"],
-            "extension": ".js",
-            "name_field": None,
-        },
-        "csharp": {
-            "function_nodes": ["method_declaration"],
-            "method_nodes": ["constructor_declaration"],
-            "call_node": "invocation_expression",
-            "import_nodes": ["using_directive"],
-            "extension": ".cs",
-            "name_field": None,
-        },
-        "html": {
-            "function_nodes": ["script_element", "style_element"],
-            "method_nodes": ["element"],  # Elements with id/class attributes
-            "call_node": "attribute",
-            "import_nodes": ["script_element", "element"],  # script and link elements
-            "extension": ".html",
-            "name_field": None,
-        },
-        "css": {
-            "function_nodes": ["rule_set", "keyframes_statement"],
-            "method_nodes": ["media_statement"],
-            "call_node": "call_expression",
-            "import_nodes": ["import_statement"],
-            "extension": ".css",
-            "name_field": None,
-        },
-    }
-
-    # Aliases
-    CONFIGS["c++"] = CONFIGS["cpp"]
-    CONFIGS["typescript"] = CONFIGS["javascript"]
-    CONFIGS["ts"] = CONFIGS["javascript"]
-    CONFIGS["js"] = CONFIGS["javascript"]
-    CONFIGS["c#"] = CONFIGS["csharp"]
-    CONFIGS["cs"] = CONFIGS["csharp"]
-
-
 class Slicer:
     """Slicer that extracts code snippets from files."""
 
@@ -149,9 +55,9 @@ class Slicer:
         """Initialize an empty slicer."""
         self.log = structlog.get_logger(__name__)
 
-    def extract_snippets(  # noqa: C901
-        self, files: list[File], language: str = "python"
-    ) -> list[Snippet]:
+    def extract_snippets_from_git_files(
+        self, files: list[GitFile], language: str = "python"
+    ) -> list[SnippetV2]:
         """Extract code snippets from a list of files.
 
         Args:
@@ -171,26 +77,17 @@ class Slicer:
 
         language = language.lower()
 
-        # Get language configuration
-        if language not in LanguageConfig.CONFIGS:
-            self.log.debug("Skipping", language=language)
+        # Initialize ASTAnalyzer
+        try:
+            analyzer = ASTAnalyzer(language)
+        except ValueError:
+            self.log.debug("Skipping unsupported language", language=language)
             return []
 
-        config = LanguageConfig.CONFIGS[language]
-
-        # Initialize tree-sitter
-        tree_sitter_name = self._get_tree_sitter_language_name(language)
-        try:
-            ts_language = get_language(tree_sitter_name)  # type: ignore[arg-type]
-            parser = Parser(ts_language)
-        except Exception as e:
-            raise RuntimeError(f"Failed to load {language} parser: {e}") from e
-
-        # Create mapping from Paths to File objects and extract paths
-        path_to_file_map: dict[Path, File] = {}
-        file_paths: list[Path] = []
+        # Validate files
+        path_to_file_map: dict[Path, GitFile] = {}
         for file in files:
-            file_path = file.as_path()
+            file_path = Path(file.path)
 
             # Validate file matches language
             if not self._file_matches_language(file_path.suffix, language):
@@ -201,31 +98,27 @@ class Slicer:
                 raise FileNotFoundError(f"File not found: {file_path}")
 
             path_to_file_map[file_path] = file
-            file_paths.append(file_path)
 
-        # Initialize state
-        state = AnalyzerState(parser=parser)
-        state.files = file_paths
-        file_contents: dict[Path, str] = {}
+        # Parse files and extract definitions using ASTAnalyzer
+        parsed_files = analyzer.parse_files(files)
+        if not parsed_files:
+            return []
 
-        # Parse all files
-        for file_path in file_paths:
-            try:
-                with file_path.open("rb") as f:
-                    source_code = f.read()
-                tree = state.parser.parse(source_code)
-                state.asts[file_path] = tree
-            except OSError:
-                # Skip files that can't be parsed
-                continue
+        functions, _, _ = analyzer.extract_definitions(
+            parsed_files, include_private=True
+        )
 
-        # Build indexes
-        self._build_definition_and_import_indexes(state, config, language)
+        # Build state from ASTAnalyzer results
+        state = self._build_state_from_ast_analyzer(parsed_files, functions)
+        config = LanguageConfig.CONFIGS[language]
+
+        # Build call graph and snippets (Slicer-specific logic)
         self._build_call_graph(state, config)
         self._build_reverse_call_graph(state)
 
         # Extract snippets for all functions
-        snippets = []
+        file_contents: dict[Path, str] = {}
+        snippets: list[SnippetV2] = []
         for qualified_name in state.def_index:
             snippet_content = self._get_snippet(
                 qualified_name,
@@ -234,7 +127,7 @@ class Slicer:
                 {"max_depth": 2, "max_functions": 8},
             )
             if "not found" not in snippet_content:
-                snippet = self._create_snippet_entity(
+                snippet = self._create_snippet_entity_from_git_files(
                     qualified_name, snippet_content, language, state, path_to_file_map
                 )
                 snippets.append(snippet)
@@ -247,62 +140,42 @@ class Slicer:
             return False
 
         try:
-            return (
-                language == LanguageMapping.get_language_for_extension(file_extension)
+            return language == LanguageMapping.get_language_for_extension(
+                file_extension
             )
         except ValueError:
             # Extension not supported, so it doesn't match any language
             return False
 
-    def _get_tree_sitter_language_name(self, language: str) -> str:
-        """Map user language names to tree-sitter language names."""
-        mapping = {
-            "c++": "cpp",
-            "c": "c",
-            "cpp": "cpp",
-            "java": "java",
-            "rust": "rust",
-            "python": "python",
-            "go": "go",
-            "javascript": "javascript",
-            "typescript": "typescript",
-            "js": "javascript",
-            "ts": "typescript",
-            "csharp": "csharp",
-            "c#": "csharp",
-            "cs": "csharp",
-            "html": "html",
-            "css": "css",
-        }
-        return mapping.get(language, language)
-
-    def _build_definition_and_import_indexes(
-        self, state: AnalyzerState, config: dict[str, Any], language: str
-    ) -> None:
-        """Build definition and import indexes."""
-        for file_path, tree in state.asts.items():
-            # Build definition index
-            for node in self._walk_tree(tree.root_node):
-                if self._is_function_definition(node, config):
-                    qualified_name = self._qualify_name(
-                        node, file_path, config, language
-                    )
-                    if qualified_name:
-                        span = (node.start_byte, node.end_byte)
-                        state.def_index[qualified_name] = FunctionInfo(
-                            file=file_path,
-                            node=node,
-                            span=span,
-                            qualified_name=qualified_name,
-                        )
-
-            # Build import map
-            file_imports = {}
-            for node in self._walk_tree(tree.root_node):
-                if self._is_import_statement(node, config):
-                    imports = self._extract_imports(node)
-                    file_imports.update(imports)
-            state.imports[file_path] = file_imports
+    def _build_state_from_ast_analyzer(
+        self,
+        parsed_files: list["ParsedFile"],
+        functions: list["FunctionDefinition"],
+    ) -> AnalyzerState:
+        """Build AnalyzerState from ASTAnalyzer results."""
+        # Create a dummy parser (not used for new parsing)
+        from tree_sitter_language_pack import get_language
+
+        ts_language = get_language("python")
+        parser = Parser(ts_language)
+
+        state = AnalyzerState(parser=parser)
+
+        # Populate files and ASTs from ParsedFile objects
+        for parsed in parsed_files:
+            state.files.append(parsed.path)
+            state.asts[parsed.path] = parsed.tree
+
+        # Populate def_index from FunctionDefinition objects
+        for func_def in functions:
+            state.def_index[func_def.qualified_name] = FunctionInfo(
+                file=func_def.file,
+                node=func_def.node,
+                span=func_def.span,
+                qualified_name=func_def.qualified_name,
+            )
+
+        return state
 
     def _build_call_graph(self, state: AnalyzerState, config: dict[str, Any]) -> None:
         """Build call graph from function definitions."""
@@ -338,214 +211,6 @@ class Slicer:
             # Add children to queue
             queue.extend(current.children)
 
-    def _is_function_definition(self, node: Node, config: dict[str, Any]) -> bool:
-        """Check if node is a function definition."""
-        return node.type in (config["function_nodes"] + config["method_nodes"])
-
-    def _is_import_statement(self, node: Node, config: dict[str, Any]) -> bool:
-        """Check if node is an import statement."""
-        return node.type in config["import_nodes"]
-
-    def _extract_function_name(
-        self, node: Node, config: dict[str, Any], language: str
-    ) -> str | None:
-        """Extract function name from a function definition node."""
-        if language == "html":
-            return self._extract_html_element_name(node)
-        if language == "css":
-            return self._extract_css_rule_name(node)
-        if language == "go" and node.type == "method_declaration":
-            return self._extract_go_method_name(node)
-        if language in ["c", "cpp"] and config["name_field"]:
-            return self._extract_c_cpp_function_name(node, config)
-        if language == "rust" and config["name_field"]:
-            return self._extract_rust_function_name(node, config)
-        return self._extract_default_function_name(node)
-
-    def _extract_go_method_name(self, node: Node) -> str | None:
-        """Extract method name from Go method declaration."""
-        for child in node.children:
-            if child.type == "field_identifier" and child.text is not None:
-                return child.text.decode("utf-8")
-        return None
-
-    def _extract_c_cpp_function_name(
-        self, node: Node, config: dict[str, Any]
-    ) -> str | None:
-        """Extract function name from C/C++ function definition."""
-        declarator = node.child_by_field_name(config["name_field"])
-        if not declarator:
-            return None
-
-        if declarator.type == "function_declarator":
-            for child in declarator.children:
-                if child.type == "identifier" and child.text is not None:
-                    return child.text.decode("utf-8")
-        elif declarator.type == "identifier" and declarator.text is not None:
-            return declarator.text.decode("utf-8")
-        return None
-
-    def _extract_rust_function_name(
-        self, node: Node, config: dict[str, Any]
-    ) -> str | None:
-        """Extract function name from Rust function definition."""
-        name_node = node.child_by_field_name(config["name_field"])
-        if name_node and name_node.type == "identifier" and name_node.text is not None:
-            return name_node.text.decode("utf-8")
-        return None
-
-    def _extract_html_element_name(self, node: Node) -> str | None:
-        """Extract meaningful name from HTML element."""
-        if node.type == "script_element":
-            return "script"
-        if node.type == "style_element":
-            return "style"
-        if node.type == "element":
-            return self._extract_html_element_info(node)
-        return None
-
-    def _extract_html_element_info(self, node: Node) -> str | None:
-        """Extract element info with ID or class."""
-        for child in node.children:
-            if child.type == "start_tag":
-                tag_name = self._get_tag_name(child)
-                element_id = self._get_element_id(child)
-                class_name = self._get_element_class(child)
-
-                if element_id:
-                    return f"{tag_name or 'element'}#{element_id}"
-                if class_name:
-                    return f"{tag_name or 'element'}.{class_name}"
-                if tag_name:
-                    return tag_name
-        return None
-
-    def _get_tag_name(self, start_tag: Node) -> str | None:
-        """Get tag name from start_tag node."""
-        for child in start_tag.children:
-            if child.type == "tag_name" and child.text:
-                try:
-                    return child.text.decode("utf-8")
-                except UnicodeDecodeError:
-                    return None
-        return None
-
-    def _get_element_id(self, start_tag: Node) -> str | None:
-        """Get element ID from start_tag node."""
-        return self._get_attribute_value(start_tag, "id")
-
-    def _get_element_class(self, start_tag: Node) -> str | None:
-        """Get first class name from start_tag node."""
-        class_value = self._get_attribute_value(start_tag, "class")
-        return class_value.split()[0] if class_value else None
-
-    def _get_attribute_value(self, start_tag: Node, attr_name: str) -> str | None:
-        """Get attribute value from start_tag node."""
-        for child in start_tag.children:
-            if child.type == "attribute":
-                name = self._get_attr_name(child)
-                if name == attr_name:
-                    return self._get_attr_value(child)
-        return None
-
-    def _get_attr_name(self, attr_node: Node) -> str | None:
-        """Get attribute name."""
-        for child in attr_node.children:
-            if child.type == "attribute_name" and child.text:
-                try:
-                    return child.text.decode("utf-8")
-                except UnicodeDecodeError:
-                    return None
-        return None
-
-    def _get_attr_value(self, attr_node: Node) -> str | None:
-        """Get attribute value."""
-        for child in attr_node.children:
-            if child.type == "quoted_attribute_value":
-                for val_child in child.children:
-                    if val_child.type == "attribute_value" and val_child.text:
-                        try:
-                            return val_child.text.decode("utf-8")
-                        except UnicodeDecodeError:
-                            return None
-        return None
-
-    def _extract_css_rule_name(self, node: Node) -> str | None:
-        """Extract meaningful name from CSS rule."""
-        if node.type == "rule_set":
-            return self._extract_css_selector(node)
-        if node.type == "keyframes_statement":
-            return self._extract_keyframes_name(node)
-        if node.type == "media_statement":
-            return "@media"
-        return None
-
-    def _extract_css_selector(self, rule_node: Node) -> str | None:
-        """Extract CSS selector from rule_set."""
-        for child in rule_node.children:
-            if child.type == "selectors":
-                selector_parts = []
-                for selector_child in child.children:
-                    part = self._get_selector_part(selector_child)
-                    if part:
-                        selector_parts.append(part)
-                if selector_parts:
-                    return "".join(selector_parts[:2])  # First couple selectors
-        return None
-
-    def _get_selector_part(self, selector_node: Node) -> str | None:
-        """Get a single selector part."""
-        if selector_node.type == "class_selector":
-            return self._extract_class_selector(selector_node)
-        if selector_node.type == "id_selector":
-            return self._extract_id_selector(selector_node)
-        if selector_node.type == "type_selector" and selector_node.text:
-            return selector_node.text.decode("utf-8")
-        return None
-
-    def _extract_class_selector(self, node: Node) -> str | None:
-        """Extract class selector name."""
-        for child in node.children:
-            if child.type == "class_name":
-                for name_child in child.children:
-                    if name_child.type == "identifier" and name_child.text:
-                        return f".{name_child.text.decode('utf-8')}"
-        return None
-
-    def _extract_id_selector(self, node: Node) -> str | None:
-        """Extract ID selector name."""
-        for child in node.children:
-            if child.type == "id_name":
-                for name_child in child.children:
-                    if name_child.type == "identifier" and name_child.text:
-                        return f"#{name_child.text.decode('utf-8')}"
-        return None
-
-    def _extract_keyframes_name(self, node: Node) -> str | None:
-        """Extract keyframes animation name."""
-        for child in node.children:
-            if child.type == "keyframes_name" and child.text:
-                return f"@keyframes-{child.text.decode('utf-8')}"
-        return None
-
-    def _extract_default_function_name(self, node: Node) -> str | None:
-        """Extract function name using default identifier search."""
-        for child in node.children:
-            if child.type == "identifier" and child.text is not None:
-                return child.text.decode("utf-8")
-        return None
-
-    def _qualify_name(
-        self, node: Node, file_path: Path, config: dict[str, Any], language: str
-    ) -> str | None:
-        """Create qualified name for a function node."""
-        function_name = self._extract_function_name(node, config, language)
-        if not function_name:
-            return None
-
-        module_name = file_path.stem
-        return f"{module_name}.{function_name}"
-
     def _get_file_content(self, file_path: Path, file_contents: dict[Path, str]) -> str:
         """Get cached file content."""
         if file_path not in file_contents:
@@ -614,7 +279,8 @@ class Slicer:
             if callers:
                 snippet_lines.append("")
                 snippet_lines.append("# === USAGE EXAMPLES ===")
-                for caller in list(callers)[:2]:  # Show up to 2 examples
+                # Show up to 2 examples, sorted for deterministic order
+                for caller in sorted(callers)[:2]:
                     call_line = self._find_function_call_line(
                         caller, function_name, state, file_contents
                     )
@@ -625,37 +291,37 @@ class Slicer:
 
         return "\n".join(snippet_lines)
 
-    def _create_snippet_entity(
+    def _create_snippet_entity_from_git_files(
         self,
         qualified_name: str,
         snippet_content: str,
         language: str,
         state: AnalyzerState,
-        path_to_file_map: dict[Path, File],
-    ) -> Snippet:
+        path_to_file_map: dict[Path, GitFile],
+    ) -> SnippetV2:
         """Create a Snippet domain entity from extracted content."""
         # Determine all files that this snippet derives from
-        derives_from_files = self._find_source_files_for_snippet(
+        derives_from_files = self._find_source_files_for_snippet_from_git_files(
             qualified_name, snippet_content, state, path_to_file_map
         )
 
         # Create the snippet entity
-        snippet = Snippet(derives_from=derives_from_files)
-
-        # Add the original content
-        snippet.add_original_content(snippet_content, language)
-
-        return snippet
+        return SnippetV2(
+            derives_from=derives_from_files,
+            content=snippet_content,
+            extension=language,
+            sha=SnippetV2.compute_sha(snippet_content),
+        )
 
-    def _find_source_files_for_snippet(
+    def _find_source_files_for_snippet_from_git_files(
         self,
         qualified_name: str,
         snippet_content: str,
         state: AnalyzerState,
-        path_to_file_map: dict[Path, File],
-    ) -> list[File]:
+        path_to_file_map: dict[Path, GitFile],
+    ) -> list[GitFile]:
         """Find all source files that a snippet derives from."""
-        source_files: list[File] = []
+        source_files: list[GitFile] = []
         source_file_paths: set[Path] = set()
 
         # Add the primary function's file
@@ -835,7 +501,7 @@ class Slicer:
             # Add direct dependencies
             to_visit.extend(
                 (callee, depth + 1)
-                for callee in state.call_graph.get(current, set())
+                for callee in sorted(state.call_graph.get(current, set()))
                 if callee not in visited and callee in state.def_index
             )
 
@@ -850,26 +516,26 @@ class Slicer:
         in_degree: dict[str, int] = defaultdict(int)
         graph: dict[str, set[str]] = defaultdict(set)
 
-        for func in functions:
-            for callee in state.call_graph.get(func, set()):
+        for func in sorted(functions):
+            for callee in sorted(state.call_graph.get(func, set())):
                 if callee in functions:
                     graph[func].add(callee)
                     in_degree[callee] += 1
 
         # Find roots
-        queue = [f for f in functions if in_degree[f] == 0]
+        queue = [f for f in sorted(functions) if in_degree[f] == 0]
         result = []
 
         while queue:
             current = queue.pop(0)
             result.append(current)
-            for neighbor in graph[current]:
+            for neighbor in sorted(graph[current]):
                 in_degree[neighbor] -= 1
                 if in_degree[neighbor] == 0:
                     queue.append(neighbor)
 
         # Add any remaining (cycles)
-        for func in functions:
+        for func in sorted(functions):
             if func not in result:
                 result.append(func)