emdash-core 0.1.7__py3-none-any.whl

emdash_core/ingestion/parsers/base_parser.py

@@ -0,0 +1,66 @@
+"""Abstract base class for language-specific parsers."""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List, Optional
+
+from ...core.models import FileEntities
+
+
+class BaseLanguageParser(ABC):
+    """Abstract base class for language-specific parsers."""
+
+    def __init__(self, file_path: Path, repo_root: Optional[Path] = None):
+        """Initialize parser.
+
+        Args:
+            file_path: Path to source file
+            repo_root: Root directory of repository (for module name calculation)
+        """
+        self.file_path = file_path
+        self.repo_root = repo_root or file_path.parent
+        self.module_name = self._calculate_module_name()
+
+    @abstractmethod
+    def parse(self) -> FileEntities:
+        """Parse file and extract code entities.
+
+        Returns:
+            FileEntities containing classes, functions, imports, etc.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def get_supported_extensions(cls) -> List[str]:
+        """Return list of file extensions this parser handles.
+
+        Returns:
+            List of file extensions (e.g., ['.py'] or ['.ts', '.tsx', '.js', '.jsx'])
+        """
+        pass
+
+    def _calculate_module_name(self) -> str:
+        """Calculate module name from file path (language-agnostic logic).
+
+        Returns:
+            Module name like "src.module.submodule"
+        """
+        try:
+            # Get relative path from repo root
+            relative_path = self.file_path.relative_to(self.repo_root)
+
+            # Convert path to module name
+            parts = list(relative_path.parts[:-1])  # Exclude filename
+            filename = relative_path.stem  # Get filename without extension
+
+            # Skip index files (index.ts, __init__.py)
+            if filename not in ["__init__", "index"]:
+                parts.append(filename)
+
+            module_name = ".".join(parts) if parts else filename
+            return module_name
+
+        except ValueError:
+            # File is outside repo root
+            return self.file_path.stem

emdash_core/ingestion/parsers/call_graph_builder.py

@@ -0,0 +1,121 @@
+"""Build call graph by analyzing function calls in Python AST."""
+
+import ast
+from typing import List
+
+from ...core.models import FunctionEntity
+
+
+class CallGraphBuilder:
+    """Builds call relationships between functions."""
+
+    def __init__(self, tree: ast.AST, module_name: str):
+        """Initialize call graph builder.
+
+        Args:
+            tree: Python AST
+            module_name: Module name
+        """
+        self.tree = tree
+        self.module_name = module_name
+
+    def build(self, functions: List[FunctionEntity]):
+        """Build call graph for all functions.
+
+        Mutates the functions list by populating the `calls` attribute.
+
+        Args:
+            functions: List of FunctionEntity objects to analyze
+        """
+        # Create a mapping from function name to qualified name
+        function_map = {func.name: func.qualified_name for func in functions}
+
+        # For each function, analyze its body for calls
+        for func in functions:
+            func.calls = self._extract_calls(func, function_map)
+
+    def _extract_calls(
+        self,
+        function: FunctionEntity,
+        function_map: dict
+    ) -> List[str]:
+        """Extract function calls from a function body.
+
+        Args:
+            function: FunctionEntity to analyze
+            function_map: Mapping of function names to qualified names
+
+        Returns:
+            List of qualified names of called functions
+        """
+        calls = []
+
+        # Find the function node in the AST
+        for node in ast.walk(self.tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                if node.lineno == function.line_start:
+                    # Found the function node
+                    calls = self._find_calls_in_node(node, function_map)
+                    break
+
+        return calls
+
+    def _find_calls_in_node(
+        self,
+        node: ast.AST,
+        function_map: dict
+    ) -> List[str]:
+        """Find all function calls within a node.
+
+        Args:
+            node: AST node to analyze
+            function_map: Mapping of function names to qualified names
+
+        Returns:
+            List of called function qualified names
+        """
+        calls = []
+
+        for child in ast.walk(node):
+            if isinstance(child, ast.Call):
+                called_name = self._get_call_name(child.func)
+
+                if called_name:
+                    # Try to resolve to qualified name
+                    if called_name in function_map:
+                        calls.append(function_map[called_name])
+                    else:
+                        # Keep the raw name even if we can't resolve it
+                        # In a second pass, we could resolve these across files
+                        calls.append(called_name)
+
+        return list(set(calls))  # Remove duplicates
+
+    def _get_call_name(self, node: ast.expr) -> str:
+        """Get the name of a called function from a Call node.
+
+        Args:
+            node: Function expression node
+
+        Returns:
+            Function name or empty string
+        """
+        if isinstance(node, ast.Name):
+            # Direct call: foo()
+            return node.id
+
+        elif isinstance(node, ast.Attribute):
+            # Method call: obj.foo()
+            # We'll track these as "Type.method" if possible
+            value_name = self._get_call_name(node.value)
+            if value_name:
+                return f"{value_name}.{node.attr}"
+            else:
+                return node.attr
+
+        elif isinstance(node, ast.Call):
+            # Chained call: foo()()
+            return self._get_call_name(node.func)
+
+        else:
+            return ""

emdash_core/ingestion/parsers/class_extractor.py

@@ -0,0 +1,154 @@
+"""Extract class definitions from Python AST."""
+
+import ast
+from pathlib import Path
+from typing import List
+
+from ...core.models import ClassEntity
+
+
+class ClassExtractor:
+    """Extracts class definitions from Python AST."""
+
+    def __init__(self, tree: ast.AST, file_path: Path, module_name: str):
+        """Initialize class extractor.
+
+        Args:
+            tree: Python AST
+            file_path: Path to source file
+            module_name: Module name (e.g., "src.module")
+        """
+        self.tree = tree
+        self.file_path = file_path
+        self.module_name = module_name
+
+    def extract(self) -> List[ClassEntity]:
+        """Extract all class definitions.
+
+        Returns:
+            List of ClassEntity objects
+        """
+        classes = []
+
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.ClassDef):
+                class_entity = self._extract_class(node)
+                if class_entity:
+                    classes.append(class_entity)
+
+        return classes
+
+    def _extract_class(self, node: ast.ClassDef) -> ClassEntity:
+        """Extract a single class definition.
+
+        Args:
+            node: ClassDef AST node
+
+        Returns:
+            ClassEntity
+        """
+        # Qualified name
+        qualified_name = f"{self.module_name}.{node.name}" if self.module_name else node.name
+
+        # Docstring
+        docstring = ast.get_docstring(node)
+
+        # Check if abstract
+        is_abstract = self._is_abstract(node)
+
+        # Decorators
+        decorators = [self._get_decorator_name(dec) for dec in node.decorator_list]
+
+        # Base classes
+        base_classes = []
+        for base in node.bases:
+            base_name = self._get_name(base)
+            if base_name:
+                base_classes.append(base_name)
+
+        # Extract attributes (class variables)
+        attributes = []
+        for item in node.body:
+            if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):
+                attributes.append(item.target.id)
+            elif isinstance(item, ast.Assign):
+                for target in item.targets:
+                    if isinstance(target, ast.Name):
+                        attributes.append(target.id)
+
+        # Extract method names (will be populated later with qualified names)
+        methods = []
+        for item in node.body:
+            if isinstance(item, ast.FunctionDef) or isinstance(item, ast.AsyncFunctionDef):
+                method_qualified_name = f"{qualified_name}.{item.name}"
+                methods.append(method_qualified_name)
+
+        return ClassEntity(
+            name=node.name,
+            qualified_name=qualified_name,
+            file_path=str(self.file_path),
+            line_start=node.lineno,
+            line_end=node.end_lineno or node.lineno,
+            docstring=docstring,
+            is_abstract=is_abstract,
+            decorators=decorators,
+            base_classes=base_classes,
+            attributes=attributes,
+            methods=methods,
+        )
+
+    def _is_abstract(self, node: ast.ClassDef) -> bool:
+        """Check if a class is abstract (has ABC or abstractmethod)."""
+        # Check if inherits from ABC
+        for base in node.bases:
+            base_name = self._get_name(base)
+            if base_name in ["ABC", "abc.ABC"]:
+                return True
+
+        # Check if any method has @abstractmethod
+        for item in node.body:
+            if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                for dec in item.decorator_list:
+                    dec_name = self._get_decorator_name(dec)
+                    if "abstractmethod" in dec_name:
+                        return True
+
+        return False
+
+    def _get_decorator_name(self, decorator: ast.expr) -> str:
+        """Get the name of a decorator.
+
+        Args:
+            decorator: Decorator AST node
+
+        Returns:
+            Decorator name as string
+        """
+        if isinstance(decorator, ast.Name):
+            return decorator.id
+        elif isinstance(decorator, ast.Attribute):
+            return f"{self._get_name(decorator.value)}.{decorator.attr}"
+        elif isinstance(decorator, ast.Call):
+            return self._get_decorator_name(decorator.func)
+        else:
+            return "unknown"
+
+    def _get_name(self, node: ast.expr) -> str:
+        """Get the name from an expression node.
+
+        Args:
+            node: AST expression node
+
+        Returns:
+            Name as string
+        """
+        if isinstance(node, ast.Name):
+            return node.id
+        elif isinstance(node, ast.Attribute):
+            value_name = self._get_name(node.value)
+            return f"{value_name}.{node.attr}" if value_name else node.attr
+        elif isinstance(node, ast.Subscript):
+            # For generics like List[int]
+            return self._get_name(node.value)
+        else:
+            return ""

emdash_core/ingestion/parsers/function_extractor.py

@@ -0,0 +1,202 @@
+"""Extract function and method definitions from Python AST."""
+
+import ast
+from pathlib import Path
+from typing import List, Optional
+
+from ...core.models import FunctionEntity
+
+
+class FunctionExtractor:
+    """Extracts function and method definitions from Python AST."""
+
+    def __init__(self, tree: ast.AST, file_path: Path, module_name: str):
+        """Initialize function extractor.
+
+        Args:
+            tree: Python AST
+            file_path: Path to source file
+            module_name: Module name
+        """
+        self.tree = tree
+        self.file_path = file_path
+        self.module_name = module_name
+        self.current_class: Optional[str] = None
+
+    def extract(self) -> List[FunctionEntity]:
+        """Extract all function and method definitions.
+
+        Returns:
+            List of FunctionEntity objects
+        """
+        functions = []
+
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.ClassDef):
+                # Track current class for method extraction
+                self._extract_methods(node, functions)
+            elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                # Only extract top-level functions (not methods)
+                # Methods are extracted via _extract_methods
+                if not self._is_method(node):
+                    function_entity = self._extract_function(node, is_method=False)
+                    if function_entity:
+                        functions.append(function_entity)
+
+        return functions
+
+    def _extract_methods(self, class_node: ast.ClassDef, functions: List[FunctionEntity]):
+        """Extract methods from a class.
+
+        Args:
+            class_node: ClassDef AST node
+            functions: List to append extracted methods to
+        """
+        class_qualified_name = f"{self.module_name}.{class_node.name}" if self.module_name else class_node.name
+
+        for item in class_node.body:
+            if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                function_entity = self._extract_function(
+                    item,
+                    is_method=True,
+                    parent_class=class_qualified_name
+                )
+                if function_entity:
+                    functions.append(function_entity)
+
+    def _is_method(self, node: ast.FunctionDef) -> bool:
+        """Check if a function node is a method (inside a class).
+
+        This is a simplification - in practice we extract methods separately.
+        """
+        return False
+
+    def _extract_function(
+        self,
+        node: ast.FunctionDef,
+        is_method: bool = False,
+        parent_class: Optional[str] = None
+    ) -> FunctionEntity:
+        """Extract a single function or method.
+
+        Args:
+            node: FunctionDef or AsyncFunctionDef AST node
+            is_method: Whether this is a class method
+            parent_class: Qualified name of parent class (if method)
+
+        Returns:
+            FunctionEntity
+        """
+        # Qualified name
+        if is_method and parent_class:
+            qualified_name = f"{parent_class}.{node.name}"
+        else:
+            qualified_name = f"{self.module_name}.{node.name}" if self.module_name else node.name
+
+        # Docstring
+        docstring = ast.get_docstring(node)
+
+        # Parameters
+        parameters = self._extract_parameters(node.args)
+
+        # Return annotation
+        return_annotation = None
+        if node.returns:
+            return_annotation = ast.unparse(node.returns)
+
+        # Check if async
+        is_async = isinstance(node, ast.AsyncFunctionDef)
+
+        # Decorators
+        decorators = []
+        is_static = False
+        is_classmethod = False
+
+        for dec in node.decorator_list:
+            dec_name = self._get_decorator_name(dec)
+            decorators.append(dec_name)
+
+            if dec_name == "staticmethod":
+                is_static = True
+            elif dec_name == "classmethod":
+                is_classmethod = True
+
+        # Cyclomatic complexity (simplified - count branches)
+        complexity = self._calculate_complexity(node)
+
+        return FunctionEntity(
+            name=node.name,
+            qualified_name=qualified_name,
+            file_path=str(self.file_path),
+            line_start=node.lineno,
+            line_end=node.end_lineno or node.lineno,
+            docstring=docstring,
+            parameters=parameters,
+            return_annotation=return_annotation,
+            is_async=is_async,
+            is_method=is_method,
+            is_static=is_static,
+            is_classmethod=is_classmethod,
+            decorators=decorators,
+            cyclomatic_complexity=complexity,
+        )
+
+    def _extract_parameters(self, args: ast.arguments) -> List[str]:
+        """Extract parameter names from function arguments.
+
+        Args:
+            args: arguments AST node
+
+        Returns:
+            List of parameter names with optional type annotations
+        """
+        parameters = []
+
+        # Regular args
+        for arg in args.args:
+            param = arg.arg
+            if arg.annotation:
+                param += f": {ast.unparse(arg.annotation)}"
+            parameters.append(param)
+
+        # *args
+        if args.vararg:
+            param = f"*{args.vararg.arg}"
+            if args.vararg.annotation:
+                param += f": {ast.unparse(args.vararg.annotation)}"
+            parameters.append(param)
+
+        # **kwargs
+        if args.kwarg:
+            param = f"**{args.kwarg.arg}"
+            if args.kwarg.annotation:
+                param += f": {ast.unparse(args.kwarg.annotation)}"
+            parameters.append(param)
+
+        return parameters
+
+    def _get_decorator_name(self, decorator: ast.expr) -> str:
+        """Get the name of a decorator."""
+        if isinstance(decorator, ast.Name):
+            return decorator.id
+        elif isinstance(decorator, ast.Attribute):
+            return f"{ast.unparse(decorator.value)}.{decorator.attr}"
+        elif isinstance(decorator, ast.Call):
+            return self._get_decorator_name(decorator.func)
+        else:
+            return "unknown"
+
+    def _calculate_complexity(self, node: ast.FunctionDef) -> int:
+        """Calculate cyclomatic complexity (simplified).
+
+        Counts decision points: if, while, for, except, with, and, or
+        """
+        complexity = 1  # Base complexity
+
+        for child in ast.walk(node):
+            if isinstance(child, (ast.If, ast.While, ast.For, ast.ExceptHandler, ast.With)):
+                complexity += 1
+            elif isinstance(child, (ast.And, ast.Or)):
+                complexity += 1
+
+        return complexity

emdash_core/ingestion/parsers/import_analyzer.py

@@ -0,0 +1,119 @@
+"""Analyze import statements from Python AST."""
+
+import ast
+from pathlib import Path
+from typing import List, Tuple
+
+from ...core.models import ImportStatement, ModuleEntity
+
+
+class ImportAnalyzer:
+    """Analyzes import statements and builds module dependency graph."""
+
+    def __init__(self, tree: ast.AST, file_path: Path, module_name: str):
+        """Initialize import analyzer.
+
+        Args:
+            tree: Python AST
+            file_path: Path to source file
+            module_name: Module name
+        """
+        self.tree = tree
+        self.file_path = file_path
+        self.module_name = module_name
+
+    def extract(self) -> Tuple[List[ImportStatement], List[ModuleEntity]]:
+        """Extract all import statements and create module entities.
+
+        Returns:
+            Tuple of (import_statements, module_entities)
+        """
+        import_statements = []
+        modules = {}  # Dict to avoid duplicates
+
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.Import):
+                for alias in node.names:
+                    stmt = ImportStatement(
+                        file_path=str(self.file_path),
+                        line_number=node.lineno,
+                        module=alias.name,
+                        imported_names=[alias.name],
+                        alias=alias.asname,
+                        import_type="import",
+                    )
+                    import_statements.append(stmt)
+
+                    # Create module entity
+                    if alias.name not in modules:
+                        modules[alias.name] = self._create_module_entity(alias.name)
+
+            elif isinstance(node, ast.ImportFrom):
+                if node.module:
+                    imported_names = [alias.name for alias in node.names]
+
+                    stmt = ImportStatement(
+                        file_path=str(self.file_path),
+                        line_number=node.lineno,
+                        module=node.module,
+                        imported_names=imported_names,
+                        alias=None,
+                        import_type="from_import",
+                    )
+                    import_statements.append(stmt)
+
+                    # Create module entity
+                    if node.module not in modules:
+                        modules[node.module] = self._create_module_entity(node.module)
+
+        return import_statements, list(modules.values())
+
+    def _create_module_entity(self, module_name: str) -> ModuleEntity:
+        """Create a module entity from an import.
+
+        Args:
+            module_name: Name of imported module
+
+        Returns:
+            ModuleEntity
+        """
+        # Determine if module is external (stdlib or third-party)
+        is_external = self._is_external_module(module_name)
+
+        # Extract package name (first component)
+        package = module_name.split('.')[0] if '.' in module_name else module_name
+
+        return ModuleEntity(
+            name=module_name,
+            import_path=module_name,
+            is_external=is_external,
+            package=package,
+        )
+
+    def _is_external_module(self, module_name: str) -> bool:
+        """Check if a module is external (not part of current codebase).
+
+        Args:
+            module_name: Module name
+
+        Returns:
+            True if external, False if internal
+        """
+        # Simple heuristic: if the module starts with a well-known package, it's external
+        # In a more sophisticated version, we'd check against the actual codebase structure
+
+        common_stdlib = {
+            'os', 'sys', 'json', 're', 'math', 'datetime', 'collections',
+            'itertools', 'functools', 'pathlib', 'typing', 'abc', 'asyncio',
+            'unittest', 'logging', 'argparse', 'dataclasses', 'enum',
+        }
+
+        common_third_party = {
+            'numpy', 'pandas', 'requests', 'flask', 'django', 'fastapi',
+            'sqlalchemy', 'pydantic', 'pytest', 'click', 'rich', 'boto3',
+            'tensorflow', 'torch', 'sklearn', 'matplotlib', 'seaborn',
+        }
+
+        root_package = module_name.split('.')[0]
+
+        return root_package in common_stdlib or root_package in common_third_party