apisec-code-bolt 0.1.0__py3-none-any.whl

apisec_code_bolt/parsing/python/parser.py

@@ -0,0 +1,719 @@
+"""
+Python parser using LibCST.
+
+This module provides the main Python parsing interface that:
+- Parses Python source files into CST
+- Extracts structural information using visitors
+- Resolves types and schemas
+- Produces ParsedFile objects for analysis
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Iterator, Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, ClassVar
+
+import libcst as cst
+
+from ...core.types import (
+    CodeLocation,
+    Language,
+    ParseError,
+    QualifiedName,
+)
+from ..base import (
+    BaseParser,
+    ParsedArgument,
+    ParsedAssignment,
+    ParsedCallSite,
+    ParsedClass,
+    ParsedDecorator,
+    ParsedField,
+    ParsedFile,
+    ParsedFunction,
+    ParsedImport,
+    ParsedParameter,
+    ParsedReturn,
+    ParserRegistry,
+)
+from .type_resolver import TypeResolver
+from .visitors import PythonExtractor
+
+if TYPE_CHECKING:
+    from .visitors import ExtractedClass, ExtractedField, ExtractedFunction
+
+
+# =============================================================================
+# Python Parser
+# =============================================================================
+
+
+class PythonParser(BaseParser):
+    """
+    Parser for Python source files using LibCST.
+
+    Handles:
+    - Full CST parsing with position tracking
+    - Function and class extraction
+    - Import analysis
+    - Call site extraction
+    - Assignment tracking
+    - Type resolution
+    """
+
+    LANGUAGE: ClassVar[Language] = Language.PYTHON
+    SUPPORTED_EXTENSIONS: ClassVar[frozenset[str]] = frozenset({".py", ".pyw", ".pyi"})
+
+    def __init__(self) -> None:
+        self._type_resolver = TypeResolver()
+
+    def parse_file(self, file_path: Path | str) -> ParsedFile:
+        """
+        Parse a Python source file.
+
+        Args:
+            file_path: Path to the Python file
+
+        Returns:
+            ParsedFile with all extracted information
+
+        Raises:
+            ParseError: If the file cannot be parsed
+        """
+        start_time = time.perf_counter()
+
+        # Ensure Path object
+        if isinstance(file_path, str):
+            file_path = Path(file_path)
+
+        try:
+            source = file_path.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            # Try with different encodings
+            try:
+                source = file_path.read_text(encoding="latin-1")
+            except Exception as e:
+                return ParsedFile(
+                    path=file_path,
+                    language=Language.PYTHON,
+                    success=False,
+                    error=ParseError(str(e), file_path),
+                )
+        except Exception as e:
+            return ParsedFile(
+                path=file_path,
+                language=Language.PYTHON,
+                success=False,
+                error=ParseError(str(e), file_path),
+            )
+
+        result = self.parse_source(source, file_path)
+
+        # Calculate parse time
+        elapsed_ms = int((time.perf_counter() - start_time) * 1000)
+        result.parse_time_ms = elapsed_ms
+
+        return result
+
+    def parse_source(self, source: str, file_path: Path | None = None) -> ParsedFile:
+        """
+        Parse Python source code string.
+
+        Args:
+            source: Python source code
+            file_path: Optional file path for location info
+
+        Returns:
+            ParsedFile with all extracted information
+        """
+        path = file_path or Path("unknown.py")
+
+        # Infer module name from path
+        module_name = self._infer_module_name(path)
+
+        try:
+            # Parse with LibCST
+            tree = cst.parse_module(source)
+
+            # Wrap for metadata (position tracking)
+            wrapper = cst.MetadataWrapper(tree)
+
+            # Create extractor and visit
+            extractor = PythonExtractor(source, path, module_name)
+            extractor.set_metadata_wrapper(wrapper)
+
+            # Walk the tree
+            wrapper.visit(extractor)
+
+            # Convert extracted data to ParsedFile format
+            return self._build_parsed_file(
+                path=path,
+                source=source,
+                extractor=extractor,
+                module_name=module_name,
+            )
+
+        except cst.ParserSyntaxError as e:
+            # Syntax error in source
+            # LibCST exception attributes vary by version
+            line_num = getattr(e, "raw_line", None) or 0
+            col_num = getattr(e, "raw_column", None) or 0
+            msg = str(e)
+
+            return ParsedFile(
+                path=path,
+                language=Language.PYTHON,
+                success=False,
+                error=ParseError(
+                    f"Syntax error: {msg}",
+                    path,
+                    line=line_num,
+                    column=col_num,
+                ),
+                line_count=source.count("\n") + 1,
+            )
+        except Exception as e:
+            return ParsedFile(
+                path=path,
+                language=Language.PYTHON,
+                success=False,
+                error=ParseError(str(e), path),
+                line_count=source.count("\n") + 1,
+            )
+
+    def parse_files(self, file_paths: Sequence[Path]) -> Iterator[ParsedFile]:
+        """
+        Parse multiple Python files.
+
+        Yields ParsedFile for each file, including failures.
+        """
+        for path in file_paths:
+            if self.can_parse(path):
+                yield self.parse_file(path)
+
+    # =========================================================================
+    # Conversion Methods
+    # =========================================================================
+
+    def _build_parsed_file(
+        self,
+        path: Path,
+        source: str,
+        extractor: PythonExtractor,
+        module_name: str,
+    ) -> ParsedFile:
+        """Build ParsedFile from extraction results."""
+        # Convert imports
+        imports = [self._convert_import(imp) for imp in extractor.imports]
+
+        # Convert functions
+        functions = [self._convert_function(func, path) for func in extractor.functions]
+
+        # Convert classes
+        classes = [self._convert_class(cls, path) for cls in extractor.classes]
+
+        # Convert call sites
+        call_sites = [self._convert_call(call, path) for call in extractor.calls]
+
+        # Convert assignments (pass module_name for target_qualified_name)
+        assignments = [
+            self._convert_assignment(assign, path, module_name) for assign in extractor.assignments
+        ]
+
+        # Build symbol table
+        all_symbols = {}
+        for func in functions:
+            all_symbols[func.qualified_name.full] = func
+        for cls in classes:
+            all_symbols[cls.qualified_name.full] = cls
+            for method in cls.methods:
+                all_symbols[method.qualified_name.full] = method
+
+        return ParsedFile(
+            path=path,
+            language=Language.PYTHON,
+            success=True,
+            module_name=module_name,
+            module_docstring=extractor.module_docstring,
+            imports=imports,
+            functions=functions,
+            classes=classes,
+            all_symbols=all_symbols,
+            call_sites=call_sites,
+            assignments=assignments,
+            module_variables=extractor.module_variables,
+            line_count=source.count("\n") + 1,
+        )
+
+    def _convert_import(self, imp) -> ParsedImport:
+        """Convert extracted import to ParsedImport."""
+        names = [name for name, _ in imp.names] if imp.is_from_import else []
+
+        return ParsedImport(
+            module=imp.module,
+            names=names,
+            alias=imp.names[0][1] if imp.names and imp.names[0][1] else None,
+            is_relative=imp.is_relative,
+            relative_level=imp.relative_level,
+            location=CodeLocation(
+                file=Path("unknown"),
+                line=imp.line,
+            )
+            if imp.line
+            else None,
+        )
+
+    def _convert_function(self, func: ExtractedFunction, path: Path) -> ParsedFunction:
+        """Convert extracted function to ParsedFunction."""
+        # Convert parameters
+        params = [
+            ParsedParameter(
+                name=p.name,
+                type_annotation=p.annotation,
+                default_value=p.default,
+                is_variadic=p.is_variadic,
+                is_keyword_variadic=p.is_keyword_variadic,
+            )
+            for p in func.parameters
+        ]
+
+        # Convert decorators
+        decorators = [
+            ParsedDecorator(
+                name=d.name,
+                qualified_name=QualifiedName(module="", name=d.full_name),
+                arguments=d.arguments,
+                positional_args=d.positional_args,
+                location=d.location,
+            )
+            for d in func.decorators
+        ]
+
+        # Build qualified name
+        parts = func.qualified_name.rsplit(".", 1)
+        module = parts[0] if len(parts) > 1 else ""
+        name = parts[-1]
+
+        # Convert return statements
+        return_statements = [
+            ParsedReturn(
+                line=r.line,
+                returns_none=r.returns_none,
+                returns_call=r.returns_call,
+                returns_variable=r.returns_variable,
+                returns_literal=r.returns_literal,
+                returns_expression=r.returns_expression,
+                returns_lambda=getattr(r, "returns_lambda", False),
+                returns_comprehension=getattr(r, "returns_comprehension", False),
+                call_name=r.call_name,
+                variable_name=r.variable_name,
+                literal_type=r.literal_type,
+                expression_text=r.expression_text,
+            )
+            for r in getattr(func, "return_statements", [])
+        ]
+
+        return ParsedFunction(
+            name=func.name,
+            qualified_name=QualifiedName(module=module, name=name),
+            location=CodeLocation(
+                file=path,
+                line=func.line,
+                column=func.column,
+                end_line=func.end_line,
+            ),
+            parameters=params,
+            return_type=func.return_annotation,
+            decorators=decorators,
+            is_async=func.is_async,
+            binding=func.binding,
+            owner_type=func.owner_type,
+            docstring=func.docstring,
+            body_line_count=func.end_line - func.line if func.end_line > func.line else 1,
+            body_source=func.body_source,
+            has_yield=getattr(func, "has_yield", False),
+            has_return=getattr(func, "has_return", False),
+            local_variables=func.local_variables,
+            control_flow_info=getattr(func, "control_flow_info", {}),
+            return_statements=return_statements,
+        )
+
+    def _convert_class(self, cls: ExtractedClass, path: Path) -> ParsedClass:
+        """Convert extracted class to ParsedClass."""
+        # Convert methods
+        methods = [self._convert_function(method, path) for method in cls.methods]
+
+        # Convert decorators
+        decorators = [
+            ParsedDecorator(
+                name=d.name,
+                qualified_name=QualifiedName(module="", name=d.full_name),
+                arguments=d.arguments,
+                positional_args=d.positional_args,
+                location=d.location,
+            )
+            for d in cls.decorators
+        ]
+
+        # Convert fields (for Pydantic models and dataclasses)
+        fields = [self._convert_field(f, path) for f in cls.fields]
+
+        # Build qualified name
+        parts = cls.qualified_name.rsplit(".", 1)
+        module = parts[0] if len(parts) > 1 else ""
+        name = parts[-1]
+
+        # Detect if this is a Pydantic model
+        is_pydantic = cls.is_pydantic_model or any(
+            base in {"BaseModel", "BaseSettings", "pydantic.BaseModel", "pydantic.BaseSettings"}
+            for base in cls.bases
+        )
+
+        return ParsedClass(
+            name=cls.name,
+            qualified_name=QualifiedName(module=module, name=name),
+            location=CodeLocation(
+                file=path,
+                line=cls.line,
+                column=cls.column,
+                end_line=cls.end_line,
+            ),
+            base_classes=cls.bases,
+            decorators=decorators,
+            fields=fields,
+            methods=methods,
+            class_variables=cls.class_variables,
+            instance_variables=cls.instance_variables,
+            docstring=cls.docstring,
+            is_dataclass=cls.is_dataclass,
+            is_pydantic_model=is_pydantic,
+        )
+
+    def _convert_field(self, f: ExtractedField, path: Path) -> ParsedField:
+        """Convert extracted field to ParsedField."""
+        # Extract constraints from field_info
+        constraints = {}
+        for key in ["min_length", "max_length", "gt", "ge", "lt", "le", "regex", "pattern"]:
+            if key in f.field_info:
+                constraints[key] = f.field_info[key]
+
+        # Determine if required
+        is_required = f.default is None or f.default == "..."
+        if "default" in f.field_info:
+            is_required = False
+
+        return ParsedField(
+            name=f.name,
+            type_annotation=f.annotation,
+            default_value=f.default,
+            field_info=f.field_info,
+            is_required=is_required,
+            alias=f.field_info.get("alias"),
+            description=f.field_info.get("description"),
+            constraints=constraints,
+        )
+
+    def _convert_call(self, call, path: Path) -> ParsedCallSite:
+        """Convert extracted call to ParsedCallSite."""
+        # Convert arguments
+        arguments = [
+            ParsedArgument(
+                position=arg.position,
+                name=arg.keyword,
+                is_literal=arg.is_literal,
+                literal_value=arg.literal_value,
+                literal_type=arg.literal_type,
+                is_variable=arg.is_name,
+                variable_name=arg.name_value,
+                is_expression=not arg.is_literal and not arg.is_name,
+                expression_text=arg.value_source
+                if not arg.is_literal and not arg.is_name
+                else None,
+                is_call_result=getattr(arg, "is_call_result", False),
+                called_function=getattr(arg, "called_function", None),
+                is_spread=arg.is_starred,
+                is_keyword_spread=arg.is_double_starred,
+                is_string_interpolation=getattr(arg, "is_string_interpolation", False),
+                is_concatenation=getattr(arg, "is_concatenation", False),
+                is_format_call=getattr(arg, "is_format_call", False),
+                container_type=getattr(arg, "container_type", None),
+                source_variables=getattr(arg, "source_variables", None) or [],
+            )
+            for arg in call.arguments
+        ]
+
+        # Parse callee for qualified name
+        callee_qn = None
+        if "." in call.callee:
+            parts = call.callee.rsplit(".", 1)
+            callee_qn = QualifiedName(module=parts[0], name=parts[1])
+        else:
+            callee_qn = QualifiedName(module="", name=call.callee)
+
+        # Caller qualified name
+        caller_qn = None
+        if call.in_function:
+            parts = call.in_function.rsplit(".", 1)
+            caller_qn = QualifiedName(
+                module=parts[0] if len(parts) > 1 else "",
+                name=parts[-1],
+            )
+
+        return ParsedCallSite(
+            callee_name=call.callee,
+            callee_qualified_name=callee_qn,
+            callee_resolved=False,  # Will be resolved later in analysis
+            location=CodeLocation(
+                file=path,
+                line=call.line,
+                column=call.column,
+                end_line=call.end_line or None,
+            ),
+            caller_function=caller_qn,
+            arguments=arguments,
+            is_method_call=call.is_method_call,
+            receiver_expression=call.receiver,
+            # Control flow context
+            in_loop=getattr(call, "in_loop", False),
+            in_conditional=getattr(call, "in_conditional", False),
+            in_try=getattr(call, "in_try", False),
+            in_except=getattr(call, "in_except", False),
+            in_finally=getattr(call, "in_finally", False),
+            in_with=getattr(call, "in_with", False),
+            in_comprehension=getattr(call, "in_comprehension", False),
+            loop_depth=getattr(call, "loop_depth", 0),
+            conditional_depth=getattr(call, "conditional_depth", 0),
+        )
+
+    def _convert_assignment(self, assign, path: Path, module_name: str = "") -> ParsedAssignment:
+        """Convert extracted assignment to ParsedAssignment."""
+        in_function_str = (
+            getattr(assign.in_function, "full", str(assign.in_function))
+            if getattr(assign, "in_function", None) is not None
+            else None
+        )
+        # Build target_qualified_name so flow-sensitive CFG can attribute
+        # assignments to functions (value-flow / variable_derives_from).
+        target_qualified_name = self._assignment_target_qualified_name(
+            assign.target,
+            in_function_str,
+            module_name,
+        )
+        return ParsedAssignment(
+            target=assign.target,
+            location=CodeLocation(
+                file=path,
+                line=assign.line,
+            ),
+            source_type=(
+                "literal"
+                if assign.is_literal
+                else "call"
+                if assign.is_call
+                else "variable"
+                if assign.is_name
+                else "expression"
+            ),
+            source_value=assign.value_source,
+            source_call=assign.called_function if assign.is_call else None,
+            in_function=in_function_str,
+            type_annotation=assign.annotation,
+            target_qualified_name=target_qualified_name,
+            source_variables=getattr(assign, "source_variables", None) or [],
+            is_method_call=getattr(assign, "is_method_call", False),
+            is_string_interpolation=getattr(assign, "is_string_interpolation", False),
+        )
+
+    def _assignment_target_qualified_name(
+        self,
+        target: str,
+        in_function: str | None,
+        module_name: str,
+    ) -> QualifiedName | None:
+        """
+        Build qualified name for an assignment target for flow-sensitive analysis.
+        - Inside a function: "module.func_name.target" so it starts with func qname.
+        - Module-level: "module_name.target" (two parts for module-level binding).
+        """
+        if in_function:
+            # Local assignment: full = "module.func.target"
+            if "." in in_function:
+                mod, func_part = in_function.rsplit(".", 1)
+                name = f"{func_part}.{target}"
+            else:
+                mod = ""
+                name = f"{in_function}.{target}"
+            return QualifiedName(module=mod, name=name)
+        if module_name:
+            return QualifiedName(module=module_name, name=target)
+        return None
+
+    # =========================================================================
+    # Helper Methods
+    # =========================================================================
+
+    def _infer_module_name(self, file_path: Path) -> str:
+        """
+        Infer Python module name from file path.
+
+        e.g., /project/src/app/routes/users.py -> app.routes.users
+        """
+        # Remove .py extension
+        stem = file_path.stem
+        if stem == "__init__":
+            # For __init__.py, use parent directory name
+            return file_path.parent.name
+
+        # Try to build module path from directory structure
+        parts = []
+        current = file_path.parent
+
+        # Walk up looking for __init__.py or until we hit a non-package directory
+        while current.name:
+            init_file = current / "__init__.py"
+            if not init_file.exists():
+                break
+            parts.append(current.name)
+            current = current.parent
+
+        parts.reverse()
+        parts.append(stem)
+
+        return ".".join(parts) if parts else stem
+
+
+# =============================================================================
+# Parser Registration
+# =============================================================================
+
+
+# Create singleton parser instance
+_python_parser = PythonParser()
+
+
+def get_python_parser() -> PythonParser:
+    """Get the Python parser instance."""
+    return _python_parser
+
+
+# Register with parser registry
+ParserRegistry.register(_python_parser)
+
+
+# =============================================================================
+# Project-Level Parsing
+# =============================================================================
+
+
+def _parse_file_worker(path: Path) -> ParsedFile:
+    """Top-level worker function for ProcessPoolExecutor (must be picklable)."""
+    parser = PythonParser()
+    return parser.parse_file(path)
+
+
+class PythonProjectParser:
+    """
+    Parser for entire Python projects.
+
+    Handles:
+    - Multi-file parsing (parallel when > PARALLEL_THRESHOLD files)
+    - Cross-file type resolution
+    - Import resolution
+    - Building complete project model
+    """
+
+    PARALLEL_THRESHOLD = 20
+
+    def __init__(self) -> None:
+        self._parser = PythonParser()
+        self._type_resolver = TypeResolver()
+        self._parsed_files: dict[Path, ParsedFile] = {}
+        self._all_classes: dict[str, ParsedClass] = {}
+        self._all_functions: dict[str, ParsedFunction] = {}
+
+    def parse_project(self, file_paths: list[Path]) -> dict[Path, ParsedFile]:
+        """
+        Parse all Python files in a project.
+
+        Uses ProcessPoolExecutor for parallelism on large projects.
+        """
+        import logging
+
+        logger = logging.getLogger(__name__)
+
+        if len(file_paths) >= self.PARALLEL_THRESHOLD:
+            self._parse_parallel(file_paths, logger)
+        else:
+            self._parse_serial(file_paths)
+
+        self._resolve_references()
+        return self._parsed_files
+
+    def _parse_serial(self, file_paths: list[Path]) -> None:
+        for path in file_paths:
+            parsed = self._parser.parse_file(path)
+            self._register_parsed(path, parsed)
+
+    def _parse_parallel(self, file_paths: list[Path], logger: logging.Logger) -> None:
+        import os
+        from concurrent.futures import ProcessPoolExecutor, as_completed
+
+        max_workers = min(os.cpu_count() or 1, 8)
+        logger.info("Parallel parsing %d files with %d workers", len(file_paths), max_workers)
+
+        try:
+            with ProcessPoolExecutor(max_workers=max_workers) as pool:
+                future_to_path = {
+                    pool.submit(_parse_file_worker, path): path for path in file_paths
+                }
+                for future in as_completed(future_to_path):
+                    path = future_to_path[future]
+                    try:
+                        parsed = future.result()
+                        self._register_parsed(path, parsed)
+                    except Exception as e:
+                        logger.warning("Worker failed for %s: %s", path, e)
+                        self._parsed_files[path] = ParsedFile(
+                            path=path,
+                            language=Language.PYTHON,
+                            success=False,
+                            error=str(e),
+                        )
+        except Exception as e:
+            logger.warning("Parallel parsing failed, falling back to serial: %s", e)
+            self._parse_serial(file_paths)
+
+    def _register_parsed(self, path: Path, parsed: ParsedFile) -> None:
+        self._parsed_files[path] = parsed
+        if parsed.success:
+            for imp in parsed.imports:
+                self._type_resolver.add_import(imp)
+            for cls in parsed.classes:
+                self._all_classes[cls.qualified_name.full] = cls
+                self._type_resolver.add_class(cls, path)
+            for func in parsed.functions:
+                self._all_functions[func.qualified_name.full] = func
+
+    def _resolve_references(self) -> None:
+        """Resolve cross-file type references and imports."""
+        # This would involve:
+        # 1. Resolving import targets to actual definitions
+        # 2. Resolving type annotations to class definitions
+        # 3. Building inheritance hierarchies
+        # 4. Resolving function call targets
+        pass
+
+    def get_class(self, qualified_name: str) -> ParsedClass | None:
+        """Get a class by qualified name."""
+        return self._all_classes.get(qualified_name)
+
+    def get_function(self, qualified_name: str) -> ParsedFunction | None:
+        """Get a function by qualified name."""
+        return self._all_functions.get(qualified_name)
+
+    def get_type_resolver(self) -> TypeResolver:
+        """Get the type resolver with all registered types."""
+        return self._type_resolver