apisec-code-bolt 0.1.0__py3-none-any.whl

apisec_code_bolt/core/discovery.py

@@ -0,0 +1,433 @@
+"""
+File discovery module for traversing project directories.
+
+This module handles:
+- Recursive directory traversal
+- .gitignore pattern matching
+- File filtering by extension, size, and exclusion patterns
+- Symlink handling
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pathspec
+
+from .config import FileDiscoveryConfig
+from .types import AnalysisNote, Language
+
+if TYPE_CHECKING:
+    pass
+
+
+# =============================================================================
+# File Discovery Result
+# =============================================================================
+
+
+@dataclass
+class DiscoveredFile:
+    """A file discovered for analysis."""
+
+    path: Path
+    relative_path: Path
+    language: Language | None
+    size_bytes: int
+
+    # Why was this file selected/skipped?
+    skip_reason: str | None = None
+
+
+@dataclass
+class DiscoveryResult:
+    """Result of file discovery."""
+
+    root: Path
+    files: list[DiscoveredFile] = field(default_factory=list)
+    skipped_files: list[DiscoveredFile] = field(default_factory=list)
+
+    # Statistics
+    total_scanned: int = 0
+    total_selected: int = 0
+    total_skipped: int = 0
+    total_size_bytes: int = 0
+
+    # Warnings
+    notes: list[AnalysisNote] = field(default_factory=list)
+
+    def files_by_language(self, language: Language) -> list[DiscoveredFile]:
+        """Get files for a specific language."""
+        return [f for f in self.files if f.language == language]
+
+
+# =============================================================================
+# Language Detection
+# =============================================================================
+
+
+# Extension to language mapping
+EXTENSION_TO_LANGUAGE: dict[str, Language] = {
+    # Python
+    ".py": Language.PYTHON,
+    ".pyw": Language.PYTHON,
+    ".pyi": Language.PYTHON,  # Type stubs
+    # Java
+    ".java": Language.JAVA,
+    # C# / .NET
+    ".cs": Language.CSHARP,
+    # JavaScript / TypeScript
+    ".js": Language.JAVASCRIPT,
+    ".mjs": Language.JAVASCRIPT,
+    ".cjs": Language.JAVASCRIPT,
+    ".ts": Language.JAVASCRIPT,
+    ".tsx": Language.JAVASCRIPT,
+}
+
+
+def detect_language(file_path: Path) -> Language | None:
+    """Detect programming language from file extension."""
+    suffix = file_path.suffix.lower()
+    return EXTENSION_TO_LANGUAGE.get(suffix)
+
+
+# =============================================================================
+# Gitignore Handling
+# =============================================================================
+
+
+class GitignoreManager:
+    """
+    Manages .gitignore patterns for a project.
+
+    Handles:
+    - Loading .gitignore from project root
+    - Loading nested .gitignore files in subdirectories
+    - Combining patterns for accurate matching
+    """
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+        self._specs: dict[Path, pathspec.PathSpec] = {}
+        self._load_gitignore(root)
+
+    def _load_gitignore(self, directory: Path) -> None:
+        """Load .gitignore from a directory if it exists."""
+        gitignore_path = directory / ".gitignore"
+        if gitignore_path.exists() and gitignore_path.is_file():
+            try:
+                with open(gitignore_path, encoding="utf-8", errors="ignore") as f:
+                    patterns = f.read().splitlines()
+                # Filter empty lines and comments
+                patterns = [
+                    p.strip() for p in patterns if p.strip() and not p.strip().startswith("#")
+                ]
+                if patterns:
+                    self._specs[directory] = pathspec.PathSpec.from_lines("gitwildmatch", patterns)
+            except Exception:
+                # Silently ignore gitignore parse errors
+                pass
+
+    def is_ignored(self, file_path: Path) -> bool:
+        """
+        Check if a file path should be ignored.
+
+        Checks against all applicable .gitignore files from root to file's directory.
+        """
+        try:
+            file_path.relative_to(self.root)
+        except ValueError:
+            return False
+
+        # Check each gitignore spec
+        for spec_dir, spec in self._specs.items():
+            try:
+                # Get path relative to the gitignore's directory
+                rel_to_spec = file_path.relative_to(spec_dir)
+                if spec.match_file(str(rel_to_spec)):
+                    return True
+            except ValueError:
+                continue
+
+        return False
+
+    def load_nested_gitignore(self, directory: Path) -> None:
+        """Load .gitignore from a nested directory."""
+        if directory not in self._specs:
+            self._load_gitignore(directory)
+
+
+# =============================================================================
+# File Discovery
+# =============================================================================
+
+
+class FileDiscoverer:
+    """
+    Discovers files for analysis in a project directory.
+
+    Handles:
+    - Recursive traversal
+    - .gitignore patterns
+    - File filtering (extension, size, exclusions)
+    - Symlink handling
+    """
+
+    def __init__(
+        self,
+        root: Path,
+        config: FileDiscoveryConfig | None = None,
+    ) -> None:
+        self.root = root.resolve()
+        self.config = config or FileDiscoveryConfig()
+
+        # Initialize gitignore manager
+        self._gitignore: GitignoreManager | None = None
+        if self.config.respect_gitignore:
+            self._gitignore = GitignoreManager(self.root)
+
+        # Build exclusion spec from config patterns
+        self._exclusion_spec = self._build_exclusion_spec()
+
+        # Max file size in bytes
+        self._max_size_bytes = int(self.config.max_file_size_mb * 1024 * 1024)
+
+    def _build_exclusion_spec(self) -> pathspec.PathSpec:
+        """Build pathspec from exclusion patterns."""
+        patterns = list(self.config.exclude_patterns)
+        return pathspec.PathSpec.from_lines("gitwildmatch", patterns)
+
+    def _should_skip_directory(self, dir_path: Path) -> tuple[bool, str | None]:
+        """
+        Check if a directory should be skipped entirely.
+
+        Returns (should_skip, reason).
+        """
+        dir_name = dir_path.name
+
+        # Always-excluded directories (hardcoded for safety)
+        if dir_name in self.config.ALWAYS_EXCLUDE:
+            return True, f"always_excluded:{dir_name}"
+
+        # Skip hidden directories (starting with .)
+        if dir_name.startswith(".") and dir_name not in {"."}:
+            return True, "hidden_directory"
+
+        # Check gitignore
+        if self._gitignore and self._gitignore.is_ignored(dir_path):
+            return True, "gitignore"
+
+        return False, None
+
+    def _should_skip_file(self, file_path: Path) -> tuple[bool, str | None]:
+        """
+        Check if a file should be skipped.
+
+        Returns (should_skip, reason).
+        """
+        file_name = file_path.name
+
+        # Skip hidden files
+        if file_name.startswith("."):
+            return True, "hidden_file"
+
+        # Check file extension - only analyze known languages
+        language = detect_language(file_path)
+        if language is None:
+            return True, "unsupported_extension"
+
+        # Check gitignore
+        if self._gitignore and self._gitignore.is_ignored(file_path):
+            return True, "gitignore"
+
+        # Check exclusion patterns
+        try:
+            relative = file_path.relative_to(self.root)
+            if self._exclusion_spec.match_file(str(relative)):
+                return True, "exclusion_pattern"
+        except ValueError:
+            pass
+
+        # Check test files if not including tests
+        if not self.config.include_tests:
+            relative_str = str(file_path.relative_to(self.root))
+            if self._is_test_file(file_path, relative_str):
+                return True, "test_file"
+
+        # Check file size
+        try:
+            size = file_path.stat().st_size
+            if size > self._max_size_bytes:
+                return True, f"file_too_large:{size}"
+        except OSError:
+            return True, "stat_error"
+
+        return False, None
+
+    def _is_test_file(self, file_path: Path, relative_str: str) -> bool:
+        """Check if a file is a test file."""
+        name = file_path.stem.lower()
+
+        # Common test file patterns
+        if name.startswith("test_") or name.endswith("_test"):
+            return True
+        if name in {"conftest", "test", "tests"}:
+            return True
+
+        # Test directories
+        parts = relative_str.lower().split(os.sep)
+        return bool(any(p in {"test", "tests", "testing", "__tests__"} for p in parts))
+
+    def discover(self) -> DiscoveryResult:
+        """
+        Discover all files for analysis.
+
+        Returns DiscoveryResult with selected and skipped files.
+        """
+        result = DiscoveryResult(root=self.root)
+
+        for file_info in self._walk_directory(self.root):
+            result.total_scanned += 1
+
+            if file_info.skip_reason:
+                result.skipped_files.append(file_info)
+                result.total_skipped += 1
+            else:
+                # Check max files limit
+                if result.total_selected >= self.config.max_files:
+                    file_info.skip_reason = "max_files_reached"
+                    result.skipped_files.append(file_info)
+                    result.total_skipped += 1
+                    result.notes.append(
+                        AnalysisNote(
+                            level="warning",
+                            message=f"Max files limit ({self.config.max_files}) reached, skipping remaining files",
+                        )
+                    )
+                    break
+
+                result.files.append(file_info)
+                result.total_selected += 1
+                result.total_size_bytes += file_info.size_bytes
+
+        return result
+
+    def _walk_directory(self, directory: Path) -> Iterator[DiscoveredFile]:
+        """
+        Walk a directory tree, yielding DiscoveredFile for each file.
+
+        Handles symlinks according to configuration.
+        """
+        try:
+            entries = list(directory.iterdir())
+        except PermissionError:
+            return
+        except OSError:
+            return
+
+        # Sort for deterministic ordering
+        entries.sort(key=lambda p: p.name.lower())
+
+        # Process files first, then directories
+        files = []
+        dirs = []
+
+        for entry in entries:
+            try:
+                if entry.is_symlink():
+                    if not self.config.follow_symlinks:
+                        continue
+                    # Resolve symlink
+                    entry = entry.resolve()
+
+                if entry.is_file():
+                    files.append(entry)
+                elif entry.is_dir():
+                    dirs.append(entry)
+            except OSError:
+                continue
+
+        # Yield files
+        for file_path in files:
+            skip, reason = self._should_skip_file(file_path)
+
+            try:
+                size = file_path.stat().st_size
+                relative = file_path.relative_to(self.root)
+            except (OSError, ValueError):
+                continue
+
+            language = detect_language(file_path) if not skip else None
+
+            yield DiscoveredFile(
+                path=file_path,
+                relative_path=relative,
+                language=language,
+                size_bytes=size,
+                skip_reason=reason,
+            )
+
+        # Recurse into directories
+        for dir_path in dirs:
+            skip, reason = self._should_skip_directory(dir_path)
+            if skip:
+                continue
+
+            # Load nested gitignore
+            if self._gitignore:
+                self._gitignore.load_nested_gitignore(dir_path)
+
+            yield from self._walk_directory(dir_path)
+
+
+# =============================================================================
+# Convenience Functions
+# =============================================================================
+
+
+def discover_files(
+    root: Path,
+    config: FileDiscoveryConfig | None = None,
+) -> DiscoveryResult:
+    """
+    Discover files for analysis in a project directory.
+
+    Args:
+        root: Project root directory
+        config: File discovery configuration
+
+    Returns:
+        DiscoveryResult with files to analyze
+    """
+    discoverer = FileDiscoverer(root, config)
+    return discoverer.discover()
+
+
+def discover_python_files(
+    root: Path,
+    config: FileDiscoveryConfig | None = None,
+) -> list[Path]:
+    """
+    Discover Python files in a project directory.
+
+    Convenience function that returns just the paths.
+    """
+    result = discover_files(root, config)
+    return [f.path for f in result.files if f.language == Language.PYTHON]
+
+
+def discover_java_files(
+    root: Path,
+    config: FileDiscoveryConfig | None = None,
+) -> list[Path]:
+    """
+    Discover Java files in a project directory.
+
+    Convenience function that returns just the paths.
+    """
+    result = discover_files(root, config)
+    return [f.path for f in result.files if f.language == Language.JAVA]

apisec_code_bolt/core/log_format.py

@@ -0,0 +1,115 @@
+"""
+Structured JSON logging for the CLI.
+
+When --log-format json is passed, all diagnostic output is emitted as
+newline-delimited JSON (NDJSON) to stderr instead of Rich-formatted text.
+Each line is a self-contained JSON object with a "level" and "event" key.
+
+The analyzer_summary() function emits the machine-readable scan result that
+Mohsin's deployment pipeline consumes to detect regressions and surface errors.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat(timespec="seconds")
+
+
+def _emit(obj: dict[str, Any]) -> None:
+    """Write one NDJSON record to stderr."""
+    sys.stderr.write(json.dumps(obj, default=str) + "\n")
+    sys.stderr.flush()
+
+
+def log_info(event: str, **kwargs: Any) -> None:
+    _emit({"ts": _now(), "level": "info", "event": event, **kwargs})
+
+
+def log_warning(event: str, **kwargs: Any) -> None:
+    _emit({"ts": _now(), "level": "warning", "event": event, **kwargs})
+
+
+def log_error(event: str, **kwargs: Any) -> None:
+    _emit({"ts": _now(), "level": "error", "event": event, **kwargs})
+
+
+def analyzer_summary(
+    project_root: Path,
+    result: Any,  # AnalysisResult — avoid circular import
+    probe_version: str,
+) -> dict[str, Any]:
+    """
+    Build the structured summary record emitted after a successful scan.
+
+    Shape (all fields always present, never null):
+    {
+      "ts": "2026-06-12T15:00:00+00:00",
+      "level": "info",
+      "event": "scan_complete",
+      "version": "0.1.0",
+      "project": "/path/to/project",
+      "routes_found": 42,
+      "files_analyzed": 120,
+      "files_failed": 2,
+      "frameworks": ["django", "celery"],
+      "languages": {"python": 118, "javascript": 2},
+      "parse_errors": [
+        {"file": "src/foo.py", "line": 12, "error": "SyntaxError: invalid syntax"}
+      ],
+      "stage_times_ms": {
+        "discover": 12, "parse": 340, "resolve": 5, "detect_frameworks": 3,
+        "flow_bindings": 8, "call_graph": 22, "extract": 45,
+        "analyze_flows": 180, "enrich": 14
+      },
+      "extractor_times_ms": {
+        "spring_boot": 38, "django": 7
+      },
+      "total_time_ms": 629
+    }
+    """
+    frameworks = [
+        f.name.lower() if hasattr(f, "name") else str(f) for f in result.frameworks_detected
+    ]
+
+    # Prefer structured error details (file + line + error) when available;
+    # fall back to splitting the legacy "path: message" strings.
+    if hasattr(result, "parse_error_details") and result.parse_error_details:
+        structured_errors = result.parse_error_details
+    else:
+        structured_errors = []
+        for err_str in result.parse_errors:
+            if ": " in err_str:
+                file_part, _, msg_part = err_str.partition(": ")
+                structured_errors.append({"file": str(file_part), "error": msg_part})
+            else:
+                structured_errors.append({"file": "", "error": err_str})
+
+    extractor_times = getattr(result, "extractor_times_ms", {})
+    routes_by_framework = getattr(result, "routes_by_framework", {})
+    files_skipped = getattr(result, "files_skipped", 0)
+
+    return {
+        "ts": _now(),
+        "level": "info",
+        "event": "scan_complete",
+        "version": probe_version,
+        "project": str(project_root),
+        "routes_found": len(result.manifest.entry_points),
+        "routes_by_framework": routes_by_framework,
+        "files_analyzed": result.files_analyzed,
+        "files_failed": result.files_failed,
+        "files_skipped": files_skipped,
+        "frameworks": frameworks,
+        "languages": result.files_by_language,
+        "parse_errors": structured_errors,
+        "stage_times_ms": result.stage_times_ms,
+        "extractor_times_ms": extractor_times,
+        "total_time_ms": result.total_time_ms,
+    }