lucidscan 0.5.12__py3-none-any.whl

lucidscan/mcp/watcher.py

@@ -0,0 +1,237 @@
+"""File watcher for incremental LucidScan checks.
+
+Watches for file changes and runs incremental quality checks.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Set
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+from lucidscan.config import LucidScanConfig
+from lucidscan.core.logging import get_logger
+from lucidscan.mcp.tools import MCPToolExecutor
+
+LOGGER = get_logger(__name__)
+
+
+class LucidScanFileWatcher:
+    """Watches for file changes and runs incremental checks."""
+
+    # Default patterns to ignore
+    DEFAULT_IGNORE_PATTERNS = [
+        ".git",
+        "__pycache__",
+        "node_modules",
+        ".venv",
+        "venv",
+        ".lucidscan",
+        ".mypy_cache",
+        ".pytest_cache",
+        ".ruff_cache",
+        "*.pyc",
+        "*.pyo",
+        ".coverage",
+        "htmlcov",
+        "dist",
+        "build",
+        "*.egg-info",
+    ]
+
+    def __init__(
+        self,
+        project_root: Path,
+        config: LucidScanConfig,
+        debounce_ms: int = 1000,
+        ignore_patterns: Optional[List[str]] = None,
+    ):
+        """Initialize LucidScanFileWatcher.
+
+        Args:
+            project_root: Project root directory to watch.
+            config: LucidScan configuration.
+            debounce_ms: Debounce delay in milliseconds.
+            ignore_patterns: Additional patterns to ignore.
+        """
+        self.project_root = project_root
+        self.config = config
+        self.debounce_ms = debounce_ms
+        self.executor = MCPToolExecutor(project_root, config)
+
+        # Combine default and custom ignore patterns
+        self.ignore_patterns = set(self.DEFAULT_IGNORE_PATTERNS)
+        if ignore_patterns:
+            self.ignore_patterns.update(ignore_patterns)
+
+        self._pending_files: Set[Path] = set()
+        self._debounce_task: Optional[asyncio.Task] = None
+        self._callbacks: List[Callable[[Dict[str, Any]], None]] = []
+        self._observer: Optional[Observer] = None  # type: ignore[valid-type]
+        self._running = False
+
+    def on_result(self, callback: Callable[[Dict[str, Any]], None]):
+        """Register callback for scan results.
+
+        Args:
+            callback: Function to call with scan results.
+        """
+        self._callbacks.append(callback)
+
+    async def start(self):
+        """Start watching for file changes."""
+        if self._running:
+            LOGGER.warning("File watcher already running")
+            return
+
+        self._running = True
+        handler = _FileChangeHandler(self._on_file_change)
+        self._observer = Observer()
+        self._observer.schedule(handler, str(self.project_root), recursive=True)
+        self._observer.start()
+
+        LOGGER.info(f"Watching {self.project_root} for changes...")
+
+        try:
+            while self._running:
+                await asyncio.sleep(0.1)
+        finally:
+            self.stop()
+
+    def stop(self):
+        """Stop the file watcher."""
+        self._running = False
+        if self._observer:
+            self._observer.stop()
+            self._observer.join()
+            self._observer = None
+        LOGGER.info("File watcher stopped")
+
+    def _on_file_change(self, path: Path):
+        """Handle file change event.
+
+        Args:
+            path: Path to the changed file.
+        """
+        # Skip ignored paths
+        if self._should_ignore(path):
+            return
+
+        # Skip non-files
+        if not path.is_file():
+            return
+
+        LOGGER.debug(f"File changed: {path}")
+
+        # Add to pending files
+        self._pending_files.add(path)
+
+        # Cancel existing debounce task
+        if self._debounce_task and not self._debounce_task.done():
+            self._debounce_task.cancel()
+
+        # Schedule new debounce task
+        loop = asyncio.get_event_loop()
+        self._debounce_task = loop.create_task(self._process_pending())
+
+    async def _process_pending(self):
+        """Process pending file changes after debounce."""
+        await asyncio.sleep(self.debounce_ms / 1000)
+
+        files = list(self._pending_files)
+        self._pending_files.clear()
+
+        if not files:
+            return
+
+        LOGGER.info(f"Processing {len(files)} changed file(s)...")
+
+        # Get relative paths
+        relative_files = []
+        for f in files:
+            try:
+                rel = f.relative_to(self.project_root)
+                relative_files.append(str(rel))
+            except ValueError:
+                # File not under project root
+                relative_files.append(str(f))
+
+        try:
+            # Run incremental check
+            result = await self.executor.scan(
+                domains=["all"],
+                files=relative_files,
+            )
+
+            # Add file list to result
+            result["changed_files"] = relative_files
+
+            # Notify callbacks
+            for callback in self._callbacks:
+                try:
+                    callback(result)
+                except Exception as e:
+                    LOGGER.error(f"Callback error: {e}")
+
+        except Exception as e:
+            LOGGER.error(f"Scan failed: {e}")
+            # Notify callbacks of error
+            error_result = {
+                "error": str(e),
+                "changed_files": relative_files,
+            }
+            for callback in self._callbacks:
+                try:
+                    callback(error_result)
+                except Exception as ce:
+                    LOGGER.error(f"Callback error: {ce}")
+
+    def _should_ignore(self, path: Path) -> bool:
+        """Check if path should be ignored.
+
+        Args:
+            path: Path to check.
+
+        Returns:
+            True if path should be ignored.
+        """
+        path_str = str(path)
+        path_parts = path.parts
+
+        for pattern in self.ignore_patterns:
+            # Check if pattern matches any path component
+            if pattern in path_parts:
+                return True
+            # Check glob-style patterns
+            if pattern.startswith("*") and path_str.endswith(pattern[1:]):
+                return True
+            # Check if pattern is in path string
+            if pattern in path_str:
+                return True
+
+        return False
+
+
+class _FileChangeHandler(FileSystemEventHandler):
+    """Watchdog event handler for file changes."""
+
+    def __init__(self, callback: Callable[[Path], None]):
+        """Initialize handler.
+
+        Args:
+            callback: Function to call on file change.
+        """
+        self.callback = callback
+
+    def on_modified(self, event):
+        """Handle file modification."""
+        if not event.is_directory:
+            self.callback(Path(event.src_path))
+
+    def on_created(self, event):
+        """Handle file creation."""
+        if not event.is_directory:
+            self.callback(Path(event.src_path))

lucidscan/pipeline/__init__.py

@@ -0,0 +1,17 @@
+"""Pipeline orchestration for lucidscan.
+
+Manages the execution of scan pipeline stages:
+1. Scanner execution (parallel by default)
+2. Enricher execution (sequential, in configured order)
+3. Result aggregation (metadata and summary)
+"""
+
+from lucidscan.pipeline.executor import PipelineConfig, PipelineExecutor
+from lucidscan.pipeline.parallel import ParallelScannerExecutor, ScannerResult
+
+__all__ = [
+    "PipelineConfig",
+    "PipelineExecutor",
+    "ParallelScannerExecutor",
+    "ScannerResult",
+]

lucidscan/pipeline/executor.py

@@ -0,0 +1,187 @@
+"""Pipeline executor for orchestrating scan stages."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from lucidscan.core.logging import get_logger
+from lucidscan.core.models import (
+    ScanContext,
+    ScanMetadata,
+    ScanResult,
+    UnifiedIssue,
+)
+from lucidscan.pipeline.parallel import ParallelScannerExecutor, ScannerResult
+
+if TYPE_CHECKING:
+    from lucidscan.config.models import LucidScanConfig
+
+LOGGER = get_logger(__name__)
+
+
+@dataclass
+class PipelineConfig:
+    """Configuration for pipeline execution."""
+
+    sequential_scanners: bool = False
+    max_workers: int = 4
+    enricher_order: List[str] = field(default_factory=list)
+
+
+class PipelineExecutor:
+    """Orchestrates the scan pipeline stages.
+
+    Pipeline stages:
+    1. Scanner execution (parallel by default)
+    2. Enricher execution (sequential, in configured order)
+    3. Result aggregation
+
+    Reporter execution is handled separately in CLI.
+    """
+
+    def __init__(
+        self,
+        config: "LucidScanConfig",
+        pipeline_config: Optional[PipelineConfig] = None,
+        lucidscan_version: str = "unknown",
+    ) -> None:
+        """Initialize the pipeline executor.
+
+        Args:
+            config: LucidScan configuration.
+            pipeline_config: Optional pipeline-specific configuration.
+            lucidscan_version: Version string for metadata.
+        """
+        self._config = config
+        self._pipeline_config = pipeline_config or PipelineConfig()
+        self._lucidscan_version = lucidscan_version
+
+    def execute(
+        self,
+        scanner_names: List[str],
+        context: ScanContext,
+    ) -> ScanResult:
+        """Execute the full pipeline and return results.
+
+        Args:
+            scanner_names: List of scanner plugin names to run.
+            context: Scan context for execution.
+
+        Returns:
+            Aggregated ScanResult with all issues and metadata.
+        """
+        start_time = datetime.now(timezone.utc)
+
+        # Stage 1: Scanner Execution (parallel)
+        all_issues, scanner_results = self._execute_scanners(scanner_names, context)
+
+        # Stage 2: Enricher Execution (sequential)
+        enriched_issues = self._execute_enrichers(all_issues, context)
+
+        # Stage 3: Result Aggregation
+        end_time = datetime.now(timezone.utc)
+        duration_ms = int((end_time - start_time).total_seconds() * 1000)
+
+        result = ScanResult(issues=enriched_issues)
+        result.metadata = ScanMetadata(
+            lucidscan_version=self._lucidscan_version,
+            scan_started_at=start_time.isoformat(),
+            scan_finished_at=end_time.isoformat(),
+            duration_ms=duration_ms,
+            project_root=str(context.project_root),
+            scanners_used=self._format_scanners_used(scanner_results),
+        )
+        result.summary = result.compute_summary()
+
+        return result
+
+    def _execute_scanners(
+        self,
+        scanner_names: List[str],
+        context: ScanContext,
+    ) -> tuple[List[UnifiedIssue], List[ScannerResult]]:
+        """Execute scanner stage."""
+        executor = ParallelScannerExecutor(
+            max_workers=self._pipeline_config.max_workers,
+            sequential=self._pipeline_config.sequential_scanners,
+        )
+        return executor.execute(scanner_names, context)
+
+    def _execute_enrichers(
+        self,
+        issues: List[UnifiedIssue],
+        context: ScanContext,
+    ) -> List[UnifiedIssue]:
+        """Execute enricher stage in configured order.
+
+        Enrichers run sequentially, each receiving the output
+        of the previous enricher.
+        """
+        # Import here to avoid circular imports
+        from lucidscan.plugins.enrichers import get_enricher_plugin
+
+        enriched = issues
+
+        # Get enricher order from pipeline config, then from main config
+        enricher_order = self._pipeline_config.enricher_order
+        if not enricher_order:
+            enricher_order = self._get_enricher_order_from_config()
+
+        for enricher_name in enricher_order:
+            enricher = get_enricher_plugin(enricher_name)
+            if not enricher:
+                LOGGER.warning(
+                    f"Enricher plugin '{enricher_name}' not found, skipping"
+                )
+                continue
+
+            LOGGER.info(f"Running {enricher_name} enricher...")
+
+            try:
+                enriched = enricher.enrich(enriched, context)
+                LOGGER.debug(f"{enricher_name}: processed {len(enriched)} issues")
+            except Exception as e:
+                LOGGER.error(f"Enricher {enricher_name} failed: {e}")
+                # Continue with unenriched issues on failure
+
+        return enriched
+
+    def _get_enricher_order_from_config(self) -> List[str]:
+        """Extract enricher order from config.
+
+        Looks for pipeline.enrichers or enabled enrichers in config.
+        """
+        # Check for explicit pipeline ordering in config
+        if hasattr(self._config, "pipeline") and self._config.pipeline:
+            pipeline = self._config.pipeline
+            if hasattr(pipeline, "enrichers") and pipeline.enrichers:
+                return pipeline.enrichers
+
+        # Fall back to enabled enrichers from enrichers config
+        enabled = []
+        for name, enricher_config in self._config.enrichers.items():
+            if isinstance(enricher_config, dict):
+                if enricher_config.get("enabled", True):
+                    enabled.append(name)
+            else:
+                enabled.append(name)
+
+        return enabled
+
+    def _format_scanners_used(
+        self,
+        scanner_results: List[ScannerResult],
+    ) -> List[Dict[str, Any]]:
+        """Format scanner results for metadata."""
+        return [
+            {
+                "name": r.scanner_name,
+                "version": r.scanner_version,
+                "domains": r.domains,
+                "success": r.success,
+                "error": r.error,
+            }
+            for r in scanner_results
+        ]

lucidscan/pipeline/parallel.py

@@ -0,0 +1,181 @@
+"""Parallel scanner execution using ThreadPoolExecutor."""
+
+from __future__ import annotations
+
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from typing import List, Optional, Tuple
+
+from lucidscan.core.logging import get_logger
+from lucidscan.core.models import ScanContext, UnifiedIssue
+from lucidscan.plugins.scanners import get_scanner_plugin
+
+LOGGER = get_logger(__name__)
+
+# Default number of worker threads
+DEFAULT_MAX_WORKERS = 4
+
+
+@dataclass
+class ScannerResult:
+    """Result from a single scanner execution."""
+
+    scanner_name: str
+    scanner_version: str
+    domains: List[str]
+    issues: List[UnifiedIssue] = field(default_factory=list)
+    error: Optional[str] = None
+    success: bool = True
+
+
+class ParallelScannerExecutor:
+    """Executes scanners in parallel using ThreadPoolExecutor.
+
+    Thread-safe aggregation of results using a lock.
+    """
+
+    def __init__(
+        self,
+        max_workers: int = DEFAULT_MAX_WORKERS,
+        sequential: bool = False,
+    ) -> None:
+        """Initialize the executor.
+
+        Args:
+            max_workers: Maximum number of concurrent scanner threads.
+            sequential: If True, run scanners sequentially (for debugging).
+        """
+        self._max_workers = max_workers
+        self._sequential = sequential
+        self._results_lock = threading.Lock()
+
+    def execute(
+        self,
+        scanner_names: List[str],
+        context: ScanContext,
+    ) -> Tuple[List[UnifiedIssue], List[ScannerResult]]:
+        """Execute scanners and return aggregated results.
+
+        Args:
+            scanner_names: List of scanner plugin names to execute.
+            context: Scan context for all scanners.
+
+        Returns:
+            Tuple of (all_issues, scanner_results) with thread-safe aggregation.
+        """
+        if not scanner_names:
+            return [], []
+
+        if self._sequential:
+            return self._execute_sequential(scanner_names, context)
+        return self._execute_parallel(scanner_names, context)
+
+    def _execute_parallel(
+        self,
+        scanner_names: List[str],
+        context: ScanContext,
+    ) -> Tuple[List[UnifiedIssue], List[ScannerResult]]:
+        """Execute scanners in parallel."""
+        all_issues: List[UnifiedIssue] = []
+        scanner_results: List[ScannerResult] = []
+
+        with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
+            # Submit all scanner tasks
+            future_to_scanner = {
+                executor.submit(self._run_scanner, name, context): name
+                for name in scanner_names
+            }
+
+            # Collect results as they complete
+            for future in as_completed(future_to_scanner):
+                scanner_name = future_to_scanner[future]
+                try:
+                    result = future.result()
+                    with self._results_lock:
+                        all_issues.extend(result.issues)
+                        scanner_results.append(result)
+                except Exception as e:
+                    LOGGER.error(f"Scanner {scanner_name} raised exception: {e}")
+                    with self._results_lock:
+                        scanner_results.append(
+                            ScannerResult(
+                                scanner_name=scanner_name,
+                                scanner_version="unknown",
+                                domains=[],
+                                error=str(e),
+                                success=False,
+                            )
+                        )
+
+        return all_issues, scanner_results
+
+    def _execute_sequential(
+        self,
+        scanner_names: List[str],
+        context: ScanContext,
+    ) -> Tuple[List[UnifiedIssue], List[ScannerResult]]:
+        """Execute scanners sequentially (for debugging)."""
+        all_issues: List[UnifiedIssue] = []
+        scanner_results: List[ScannerResult] = []
+
+        for name in scanner_names:
+            result = self._run_scanner(name, context)
+            all_issues.extend(result.issues)
+            scanner_results.append(result)
+
+        return all_issues, scanner_results
+
+    def _run_scanner(
+        self,
+        scanner_name: str,
+        context: ScanContext,
+    ) -> ScannerResult:
+        """Run a single scanner and return its result.
+
+        This method is thread-safe and catches all exceptions.
+        """
+        scanner = get_scanner_plugin(scanner_name, project_root=context.project_root)
+        if not scanner:
+            LOGGER.error(f"Scanner plugin '{scanner_name}' not found")
+            return ScannerResult(
+                scanner_name=scanner_name,
+                scanner_version="unknown",
+                domains=[],
+                error=f"Plugin '{scanner_name}' not found",
+                success=False,
+            )
+
+        LOGGER.info(f"Running {scanner_name} scanner...")
+
+        try:
+            issues = scanner.scan(context)
+            LOGGER.info(f"{scanner_name}: found {len(issues)} issues")
+
+            return ScannerResult(
+                scanner_name=scanner_name,
+                scanner_version=scanner.get_version(),
+                domains=[d.value for d in scanner.domains],
+                issues=issues,
+            )
+
+        except Exception as e:
+            LOGGER.error(f"Scanner {scanner_name} failed: {e}")
+            # Try to get version even on failure
+            try:
+                version = scanner.get_version()
+            except Exception:
+                version = "unknown"
+
+            try:
+                domains = [d.value for d in scanner.domains]
+            except Exception:
+                domains = []
+
+            return ScannerResult(
+                scanner_name=scanner_name,
+                scanner_version=version,
+                domains=domains,
+                error=str(e),
+                success=False,
+            )

lucidscan/plugins/__init__.py

@@ -0,0 +1,40 @@
+"""Plugin infrastructure for lucidscan.
+
+This package provides the plugin discovery and management infrastructure
+for all plugin types:
+- Scanner plugins (lucidscan.scanners) - Security scanners
+- Linter plugins (lucidscan.linters) - Code linting
+- Type checker plugins (lucidscan.type_checkers) - Type checking
+- Test runner plugins (lucidscan.test_runners) - Test execution
+- Coverage plugins (lucidscan.coverage) - Coverage analysis
+- Enricher plugins (lucidscan.enrichers) - Post-processing
+- Reporter plugins (lucidscan.reporters) - Output formatting
+
+Plugins are discovered via Python entry points.
+"""
+
+from lucidscan.plugins.discovery import (
+    discover_plugins,
+    get_plugin,
+    list_available_plugins,
+    SCANNER_ENTRY_POINT_GROUP,
+    ENRICHER_ENTRY_POINT_GROUP,
+    REPORTER_ENTRY_POINT_GROUP,
+    LINTER_ENTRY_POINT_GROUP,
+    TYPE_CHECKER_ENTRY_POINT_GROUP,
+    TEST_RUNNER_ENTRY_POINT_GROUP,
+    COVERAGE_ENTRY_POINT_GROUP,
+)
+
+__all__ = [
+    "discover_plugins",
+    "get_plugin",
+    "list_available_plugins",
+    "SCANNER_ENTRY_POINT_GROUP",
+    "ENRICHER_ENTRY_POINT_GROUP",
+    "REPORTER_ENTRY_POINT_GROUP",
+    "LINTER_ENTRY_POINT_GROUP",
+    "TYPE_CHECKER_ENTRY_POINT_GROUP",
+    "TEST_RUNNER_ENTRY_POINT_GROUP",
+    "COVERAGE_ENTRY_POINT_GROUP",
+]

lucidscan/plugins/coverage/__init__.py

@@ -0,0 +1,28 @@
+"""Coverage plugins for lucidscan.
+
+This module provides coverage analysis integrations for the quality pipeline.
+Coverage plugins are discovered via the lucidscan.coverage entry point group.
+"""
+
+from lucidscan.plugins.coverage.base import CoveragePlugin, CoverageResult, FileCoverage
+from lucidscan.plugins.discovery import (
+    discover_plugins,
+    COVERAGE_ENTRY_POINT_GROUP,
+)
+
+
+def discover_coverage_plugins():
+    """Discover all installed coverage plugins.
+
+    Returns:
+        Dictionary mapping plugin names to plugin classes.
+    """
+    return discover_plugins(COVERAGE_ENTRY_POINT_GROUP, CoveragePlugin)
+
+
+__all__ = [
+    "CoveragePlugin",
+    "CoverageResult",
+    "FileCoverage",
+    "discover_coverage_plugins",
+]