fbuild 1.2.8__py3-none-any.whl

fbuild/daemon/process_tracker.py

@@ -0,0 +1,366 @@
+"""
+Process Tracking and Cleanup Module
+
+This module manages tracking of build/deploy/monitor processes and their entire
+process trees. When client processes die, orphaned process trees are automatically
+cleaned up to prevent resource leaks and file locking issues.
+
+Key features:
+- Track root process + all children (recursive)
+- Detect dead client processes
+- Kill entire process trees recursively
+- Thread-safe operations for daemon use
+"""
+
+import _thread
+import json
+import logging
+import threading
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+import psutil
+
+
+@dataclass
+class ProcessTreeInfo:
+    """Information about a tracked process tree.
+
+    Attributes:
+        client_pid: PID of the client that initiated the operation
+        root_pid: PID of the root process
+        child_pids: List of all child PIDs (updated periodically)
+        request_id: Request ID
+        project_dir: Project directory
+        operation_type: Type of operation (deploy/monitor)
+        port: Serial port (if applicable)
+        started_at: Unix timestamp when tracking started
+        last_updated: Unix timestamp of last child PID refresh
+    """
+
+    client_pid: int
+    root_pid: int
+    child_pids: list[int] = field(default_factory=list)
+    request_id: str = ""
+    project_dir: str = ""
+    operation_type: str = ""
+    port: str | None = None
+    started_at: float = field(default_factory=time.time)
+    last_updated: float = field(default_factory=time.time)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ProcessTreeInfo":
+        """Create ProcessTreeInfo from dictionary."""
+        return cls(
+            client_pid=data["client_pid"],
+            root_pid=data["root_pid"],
+            child_pids=data.get("child_pids", []),
+            request_id=data.get("request_id", ""),
+            project_dir=data.get("project_dir", ""),
+            operation_type=data.get("operation_type", ""),
+            port=data.get("port"),
+            started_at=data.get("started_at", time.time()),
+            last_updated=data.get("last_updated", time.time()),
+        )
+
+
+class ProcessTracker:
+    """Thread-safe tracker for process trees.
+
+    This class maintains a registry of active processes and provides
+    methods to detect and cleanup orphaned process trees.
+    """
+
+    def __init__(self, registry_file: Path):
+        """Initialize the tracker.
+
+        Args:
+            registry_file: Path to JSON file for persisting process trees
+        """
+        self.registry_file = registry_file
+        self.lock = threading.Lock()
+        self._registry: dict[int, ProcessTreeInfo] = {}
+        self._load_registry()
+        logging.info(f"ProcessTracker initialized with {len(self._registry)} tracked processes")
+
+    def _load_registry(self) -> None:
+        """Load registry from disk (if it exists)."""
+        if not self.registry_file.exists():
+            return
+
+        try:
+            with open(self.registry_file) as f:
+                data = json.load(f)
+
+            with self.lock:
+                self._registry = {int(client_pid): ProcessTreeInfo.from_dict(info) for client_pid, info in data.items()}
+
+            logging.info(f"Loaded {len(self._registry)} process trees from registry")
+        except KeyboardInterrupt:
+            _thread.interrupt_main()
+            raise
+        except Exception as e:
+            logging.warning(f"Failed to load process registry: {e}")
+            self._registry = {}
+
+    def _save_registry(self) -> None:
+        """Save registry to disk atomically."""
+        try:
+            # Prepare data for serialization
+            data = {str(client_pid): info.to_dict() for client_pid, info in self._registry.items()}
+
+            # Atomic write
+            temp_file = self.registry_file.with_suffix(".tmp")
+            with open(temp_file, "w") as f:
+                json.dump(data, f, indent=2)
+
+            temp_file.replace(self.registry_file)
+
+        except KeyboardInterrupt:
+            _thread.interrupt_main()
+            raise
+        except Exception as e:
+            logging.error(f"Failed to save process registry: {e}")
+
+    def register_process(
+        self,
+        client_pid: int,
+        root_pid: int,
+        request_id: str = "",
+        project_dir: str = "",
+        operation_type: str = "",
+        port: str | None = None,
+    ) -> None:
+        """Register a new process tree.
+
+        Args:
+            client_pid: PID of client that initiated operation
+            root_pid: PID of root process
+            request_id: Request ID (optional)
+            project_dir: Project directory (optional)
+            operation_type: Type of operation (optional)
+            port: Serial port (optional)
+        """
+        with self.lock:
+            self._registry[client_pid] = ProcessTreeInfo(
+                client_pid=client_pid,
+                root_pid=root_pid,
+                request_id=request_id,
+                project_dir=project_dir,
+                operation_type=operation_type,
+                port=port,
+            )
+
+            # Immediately refresh child PIDs
+            self._update_child_pids(client_pid)
+
+        self._save_registry()
+        logging.info(f"Registered process tree: client={client_pid}, root={root_pid}, children={len(self._registry[client_pid].child_pids)}, operation={operation_type}")
+
+    def unregister_process(self, client_pid: int) -> None:
+        """Remove a process tree from tracking.
+
+        Args:
+            client_pid: Client PID to remove
+        """
+        with self.lock:
+            if client_pid in self._registry:
+                info = self._registry.pop(client_pid)
+                logging.info(f"Unregistered process tree: client={client_pid}, root={info.root_pid}")
+            else:
+                logging.warning(f"Attempted to unregister unknown client PID: {client_pid}")
+
+        self._save_registry()
+
+    def _update_child_pids(self, client_pid: int) -> None:
+        """Update child PID list for a tracked process.
+
+        This method MUST be called with self.lock held.
+
+        Args:
+            client_pid: Client PID to update
+        """
+        if client_pid not in self._registry:
+            return
+
+        info = self._registry[client_pid]
+
+        try:
+            # Get root process
+            root_proc = psutil.Process(info.root_pid)
+
+            # Get ALL descendants recursively
+            children = root_proc.children(recursive=True)
+            info.child_pids = [child.pid for child in children]
+            info.last_updated = time.time()
+
+        except psutil.NoSuchProcess:
+            # Root process died - mark as empty
+            info.child_pids = []
+            info.last_updated = time.time()
+        except KeyboardInterrupt:
+            _thread.interrupt_main()
+            raise
+        except Exception as e:
+            logging.warning(f"Failed to update child PIDs for client={client_pid}: {e}")
+
+    def refresh_all_child_pids(self) -> None:
+        """Refresh child PID lists for all tracked processes."""
+        with self.lock:
+            for client_pid in list(self._registry.keys()):
+                self._update_child_pids(client_pid)
+
+        self._save_registry()
+
+    def cleanup_orphaned_processes(self) -> list[int]:
+        """Detect and kill process trees for dead clients.
+
+        Returns:
+            List of client PIDs that were cleaned up
+        """
+        orphaned_clients = []
+
+        with self.lock:
+            for client_pid, info in list(self._registry.items()):
+                # Check if client is still alive
+                if psutil.pid_exists(client_pid):
+                    continue
+
+                # Client is dead - kill the entire process tree
+                logging.info(f"Client {client_pid} is dead, cleaning up process tree (root={info.root_pid}, children={len(info.child_pids)}, operation={info.operation_type})")
+
+                killed_count = self._kill_process_tree(info)
+                orphaned_clients.append(client_pid)
+
+                logging.info(f"Cleaned up {killed_count} processes for dead client {client_pid}")
+
+                # Remove from registry
+                del self._registry[client_pid]
+
+        if orphaned_clients:
+            logging.info(f"Orphaned clients cleaned up: {orphaned_clients}")
+            self._save_registry()
+
+        return orphaned_clients
+
+    def _kill_process_tree(self, info: ProcessTreeInfo) -> int:
+        """Kill an entire process tree (root + all children).
+
+        This method MUST be called with self.lock held.
+
+        Args:
+            info: ProcessTreeInfo containing root and child PIDs
+
+        Returns:
+            Number of processes killed
+        """
+        killed_count = 0
+        all_pids = info.child_pids + [info.root_pid]
+
+        # Refresh child list one last time before killing
+        try:
+            root_proc = psutil.Process(info.root_pid)
+            children = root_proc.children(recursive=True)
+            all_pids = [child.pid for child in children] + [info.root_pid]
+        except KeyboardInterrupt:
+            _thread.interrupt_main()
+            raise
+        except Exception:
+            pass  # Use cached PID list
+
+        # Kill children first (bottom-up to avoid orphans)
+        processes_to_kill: list[psutil.Process] = []
+        for pid in reversed(all_pids):  # Reverse to kill children before parents
+            try:
+                proc = psutil.Process(pid)
+                processes_to_kill.append(proc)
+            except psutil.NoSuchProcess:
+                pass  # Already dead
+            except KeyboardInterrupt:
+                _thread.interrupt_main()
+                raise
+            except Exception as e:
+                logging.warning(f"Failed to get process {pid}: {e}")
+
+        logging.info(f"Terminating {len(processes_to_kill)} processes")
+        # Terminate all processes
+        for proc in processes_to_kill:
+            try:
+                proc.terminate()
+                killed_count += 1
+            except psutil.NoSuchProcess:
+                pass  # Already dead
+            except KeyboardInterrupt:
+                _thread.interrupt_main()
+                raise
+            except Exception as e:
+                logging.warning(f"Failed to terminate process {proc.pid}: {e}")
+
+        # Wait for graceful termination
+        _gone, alive = psutil.wait_procs(processes_to_kill, timeout=3)
+
+        # Force kill any stragglers
+        if alive:
+            logging.warning(f"Force killing {len(alive)} stubborn processes")
+        for proc in alive:
+            try:
+                proc.kill()
+                logging.warning(f"Force killed stubborn process {proc.pid}")
+            except KeyboardInterrupt:
+                _thread.interrupt_main()
+                raise
+            except Exception as e:
+                logging.warning(f"Failed to force kill process {proc.pid}: {e}")
+
+        return killed_count
+
+    def get_tracked_clients(self) -> list[int]:
+        """Get list of all tracked client PIDs.
+
+        Returns:
+            List of client PIDs currently being tracked
+        """
+        with self.lock:
+            return list(self._registry.keys())
+
+    def get_process_info(self, client_pid: int) -> ProcessTreeInfo | None:
+        """Get process tree info for a client.
+
+        Args:
+            client_pid: Client PID to query
+
+        Returns:
+            ProcessTreeInfo if found, None otherwise
+        """
+        with self.lock:
+            return self._registry.get(client_pid)
+
+    def get_processes_by_port(self, port: str) -> list[ProcessTreeInfo]:
+        """Get all processes using a specific serial port.
+
+        Args:
+            port: Serial port to search for
+
+        Returns:
+            List of ProcessTreeInfo for processes using this port
+        """
+        with self.lock:
+            return [info for info in self._registry.values() if info.port == port]
+
+    def get_processes_by_project(self, project_dir: str) -> list[ProcessTreeInfo]:
+        """Get all processes for a specific project.
+
+        Args:
+            project_dir: Project directory to search for
+
+        Returns:
+            List of ProcessTreeInfo for processes in this project
+        """
+        with self.lock:
+            return [info for info in self._registry.values() if info.project_dir == project_dir]

fbuild/daemon/processors/__init__.py

@@ -0,0 +1,18 @@
+"""
+Daemon Request Processors - Concrete implementations of request handling.
+
+This package contains concrete processor implementations for different
+operation types (build, deploy, monitor, install_dependencies).
+"""
+
+from fbuild.daemon.processors.build_processor import BuildRequestProcessor
+from fbuild.daemon.processors.deploy_processor import DeployRequestProcessor
+from fbuild.daemon.processors.install_deps_processor import InstallDependenciesProcessor
+from fbuild.daemon.processors.monitor_processor import MonitorRequestProcessor
+
+__all__ = [
+    "BuildRequestProcessor",
+    "DeployRequestProcessor",
+    "InstallDependenciesProcessor",
+    "MonitorRequestProcessor",
+]

fbuild/daemon/processors/build_processor.py

@@ -0,0 +1,248 @@
+"""
+Build Request Processor - Handles build operations.
+
+This module implements the BuildRequestProcessor which executes build
+operations for Arduino/ESP32 projects using the appropriate orchestrator.
+"""
+
+import importlib
+import logging
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from fbuild.daemon.messages import OperationType
+from fbuild.daemon.request_processor import RequestProcessor
+
+if TYPE_CHECKING:
+    from fbuild.daemon.daemon_context import DaemonContext
+    from fbuild.daemon.messages import BuildRequest
+
+
+class BuildRequestProcessor(RequestProcessor):
+    """Processor for build requests.
+
+    This processor handles compilation of Arduino/ESP32 projects. It:
+    1. Reloads build modules to pick up code changes (for development)
+    2. Creates the appropriate orchestrator (AVR or ESP32)
+    3. Executes the build with the specified settings
+    4. Returns success/failure based on build result
+
+    Example:
+        >>> processor = BuildRequestProcessor()
+        >>> success = processor.process_request(build_request, daemon_context)
+    """
+
+    def get_operation_type(self) -> OperationType:
+        """Return BUILD operation type."""
+        return OperationType.BUILD
+
+    def get_required_locks(self, request: "BuildRequest", context: "DaemonContext") -> dict[str, str]:
+        """Build operations require only a project lock.
+
+        Args:
+            request: The build request
+            context: The daemon context
+
+        Returns:
+            Dictionary with project lock requirement
+        """
+        return {"project": request.project_dir}
+
+    def execute_operation(self, request: "BuildRequest", context: "DaemonContext") -> bool:
+        """Execute the build operation.
+
+        This is the core build logic extracted from the original
+        process_build_request function. All boilerplate (locks, status
+        updates, error handling) is handled by the base RequestProcessor.
+
+        Args:
+            request: The build request containing project_dir, environment, etc.
+            context: The daemon context with all subsystems
+
+        Returns:
+            True if build succeeded, False otherwise
+        """
+        logging.info(f"Building project: {request.project_dir}")
+
+        # Reload build modules to pick up code changes
+        # This is critical for development on Windows where daemon caching
+        # prevents testing code changes
+        self._reload_build_modules()
+
+        # Detect platform type from platformio.ini to select appropriate orchestrator
+        try:
+            from fbuild.config.ini_parser import PlatformIOConfig
+
+            project_path = Path(request.project_dir)
+            ini_path = project_path / "platformio.ini"
+
+            if not ini_path.exists():
+                logging.error(f"platformio.ini not found at {ini_path}")
+                return False
+
+            config = PlatformIOConfig(ini_path)
+            env_config = config.get_env_config(request.environment)
+            platform = env_config.get("platform", "").lower()
+
+            logging.info(f"Detected platform: {platform}")
+
+        except KeyboardInterrupt as ke:
+            from fbuild.interrupt_utils import handle_keyboard_interrupt_properly
+
+            handle_keyboard_interrupt_properly(ke)
+            raise  # Never reached, but satisfies type checker
+        except Exception as e:
+            logging.error(f"Failed to parse platformio.ini: {e}")
+            return False
+
+        # Normalize platform name (handle various platform specification formats)
+        # URL formats: "https://.../platform-espressif32.zip" -> "espressif32"
+        # PlatformIO format: "platformio/espressif32" -> "espressif32"
+        # Direct names: "atmelavr", "espressif32", "ststm32", etc.
+        platform_name = platform
+        if "platform-espressif32" in platform or "platformio/espressif32" in platform or platform == "espressif32":
+            platform_name = "espressif32"
+        elif "platform-atmelavr" in platform or "platformio/atmelavr" in platform or platform == "atmelavr":
+            platform_name = "atmelavr"
+        elif "platform-raspberrypi" in platform or "platformio/raspberrypi" in platform or platform == "raspberrypi":
+            platform_name = "raspberrypi"
+        elif "platform-ststm32" in platform or "platformio/ststm32" in platform or platform == "ststm32":
+            platform_name = "ststm32"
+
+        logging.info(f"Normalized platform: {platform_name}")
+
+        # Select orchestrator based on platform
+        if platform_name == "atmelavr":
+            module_name = "fbuild.build.orchestrator_avr"
+            class_name = "BuildOrchestratorAVR"
+        elif platform_name == "espressif32":
+            module_name = "fbuild.build.orchestrator_esp32"
+            class_name = "OrchestratorESP32"
+        elif platform_name == "raspberrypi":
+            module_name = "fbuild.build.orchestrator_rp2040"
+            class_name = "OrchestratorRP2040"
+        elif platform_name == "ststm32":
+            module_name = "fbuild.build.orchestrator_stm32"
+            class_name = "OrchestratorSTM32"
+        else:
+            logging.error(f"Unsupported platform: {platform_name}")
+            return False
+
+        # Get fresh orchestrator class after module reload
+        # Using direct import would use cached version
+        try:
+            orchestrator_class = getattr(sys.modules[module_name], class_name)
+        except (KeyError, AttributeError) as e:
+            logging.error(f"Failed to get {class_name} from {module_name}: {e}")
+            return False
+
+        # Create orchestrator and execute build
+        # Create a Cache instance for package management
+        from fbuild.packages.cache import Cache
+
+        cache = Cache(project_dir=Path(request.project_dir))
+
+        # Initialize orchestrator with cache (ESP32 requires it, AVR accepts it)
+        logging.debug(f"[BUILD_PROCESSOR] Initializing {class_name} with cache={cache}, verbose={request.verbose}")
+        logging.debug(f"[BUILD_PROCESSOR] orchestrator_class={orchestrator_class}, module={module_name}")
+        orchestrator = orchestrator_class(cache=cache, verbose=request.verbose)
+        logging.debug(f"[BUILD_PROCESSOR] orchestrator created successfully: {orchestrator}")
+        build_result = orchestrator.build(
+            project_dir=Path(request.project_dir),
+            env_name=request.environment,
+            clean=request.clean_build,
+            verbose=request.verbose,
+        )
+
+        if not build_result.success:
+            logging.error(f"Build failed: {build_result.message}")
+            return False
+
+        logging.info("Build completed successfully")
+        return True
+
+    def _reload_build_modules(self) -> None:
+        """Reload build-related modules to pick up code changes.
+
+        This is critical for development on Windows where daemon caching prevents
+        testing code changes. Reloads key modules that are frequently modified.
+
+        Order matters: reload dependencies first, then modules that import them.
+        """
+        modules_to_reload = [
+            # Core utilities and packages (reload first - no dependencies)
+            "fbuild.packages.cache",
+            "fbuild.packages.downloader",
+            "fbuild.packages.archive_utils",
+            "fbuild.packages.platformio_registry",
+            "fbuild.packages.toolchain",
+            "fbuild.packages.toolchain_esp32",
+            "fbuild.packages.toolchain_teensy",
+            "fbuild.packages.toolchain_rp2040",
+            "fbuild.packages.toolchain_stm32",
+            "fbuild.packages.arduino_core",
+            "fbuild.packages.framework_esp32",
+            "fbuild.packages.framework_teensy",
+            "fbuild.packages.framework_rp2040",
+            "fbuild.packages.framework_stm32",
+            "fbuild.packages.platform_esp32",
+            "fbuild.packages.platform_teensy",
+            "fbuild.packages.platform_rp2040",
+            "fbuild.packages.platform_stm32",
+            "fbuild.packages.library_manager",
+            "fbuild.packages.library_manager_esp32",
+            # Config system (reload early - needed to detect platform type)
+            "fbuild.config.ini_parser",
+            "fbuild.config.board_config",
+            "fbuild.config.board_loader",
+            # Build system (reload second - depends on packages)
+            "fbuild.build.archive_creator",
+            "fbuild.build.flag_builder",
+            "fbuild.build.compiler",
+            "fbuild.build.configurable_compiler",
+            "fbuild.build.linker",
+            "fbuild.build.configurable_linker",
+            "fbuild.build.source_scanner",
+            "fbuild.build.compilation_executor",
+            "fbuild.build.build_state",
+            "fbuild.build.build_info_generator",
+            "fbuild.build.build_utils",
+            # Orchestrators (reload third - depends on build system)
+            "fbuild.build.orchestrator",
+            "fbuild.build.orchestrator_avr",
+            "fbuild.build.orchestrator_esp32",
+            "fbuild.build.orchestrator_teensy",
+            "fbuild.build.orchestrator_rp2040",
+            "fbuild.build.orchestrator_stm32",
+            # Daemon processors (reload to pick up processor code changes)
+            "fbuild.daemon.processors.build_processor",
+            # Deploy and monitor (reload with build system)
+            "fbuild.deploy.deployer",
+            "fbuild.deploy.deployer_esp32",
+            "fbuild.deploy.monitor",
+            # Top-level module packages (reload last to update __init__.py imports)
+            "fbuild.build",
+            "fbuild.deploy",
+        ]
+
+        reloaded_count = 0
+        for module_name in modules_to_reload:
+            try:
+                if module_name in sys.modules:
+                    # Module already loaded - reload it to pick up changes
+                    importlib.reload(sys.modules[module_name])
+                    reloaded_count += 1
+                else:
+                    # Module not loaded yet - import it for the first time
+                    __import__(module_name)
+                    reloaded_count += 1
+            except KeyboardInterrupt as ke:
+                from fbuild.interrupt_utils import handle_keyboard_interrupt_properly
+
+                handle_keyboard_interrupt_properly(ke)
+            except Exception as e:
+                logging.warning(f"Failed to reload/import module {module_name}: {e}")
+
+        if reloaded_count > 0:
+            logging.info(f"Loaded/reloaded {reloaded_count} build modules")