fbuild 1.2.8__py3-none-any.whl → 1.2.15__py3-none-any.whl

fbuild/deploy/docker_utils.py

@@ -3,6 +3,12 @@ Docker utilities for QEMU deployment.
 
 This module provides utilities for managing Docker containers for ESP32 QEMU emulation,
 including automatic Docker daemon startup detection and image management.
+
+Enhanced Docker auto-start functionality based on patterns from fastled-wasm:
+- WSL2 backend detection on Windows
+- Docker Desktop restart workflow
+- Retry logic for Docker client connection
+- Cross-platform support (Windows, macOS, Linux)
 """
 
 import os
@@ -10,6 +16,7 @@ import platform
 import subprocess
 import sys
 import time
+from pathlib import Path
 from typing import Optional
 
 from fbuild.interrupt_utils import handle_keyboard_interrupt_properly
@@ -73,11 +80,14 @@ def get_docker_desktop_path() -> Optional[str]:
 
     if system == "Windows":
         # Common locations for Docker Desktop on Windows
+        home_dir = Path.home()
         paths = [
             r"C:\Program Files\Docker\Docker\Docker Desktop.exe",
             r"C:\Program Files (x86)\Docker\Docker\Docker Desktop.exe",
             os.path.expandvars(r"%ProgramFiles%\Docker\Docker\Docker Desktop.exe"),
             os.path.expandvars(r"%LocalAppData%\Programs\Docker\Docker\Docker Desktop.exe"),
+            str(home_dir / "AppData" / "Local" / "Docker" / "Docker Desktop.exe"),
+            str(home_dir / "AppData" / "Local" / "Programs" / "Docker" / "Docker" / "Docker Desktop.exe"),
         ]
         for path in paths:
             if os.path.exists(path):
@@ -99,11 +109,149 @@ def get_docker_desktop_path() -> Optional[str]:
     return None
 
 
+def _check_wsl2_docker_backend() -> tuple[bool, str]:
+    """Check if Docker's WSL2 backend (docker-desktop) is running on Windows.
+
+    Returns:
+        tuple[bool, str]: (is_running, status_message)
+    """
+    if platform.system() != "Windows":
+        return True, "Not Windows - WSL2 check not applicable"
+
+    try:
+        result = subprocess.run(
+            ["wsl", "--list", "--verbose"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            env=get_docker_env(),
+        )
+
+        if result.returncode != 0:
+            return False, "WSL2 not installed (wsl command failed)"
+
+        output = result.stdout
+
+        # Handle WSL output encoding issues (Git Bash adds spaces between characters)
+        cleaned_lines = []
+        for line in output.split("\n"):
+            cleaned = line.replace("\x00", "").strip()
+            # Remove spaces between single characters: "d o c k e r" -> "docker"
+            if " " in cleaned and len([c for c in cleaned.split() if len(c) == 1]) > 3:
+                cleaned = cleaned.replace(" ", "")
+            cleaned_lines.append(cleaned)
+
+        # Detect status: running/stopped
+        for line in cleaned_lines:
+            if "docker-desktop" in line.lower():
+                if "running" in line.lower():
+                    return True, "docker-desktop WSL2 backend is running"
+                elif "stopped" in line.lower():
+                    return False, "docker-desktop WSL2 backend is stopped"
+
+        return False, "docker-desktop WSL2 distribution not found"
+
+    except FileNotFoundError:
+        return False, "WSL2 not installed (wsl command not found)"
+    except subprocess.TimeoutExpired:
+        return False, "WSL2 command timed out"
+    except KeyboardInterrupt as ke:
+        handle_keyboard_interrupt_properly(ke)
+        return False, "Interrupted by user"  # Will not reach here
+    except Exception as e:
+        return False, f"Error checking WSL2 backend: {e}"
+
+
+def _kill_docker_desktop_windows() -> bool:
+    """Forcefully terminate Docker Desktop and backend processes on Windows.
+
+    Returns:
+        bool: True if processes were killed successfully
+    """
+    if platform.system() != "Windows":
+        return False
+
+    try:
+        # Kill Docker Desktop GUI
+        subprocess.run(
+            ["taskkill", "/F", "/IM", "Docker Desktop.exe"],
+            capture_output=True,
+            timeout=10,
+        )
+
+        # Kill Docker backend engine
+        subprocess.run(
+            ["taskkill", "/F", "/IM", "com.docker.backend.exe"],
+            capture_output=True,
+            timeout=10,
+        )
+
+        time.sleep(3)  # Allow cleanup
+        return True
+    except KeyboardInterrupt as ke:
+        handle_keyboard_interrupt_properly(ke)
+        return False  # Will not reach here
+    except Exception:
+        return False
+
+
+def _restart_docker_desktop_windows() -> tuple[bool, str]:
+    """Complete Docker Desktop restart workflow on Windows.
+
+    Returns:
+        tuple[bool, str]: (success, message)
+    """
+    print("  Attempting to restart Docker Desktop to fix WSL2 backend...")
+
+    docker_path = get_docker_desktop_path()
+    if not docker_path:
+        return False, "Docker Desktop executable not found"
+
+    print("  Stopping Docker Desktop...")
+    _kill_docker_desktop_windows()
+
+    time.sleep(5)  # Wait for cleanup
+
+    print("  Starting Docker Desktop...")
+    try:
+        subprocess.Popen(
+            [docker_path],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NO_WINDOW,
+        )
+    except KeyboardInterrupt as ke:
+        handle_keyboard_interrupt_properly(ke)
+        return False, "Interrupted by user"  # Will not reach here
+    except Exception as e:
+        return False, f"Failed to start Docker Desktop: {e}"
+
+    print("  Waiting for Docker Desktop and WSL2 backend to initialize...")
+
+    # Poll for up to 2 minutes
+    for attempt in range(120):
+        time.sleep(1)
+
+        # Check if Docker engine is available
+        if check_docker_daemon_running():
+            # Also check WSL2 backend
+            wsl_running, _ = _check_wsl2_docker_backend()
+            if wsl_running:
+                return True, "Docker Desktop restarted successfully - WSL2 backend is running"
+
+        # Progress indicator every 15 seconds
+        if (attempt + 1) % 15 == 0:
+            print(f"  Still waiting ({attempt + 1}s)...")
+
+    return False, "Docker Desktop started but WSL2 backend failed to initialize within 2 minutes"
+
+
 def start_docker_daemon() -> bool:
     """Attempt to start the Docker daemon.
 
     This function tries to start Docker Desktop on Windows/macOS
-    or the Docker service on Linux.
+    or the Docker service on Linux. On Windows, it includes WSL2
+    backend detection and restart workflow.
 
     Returns:
         True if Docker daemon started successfully, False otherwise
@@ -116,6 +264,22 @@ def start_docker_daemon() -> bool:
     print("Docker daemon is not running. Attempting to start...")
 
     if system == "Windows":
+        # Check WSL2 backend status on Windows
+        print("  Checking Docker Desktop WSL2 backend status...")
+        wsl_running, wsl_message = _check_wsl2_docker_backend()
+        print(f"  WSL2 Status: {wsl_message}")
+
+        # Detect split-brain state and trigger restart
+        if not wsl_running and "stopped" in wsl_message.lower():
+            print("  Issue detected: Docker Desktop app may be running but WSL2 backend is stopped")
+            success, message = _restart_docker_desktop_windows()
+            if success:
+                print(f"  {message}")
+                return True
+            else:
+                print(f"  {message}")
+                return False
+
         docker_path = get_docker_desktop_path()
         if docker_path:
             try:
@@ -126,9 +290,22 @@ def start_docker_daemon() -> bool:
                     stderr=subprocess.DEVNULL,
                     creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NO_WINDOW,
                 )
-                return _wait_for_docker_daemon()
+                success = _wait_for_docker_daemon(timeout=120)  # 2 minute timeout
+
+                if not success:
+                    # If startup failed, try full restart
+                    print("  Docker Desktop didn't start properly, attempting full restart...")
+                    success, message = _restart_docker_desktop_windows()
+                    if success:
+                        print(f"  {message}")
+                        return True
+                    else:
+                        print(f"  {message}")
+                        return False
+                return success
             except KeyboardInterrupt as ke:
                 handle_keyboard_interrupt_properly(ke)
+                return False
             except Exception as e:
                 print(f"Failed to start Docker Desktop: {e}")
                 return False
@@ -148,6 +325,7 @@ def start_docker_daemon() -> bool:
                 return _wait_for_docker_daemon()
             except KeyboardInterrupt as ke:
                 handle_keyboard_interrupt_properly(ke)
+                return False
             except Exception as e:
                 print(f"Failed to start Docker Desktop: {e}")
                 return False
@@ -176,6 +354,7 @@ def start_docker_daemon() -> bool:
                     return _wait_for_docker_daemon()
         except KeyboardInterrupt as ke:
             handle_keyboard_interrupt_properly(ke)
+            return False
         except Exception as e:
             print(f"Failed to start Docker service: {e}")
 
@@ -278,6 +457,7 @@ def pull_docker_image(image_name: str, timeout: int = 600) -> bool:
             return False
     except KeyboardInterrupt as ke:
         handle_keyboard_interrupt_properly(ke)
+        return False  # Will not reach here, but satisfies type checker
     except subprocess.TimeoutExpired:
         print(f"Timeout pulling {image_name}")
         return False

fbuild/deploy/monitor.py

@@ -450,7 +450,7 @@ class SerialMonitor:
 
             return 1
         except KeyboardInterrupt:
-            # Interrupt other threads
+            # Interrupt other threads (notify them of the interrupt)
             _thread.interrupt_main()
 
             elapsed_time = time.time() - start_time if "start_time" in locals() else 0.0

fbuild/deploy/qemu_runner.py

@@ -189,12 +189,26 @@ class QEMURunner:
             print(f"Error pulling Docker image: {e}")
             return False
 
-    def _prepare_firmware(self, firmware_path: Path, flash_size_mb: int = 4) -> Path:
+    def _prepare_firmware(self, firmware_path: Path, flash_size_mb: int = 4, machine: str = "esp32") -> Path:
         """Prepare firmware files for mounting into Docker container.
 
+        Creates a complete flash image with bootloader, partition table,
+        and application at their correct offsets for QEMU.
+
+        ESP32/ESP32-S2 Flash Layout:
+        - 0x1000: Bootloader (second stage)
+        - 0x8000: Partition table
+        - 0x10000: Application (firmware.bin)
+
+        ESP32-S3/ESP32-C3/ESP32-C6 Flash Layout:
+        - 0x0000: Bootloader (second stage)
+        - 0x8000: Partition table
+        - 0x10000: Application (firmware.bin)
+
         Args:
             firmware_path: Path to firmware.bin file
             flash_size_mb: Flash size in MB (must be 2, 4, 8, or 16)
+            machine: QEMU machine type (esp32, esp32s3, esp32c3)
 
         Returns:
             Path to the prepared firmware directory
@@ -212,19 +226,63 @@ class QEMURunner:
             # Create proper flash image for QEMU
             flash_size = flash_size_mb * 1024 * 1024
 
-            # Read firmware content
-            firmware_data = firmware_path.read_bytes()
-
-            # Create flash image: firmware at beginning, rest filled with 0xFF
-            flash_data = firmware_data + b"\xff" * (flash_size - len(firmware_data))
-
-            # Ensure we have exactly the right size
-            if len(flash_data) > flash_size:
-                raise ValueError(f"Firmware size ({len(firmware_data)} bytes) exceeds flash size ({flash_size} bytes)")
+            # ESP32 flash layout offsets - different MCUs have different bootloader offsets
+            # ESP32/ESP32-S2: 0x1000, ESP32-S3/C3/C6: 0x0
+            if machine in ["esp32", "esp32s2"]:
+                BOOTLOADER_OFFSET = 0x1000
+            else:
+                BOOTLOADER_OFFSET = 0x0
+            PARTITION_OFFSET = 0x8000
+            APP_OFFSET = 0x10000
+
+            # Start with erased flash (0xFF)
+            flash_data = bytearray(b"\xff" * flash_size)
+
+            # Try to find bootloader.bin and partitions.bin in the same directory
+            build_dir = firmware_path.parent
+            bootloader_path = build_dir / "bootloader.bin"
+            partitions_path = build_dir / "partitions.bin"
+
+            # Place bootloader at 0x1000 if available
+            if bootloader_path.exists():
+                bootloader_data = bootloader_path.read_bytes()
+                bootloader_end = BOOTLOADER_OFFSET + len(bootloader_data)
+                if bootloader_end <= flash_size:
+                    flash_data[BOOTLOADER_OFFSET:bootloader_end] = bootloader_data
+                    if self.verbose:
+                        print(f"  Bootloader: {len(bootloader_data)} bytes at 0x{BOOTLOADER_OFFSET:X}")
+                else:
+                    print("Warning: Bootloader too large to fit in flash")
+            else:
+                if self.verbose:
+                    print(f"  Warning: bootloader.bin not found at {bootloader_path}")
+
+            # Place partition table at 0x8000 if available
+            if partitions_path.exists():
+                partitions_data = partitions_path.read_bytes()
+                partitions_end = PARTITION_OFFSET + len(partitions_data)
+                if partitions_end <= flash_size:
+                    flash_data[PARTITION_OFFSET:partitions_end] = partitions_data
+                    if self.verbose:
+                        print(f"  Partitions: {len(partitions_data)} bytes at 0x{PARTITION_OFFSET:X}")
+                else:
+                    print("Warning: Partition table too large to fit in flash")
+            else:
+                if self.verbose:
+                    print(f"  Warning: partitions.bin not found at {partitions_path}")
 
-            flash_data = flash_data[:flash_size]  # Truncate to exact size
+            # Place firmware at 0x10000
+            firmware_data = firmware_path.read_bytes()
+            firmware_end = APP_OFFSET + len(firmware_data)
+            if firmware_end > flash_size:
+                raise ValueError(f"Firmware size ({len(firmware_data)} bytes) exceeds available space (flash_size={flash_size} - app_offset={APP_OFFSET})")
+            flash_data[APP_OFFSET:firmware_end] = firmware_data
+            if self.verbose:
+                print(f"  Application: {len(firmware_data)} bytes at 0x{APP_OFFSET:X}")
 
-            (temp_dir / "flash.bin").write_bytes(flash_data)
+            # Write the complete flash image
+            (temp_dir / "flash.bin").write_bytes(bytes(flash_data))
+            print(f"Created flash image: {flash_size_mb}MB ({flash_size:,} bytes)")
 
             return temp_dir
 
@@ -352,7 +410,7 @@ class QEMURunner:
         temp_firmware_dir: Optional[Path] = None
 
         try:
-            temp_firmware_dir = self._prepare_firmware(firmware_path, flash_size)
+            temp_firmware_dir = self._prepare_firmware(firmware_path, flash_size, machine)
 
             # Generate unique container name
             self.container_name = f"fbuild-qemu-{machine}-{int(time.time())}"

fbuild/ledger/board_ledger.py

@@ -2,15 +2,16 @@
 Board Ledger - Track attached chip/port mappings.
 
 This module provides a simple ledger to cache chip type detections for serial ports.
-The cache is stored in ~/.fbuild/board_ledger.json and uses file locking for
-thread-safe access.
+The cache is stored in ~/.fbuild/board_ledger.json.
 
 Features:
 - Port to chip type mapping with timestamps
 - Automatic stale entry expiration (24 hours)
-- Thread-safe file access with file locking
+- Thread-safe in-process access via threading.Lock
 - Chip type validation against known ESP32 variants
 - Integration with esptool for chip detection
+
+Note: Cross-process synchronization is handled by the daemon which holds locks in memory.
 """
 
 import json
@@ -96,7 +97,8 @@ class BoardLedger:
     """Manages port to chip type mappings with persistent storage.
 
     The ledger stores mappings in ~/.fbuild/board_ledger.json and provides
-    thread-safe access through file locking.
+    thread-safe in-process access through threading.Lock. Cross-process
+    synchronization is handled by the daemon which holds locks in memory.
 
     Example:
         >>> ledger = BoardLedger()
@@ -161,60 +163,6 @@ class BoardLedger:
         except OSError as e:
             raise BoardLedgerError(f"Failed to write ledger: {e}") from e
 
-    def _acquire_file_lock(self) -> Any:
-        """Acquire a file lock for cross-process synchronization.
-
-        Returns:
-            Lock file handle (or None on platforms without locking support)
-        """
-        self._ensure_directory()
-        lock_path = self._ledger_path.with_suffix(".lock")
-
-        try:
-            # Open lock file
-            lock_file = open(lock_path, "w", encoding="utf-8")
-
-            # Platform-specific locking
-            if sys.platform == "win32":
-                import msvcrt
-
-                msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1)
-            else:  # pragma: no cover - Unix only
-                import fcntl  # type: ignore[import-not-found]
-
-                fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
-
-            return lock_file
-        except (ImportError, OSError):
-            # Locking not available or failed - continue without lock
-            return None
-
-    def _release_file_lock(self, lock_file: Any) -> None:
-        """Release a file lock.
-
-        Args:
-            lock_file: Lock file handle from _acquire_file_lock
-        """
-        if lock_file is None:
-            return
-
-        try:
-            if sys.platform == "win32":
-                import msvcrt
-
-                try:
-                    msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
-                except OSError:
-                    pass
-            else:  # pragma: no cover - Unix only
-                import fcntl  # type: ignore[import-not-found]
-
-                fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
-
-            lock_file.close()
-        except (ImportError, OSError):
-            pass
-
     def get_chip(self, port: str) -> str | None:
         """Get the cached chip type for a port.
 
@@ -225,20 +173,16 @@ class BoardLedger:
             Chip type string (e.g., "ESP32-S3") or None if not found/stale
         """
         with self._lock:
-            lock_file = self._acquire_file_lock()
-            try:
-                data = self._read_ledger()
-                entry_data = data.get(port)
-                if entry_data is None:
-                    return None
+            data = self._read_ledger()
+            entry_data = data.get(port)
+            if entry_data is None:
+                return None
 
-                entry = LedgerEntry.from_dict(entry_data)
-                if entry.is_stale():
-                    return None
+            entry = LedgerEntry.from_dict(entry_data)
+            if entry.is_stale():
+                return None
 
-                return entry.chip_type
-            finally:
-                self._release_file_lock(lock_file)
+            return entry.chip_type
 
     def set_chip(self, port: str, chip_type: str) -> None:
         """Set the chip type for a port.
@@ -258,13 +202,9 @@ class BoardLedger:
         entry = LedgerEntry(chip_type=normalized, timestamp=time.time())
 
         with self._lock:
-            lock_file = self._acquire_file_lock()
-            try:
-                data = self._read_ledger()
-                data[port] = entry.to_dict()
-                self._write_ledger(data)
-            finally:
-                self._release_file_lock(lock_file)
+            data = self._read_ledger()
+            data[port] = entry.to_dict()
+            self._write_ledger(data)
 
     def clear(self, port: str) -> bool:
         """Clear the cached chip type for a port.
@@ -276,16 +216,12 @@ class BoardLedger:
             True if entry was cleared, False if not found
         """
         with self._lock:
-            lock_file = self._acquire_file_lock()
-            try:
-                data = self._read_ledger()
-                if port in data:
-                    del data[port]
-                    self._write_ledger(data)
-                    return True
-                return False
-            finally:
-                self._release_file_lock(lock_file)
+            data = self._read_ledger()
+            if port in data:
+                del data[port]
+                self._write_ledger(data)
+                return True
+            return False
 
     def clear_all(self) -> int:
         """Clear all entries from the ledger.
@@ -294,14 +230,10 @@ class BoardLedger:
             Number of entries cleared
         """
         with self._lock:
-            lock_file = self._acquire_file_lock()
-            try:
-                data = self._read_ledger()
-                count = len(data)
-                self._write_ledger({})
-                return count
-            finally:
-                self._release_file_lock(lock_file)
+            data = self._read_ledger()
+            count = len(data)
+            self._write_ledger({})
+            return count
 
     def clear_stale(self, threshold: float = STALE_THRESHOLD_SECONDS) -> int:
         """Remove all stale entries from the ledger.
@@ -313,22 +245,18 @@ class BoardLedger:
             Number of entries removed
         """
         with self._lock:
-            lock_file = self._acquire_file_lock()
-            try:
-                data = self._read_ledger()
-                original_count = len(data)
-
-                # Filter out stale entries
-                fresh_data = {}
-                for port, entry_data in data.items():
-                    entry = LedgerEntry.from_dict(entry_data)
-                    if not entry.is_stale(threshold):
-                        fresh_data[port] = entry_data
-
-                self._write_ledger(fresh_data)
-                return original_count - len(fresh_data)
-            finally:
-                self._release_file_lock(lock_file)
+            data = self._read_ledger()
+            original_count = len(data)
+
+            # Filter out stale entries
+            fresh_data = {}
+            for port, entry_data in data.items():
+                entry = LedgerEntry.from_dict(entry_data)
+                if not entry.is_stale(threshold):
+                    fresh_data[port] = entry_data
+
+            self._write_ledger(fresh_data)
+            return original_count - len(fresh_data)
 
     def get_all(self) -> dict[str, LedgerEntry]:
         """Get all non-stale entries in the ledger.
@@ -337,17 +265,13 @@ class BoardLedger:
             Dictionary mapping port names to LedgerEntry objects
         """
         with self._lock:
-            lock_file = self._acquire_file_lock()
-            try:
-                data = self._read_ledger()
-                result = {}
-                for port, entry_data in data.items():
-                    entry = LedgerEntry.from_dict(entry_data)
-                    if not entry.is_stale():
-                        result[port] = entry
-                return result
-            finally:
-                self._release_file_lock(lock_file)
+            data = self._read_ledger()
+            result = {}
+            for port, entry_data in data.items():
+                entry = LedgerEntry.from_dict(entry_data)
+                if not entry.is_stale():
+                    result[port] = entry
+            return result
 
     def get_environment(self, port: str) -> str | None:
         """Get the environment name for a cached port.