embedded-lab-control 0.2.0__py3-none-any.whl

labctl/sdwire/power_guard.py

@@ -0,0 +1,190 @@
+"""Power a board off before its SD card is switched to the host, and
+check that it is off before the card is taken from it."""
+
+import time
+from dataclasses import dataclass
+from typing import Optional
+
+# Why a board isn't confirmed off (BoardOffCheck.problem).
+NO_PLUG = "no_plug"
+UNREADABLE = "unreadable"
+POWERED_ON = "powered_on"
+NOT_OFF = "not_off"  # a state other than ON or OFF (e.g. unknown)
+
+
+@dataclass
+class BoardOffCheck:
+    """Result of :func:`check_board_off`: ``problem`` is None when the
+    plug reports the board OFF, else one of NO_PLUG, UNREADABLE (``error``
+    says why), POWERED_ON or NOT_OFF (``state`` says which)."""
+
+    problem: Optional[str] = None
+    state: object = None  # a PowerState when the plug was read
+    error: Optional[str] = None
+
+    @property
+    def is_off(self) -> bool:
+        return self.problem is None
+
+
+def check_board_off(sbc) -> BoardOffCheck:
+    """Whether ``sbc``'s plug reports it OFF: the one check behind every
+    "proceed only when the board is off" refusal (``sdwire host`` and
+    MCP ``sdwire_to_host``/``sdwire_ls``/``cat``/``info``, and the CLI
+    ``sdwire ls``/``cat``/``info``). Fails closed: no plug, a plug that
+    can't be read, or any state but OFF is a problem. Each caller words
+    its own refusal."""
+    if not sbc.power_plug:
+        return BoardOffCheck(NO_PLUG)
+    from labctl.power import PowerController
+    from labctl.power.base import PowerState
+
+    try:
+        state = PowerController.from_plug(sbc.power_plug).get_state()
+    except Exception as e:
+        return BoardOffCheck(UNREADABLE, error=str(e))
+    if state == PowerState.OFF:
+        return BoardOffCheck(state=state)
+    if state == PowerState.ON:
+        return BoardOffCheck(POWERED_ON, state=state)
+    return BoardOffCheck(NOT_OFF, state=state)
+
+
+def board_off_problem_text(
+    sbc_name: str, check: BoardOffCheck, *, sentence_start: bool = False
+) -> str:
+    """One sentence (no final period) saying why ``check`` doesn't confirm
+    ``sbc_name`` off; callers add their own framing and hints. Generic
+    leading words start lower-case unless ``sentence_start``; the board
+    name is never re-cased."""
+
+    def generic(text: str) -> str:
+        return text[0].upper() + text[1:] if sentence_start else text
+
+    if check.problem == NO_PLUG:
+        return f"{sbc_name} has no power plug, so labctl can't confirm it is off"
+    if check.problem == UNREADABLE:
+        return generic(f"could not read the power state of {sbc_name}: {check.error}")
+    if check.problem == POWERED_ON:
+        return f"{sbc_name} is powered on"
+    state = getattr(check.state, "value", check.state)
+    return generic(
+        f"the power state of {sbc_name} is {state}, so labctl can't confirm "
+        "it is off"
+    )
+
+
+# Second look before treating the card as on the board (see read_access).
+READ_RECHECK_SECONDS = 1.0
+
+
+def read_access(sbc, ctrl, *, force: bool = False):
+    """How a read of the SD card (CLI ``sdwire ls``/``cat``/``info``, the
+    MCP read tools) reaches it. Returns ``(needs_switch, refusal)``:
+
+    - Card already on the host (``ctrl.card_on_host()`` probes the reader
+      for media): read it in place and leave the mux alone; the board isn't
+      using the card, so its power doesn't matter. ``(False, None)``.
+    - Card on the board: switch to the host and back only when
+      :func:`check_board_off` confirms it off (or ``force``):
+      ``(True, None)``; otherwise ``(False, check)`` with the failed
+      :class:`BoardOffCheck` for the caller to word.
+
+    A card switched to the host moments ago (by another process) may still
+    be initialising and probe as "no medium"; concluding "on the board"
+    would make the cleanup switch it there, so it is probed once more
+    after :data:`READ_RECHECK_SECONDS`. (The mux can't be read back.)
+    """
+    if ctrl.card_on_host():
+        return False, None
+    time.sleep(READ_RECHECK_SECONDS)
+    if ctrl.card_on_host():
+        return False, None
+    if force:
+        return True, None
+    check = check_board_off(sbc)
+    if not check.is_off:
+        return False, check
+    return True, None
+
+
+class NoPlugError(RuntimeError):
+    """The board has no power plug, so it can't be switched off (or known
+    to be off) before its SD card is taken; callers add how to proceed."""
+
+
+def no_plug_refusal(sbc) -> str:
+    return (
+        f"{sbc.name!r} has no power plug, so labctl can't switch it off or "
+        "confirm it is off before switching its SD card to the host; nothing "
+        "was written to the card"
+    )
+
+
+def power_off_before_host_switch(sbc, *, no_plug_ok: bool = False) -> None:
+    """Switch ``sbc`` off before its SD card goes to the host.
+
+    Raises RuntimeError, before the card is touched, when the board can't
+    be switched off: moving the card away from a running board risks bus
+    contention and a corrupted card. A board with no power plug fails
+    closed too (:class:`NoPlugError`), unless ``no_plug_ok``: the caller
+    has made sure it is off. ``no_plug_ok`` never skips a power-off that a
+    plug makes possible, nor excuses one that fails.
+    """
+    if not sbc.power_plug:
+        if no_plug_ok:
+            return
+        raise NoPlugError(no_plug_refusal(sbc))
+    from labctl.power import PowerController
+
+    try:
+        ok = PowerController.from_plug(sbc.power_plug).power_off()
+    except Exception as e:
+        ok, why = False, f": {e}"
+    else:
+        why = ""
+    if not ok:
+        raise RuntimeError(
+            f"could not power off {sbc.name!r} before switching its SD card "
+            f"to the host{why}; nothing was written to the card"
+        )
+    time.sleep(1)
+
+
+def deploy_to_card(manager, sbc, ctrl, write) -> None:
+    """Power ``sbc`` off, switch its card to the host, run ``write()``, and
+    hand the card back to the board.
+
+    Raises RuntimeError for any failure. Once the board is off, the error
+    says it was left off, and whether the card is still on the host.
+    """
+    from labctl.actuators.runtime import _left_off, boards_on_plug
+
+    # A missing SDWire must not cost a running board its power.
+    try:
+        ctrl.ensure_present()
+    except Exception as e:
+        raise RuntimeError(f"{e}; nothing changed") from e
+    power_off_before_host_switch(sbc)
+    try:
+        ctrl.switch_to_host()
+        time.sleep(2)
+        write()
+    except Exception as e:
+        try:
+            ctrl.switch_to_dut()
+            card = ""
+        except Exception:
+            card = " (its SD card may still be on the host)"
+        raise RuntimeError(
+            f"deploy to {sbc.name!r} failed: {e}; "
+            f"{_left_off(sbc, boards_on_plug(manager, sbc))}{card}"
+        ) from e
+    try:
+        ctrl.switch_to_dut()
+    except Exception as e:
+        raise RuntimeError(
+            f"deployed to {sbc.name!r}, but its SD card could not be switched "
+            f"back to the board: {e}; the card is still on the host and "
+            f"{_left_off(sbc, boards_on_plug(manager, sbc))}"
+        ) from e

labctl/serial/__init__.py

@@ -0,0 +1,18 @@
+"""
+Serial port management for lab controller.
+
+Handles ser2net configuration generation, serial port discovery,
+and multi-client serial proxying.
+"""
+
+from labctl.serial.proxy import ProxyClient, ProxyManager, SerialProxy, SessionLogger
+from labctl.serial.ser2net import Ser2NetPort, generate_ser2net_config
+
+__all__ = [
+    "generate_ser2net_config",
+    "Ser2NetPort",
+    "SerialProxy",
+    "ProxyManager",
+    "ProxyClient",
+    "SessionLogger",
+]

labctl/serial/boot_test.py

@@ -0,0 +1,342 @@
+"""
+Automated boot reliability testing.
+
+Deploys a kernel image, reboots the SBC multiple times, and captures
+boot output to determine success rate.
+"""
+
+import logging
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from labctl.serial.capture import capture_serial_output
+
+logger = logging.getLogger(__name__)
+
+
+def _write_run_file(path: Path | str, text: str, dir_fd: Optional[int] = None) -> None:
+    """Write one run's output without following a planted link.
+
+    The output directory may be shared (e.g. group-writable
+    /var/lib/labctl/output), and the content is console output the DUT
+    controls. A symlink planted at ``run_NN.txt`` would otherwise redirect
+    the write to any file the service user can write, so open with
+    O_NOFOLLOW (fails with ELOOP on a symlink), and refuse a file with
+    other hard links. Anything that isn't a regular file (e.g. a planted
+    FIFO, which would otherwise block the open forever while the caller
+    holds the hardware lock) is refused without blocking. With ``dir_fd``
+    (a checked directory handle) the file is created relative to it, so the
+    directory path itself can't be swapped either. Raises OSError on
+    refusal.
+    """
+    from labctl.core.safe_open import open_regular_nonblocking
+
+    fd = open_regular_nonblocking(
+        str(path), os.O_WRONLY | os.O_CREAT, 0o644, dir_fd=dir_fd
+    )
+    try:
+        if os.fstat(fd).st_nlink > 1:
+            raise OSError(f"refusing to write {path}: it has other hard links")
+        os.ftruncate(fd, 0)
+        with os.fdopen(fd, "w", closefd=False) as f:
+            f.write(text)
+    finally:
+        os.close(fd)
+
+
+@dataclass
+class BootRunResult:
+    """Result of a single boot run."""
+
+    run_number: int
+    passed: bool
+    elapsed_seconds: float
+    pattern_matched: bool
+    output: str = ""
+    last_line: str = ""
+    error: str = ""
+
+
+@dataclass
+class BootTestResult:
+    """Result of a complete boot test."""
+
+    sbc_name: str
+    expect_pattern: str
+    total_runs: int
+    timeout_per_run: float
+    image: Optional[str] = None
+    dest: Optional[str] = None
+    partition: Optional[int] = None
+    runs: list[BootRunResult] = field(default_factory=list)
+    # Per-run output files that couldn't be written (name: reason); the boot
+    # results still stand, but the caller must be told the logs are missing.
+    output_errors: list[str] = field(default_factory=list)
+    # Why the test stopped before all its runs (e.g. the caller's claim
+    # on the board was lost), or None when every run was made.
+    stopped_reason: Optional[str] = None
+    # The deploy ran (it leaves the board powered off until the first run).
+    deployed: bool = False
+
+    @property
+    def passed_count(self) -> int:
+        return sum(1 for r in self.runs if r.passed)
+
+    @property
+    def failed_count(self) -> int:
+        return sum(1 for r in self.runs if not r.passed)
+
+    @property
+    def pass_rate(self) -> float:
+        if not self.runs:
+            return 0.0
+        return self.passed_count / len(self.runs) * 100
+
+    @property
+    def avg_boot_time(self) -> float:
+        passed = [r.elapsed_seconds for r in self.runs if r.passed]
+        if not passed:
+            return 0.0
+        return sum(passed) / len(passed)
+
+    def format_summary(self) -> str:
+        """Format as human-readable summary."""
+        lines = [
+            f"Boot Test Results: {self.sbc_name}",
+            "=" * 40,
+        ]
+
+        if self.image:
+            lines.append(
+                f"Image: {self.image} -> {self.dest} (partition {self.partition})"
+            )
+        else:
+            lines.append("Image: (no deploy, testing current SD contents)")
+
+        lines.append(f"Pattern: '{self.expect_pattern}'")
+        lines.append(f"Timeout: {self.timeout_per_run:.0f}s per boot")
+        lines.append("")
+
+        for r in self.runs:
+            status = "PASS" if r.passed else "FAIL"
+            detail = "pattern matched" if r.pattern_matched else ""
+            if not r.passed:
+                if r.error:
+                    detail = r.error
+                elif r.last_line:
+                    detail = f'timeout, last output: "{r.last_line}"'
+                else:
+                    detail = "timeout, no output"
+            lines.append(
+                f"Run {r.run_number:2d}/{self.total_runs}: "
+                f"{status} ({r.elapsed_seconds:.1f}s) - {detail}"
+            )
+
+        if self.stopped_reason:
+            lines.append(
+                f"STOPPED after {len(self.runs)} of {self.total_runs} runs: "
+                f"{self.stopped_reason}"
+            )
+            if not self.runs and self.deployed:
+                lines.append(
+                    f"{self.sbc_name} was powered off for the deploy and is "
+                    "left off."
+                )
+
+        lines.append("")
+        if self.stopped_reason and not self.runs:
+            lines.append("Result: STOPPED before the first run (no boots tested)")
+        elif self.stopped_reason:
+            lines.append(
+                f"Result: INCOMPLETE, {self.passed_count}/{len(self.runs)} "
+                f"boots successful ({self.pass_rate:.0f}%) in "
+                f"{len(self.runs)} of {self.total_runs} runs"
+            )
+        else:
+            lines.append(
+                f"Result: {self.passed_count}/{len(self.runs)} boots successful "
+                f"({self.pass_rate:.0f}%)"
+            )
+
+        if self.passed_count > 0:
+            lines.append(f"Average boot time (successful): {self.avg_boot_time:.1f}s")
+
+        # Failure mode breakdown
+        if self.failed_count > 0:
+            timeout_partial = sum(
+                1 for r in self.runs if not r.passed and not r.error and r.last_line
+            )
+            timeout_none = sum(
+                1 for r in self.runs if not r.passed and not r.error and not r.last_line
+            )
+            errors = sum(1 for r in self.runs if not r.passed and r.error)
+            lines.append("Failure modes:")
+            if timeout_partial:
+                lines.append(f"  - Timeout with partial output: {timeout_partial}")
+            if timeout_none:
+                lines.append(f"  - Timeout with no output: {timeout_none}")
+            if errors:
+                lines.append(f"  - Errors: {errors}")
+
+        if self.output_errors:
+            lines.append("")
+            lines.append(
+                f"WARNING: {len(self.output_errors)} run output file(s) "
+                "were NOT saved:"
+            )
+            lines.extend(f"  - {err}" for err in self.output_errors)
+
+        return "\n".join(lines)
+
+
+def run_boot_test(
+    sbc_name: str,
+    expect_pattern: str,
+    tcp_host: str,
+    tcp_port: int,
+    power_cycle_fn,
+    runs: int = 10,
+    timeout: float = 30.0,
+    deploy_fn=None,
+    image: str | None = None,
+    dest: str | None = None,
+    partition: int = 1,
+    output_dir: str | None = None,
+    output_dir_fd: Optional[int] = None,
+    progress_fn=None,
+    before_run_fn=None,
+) -> BootTestResult:
+    """Run automated boot reliability test.
+
+    Args:
+        sbc_name: Name of the SBC being tested
+        expect_pattern: Regex to match for "success"
+        tcp_host: Host where ser2net is running
+        tcp_port: TCP port for serial console
+        power_cycle_fn: Callable that power cycles the SBC (off, wait, on)
+        runs: Number of boot cycles
+        timeout: Seconds to wait per boot for pattern match
+        deploy_fn: Optional callable to deploy image before testing
+        image: Image filename (for reporting)
+        dest: Destination filename (for reporting)
+        partition: Partition number (for reporting)
+        output_dir: Save per-run output to files here
+        output_dir_fd: Optional open handle for ``output_dir`` (already
+            checked and created by the caller); files are created
+            relative to it and ``output_dir`` is only used for messages
+        progress_fn: Optional callback(run_number, total, result) for progress
+        before_run_fn: Optional callable run before each run's power cycle;
+            it returns None to go on, or a reason to stop the test there
+            (recorded as ``stopped_reason``). Used to stop when the
+            caller's claim on the board is lost mid-test.
+    """
+    result = BootTestResult(
+        sbc_name=sbc_name,
+        expect_pattern=expect_pattern,
+        total_runs=runs,
+        timeout_per_run=timeout,
+        image=image,
+        dest=dest,
+        partition=partition,
+    )
+
+    # Create output dir if needed (a caller-provided handle already exists).
+    # Before the deploy, which powers the board off: a failure here must
+    # leave the board as it was.
+    if output_dir and output_dir_fd is None:
+        os.makedirs(output_dir, exist_ok=True)
+
+    # Deploy if requested
+    if deploy_fn:
+        deploy_fn()
+        result.deployed = True
+
+    for i in range(1, runs + 1):
+        if before_run_fn:
+            reason = before_run_fn()
+            if reason:
+                logger.warning("Boot test of %s stopped: %s", sbc_name, reason)
+                result.stopped_reason = reason
+                break
+
+        run_result = _run_single_boot(
+            run_number=i,
+            tcp_host=tcp_host,
+            tcp_port=tcp_port,
+            power_cycle_fn=power_cycle_fn,
+            expect_pattern=expect_pattern,
+            timeout=timeout,
+        )
+
+        result.runs.append(run_result)
+
+        # Save per-run output
+        if output_dir and run_result.output:
+            name = f"run_{i:02d}.txt"
+            run_file = Path(output_dir) / name
+            try:
+                if output_dir_fd is not None:
+                    _write_run_file(name, run_result.output, dir_fd=output_dir_fd)
+                else:
+                    _write_run_file(run_file, run_result.output)
+            except OSError as e:
+                logger.warning("Not writing %s: %s", run_file, e)
+                result.output_errors.append(f"{name}: {e.strerror or e}")
+
+        if progress_fn:
+            progress_fn(i, runs, run_result)
+
+    return result
+
+
+def _run_single_boot(
+    run_number: int,
+    tcp_host: str,
+    tcp_port: int,
+    power_cycle_fn,
+    expect_pattern: str,
+    timeout: float,
+) -> BootRunResult:
+    """Execute a single boot cycle and capture output."""
+    try:
+        # Power cycle: off, wait, on
+        power_cycle_fn()
+
+        # Small delay for power to settle before capturing
+        time.sleep(1)
+
+        # Capture serial output
+        capture = capture_serial_output(
+            tcp_host=tcp_host,
+            tcp_port=tcp_port,
+            timeout=timeout,
+            until_pattern=expect_pattern,
+        )
+
+        # Extract last non-empty line for failure reporting
+        last_line = ""
+        if capture.output:
+            output_lines = capture.output.strip().splitlines()
+            if output_lines:
+                last_line = output_lines[-1][:80]  # Truncate
+
+        return BootRunResult(
+            run_number=run_number,
+            passed=capture.pattern_matched,
+            elapsed_seconds=capture.elapsed_seconds,
+            pattern_matched=capture.pattern_matched,
+            output=capture.output,
+            last_line=last_line,
+        )
+
+    except RuntimeError as e:
+        return BootRunResult(
+            run_number=run_number,
+            passed=False,
+            elapsed_seconds=0.0,
+            pattern_matched=False,
+            error=str(e),
+        )