embedded-lab-control 0.2.0__py3-none-any.whl

labctl/services.py

@@ -0,0 +1,210 @@
+"""
+Status checks for systemd services labctl depends on.
+
+Provides a single call that reports on all of labctl's own services
+plus external dependencies (ser2net). Used by `labctl services status`
+to give operators a quick at-a-glance health view without having to
+run `systemctl` against each unit individually.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from typing import Optional
+
+# Units labctl depends on. First three are labctl's own; ser2net is
+# the hardware-facing TCP-to-serial proxy used for every console.
+DEFAULT_UNITS: tuple[str, ...] = (
+    "labctl-monitor",
+    "labctl-mcp",
+    "labctl-web",
+    "ser2net",
+)
+
+
+@dataclass
+class ServiceStatus:
+    """One service's health snapshot."""
+
+    unit: str
+    active_state: str = "unknown"  # active | inactive | failed | activating | ...
+    sub_state: str = ""  # running | dead | exited | ...
+    active_since: Optional[datetime] = None
+    n_restarts: int = 0
+    exec_main_status: Optional[int] = None
+    result: str = ""  # success | exit-code | signal | oom-kill | ...
+    recent_errors: list[str] = field(default_factory=list)
+    error: Optional[str] = None  # set if systemctl itself failed
+
+    @property
+    def healthy(self) -> bool:
+        return self.active_state == "active" and self.sub_state in {"running", "exited"}
+
+    def uptime_str(self) -> str:
+        if not self.active_since:
+            return "-"
+        delta = datetime.now().astimezone() - self.active_since
+        return _format_duration(delta)
+
+
+def _format_duration(delta: timedelta) -> str:
+    total = int(delta.total_seconds())
+    if total < 0:
+        return "-"
+    days, rem = divmod(total, 86400)
+    hours, rem = divmod(rem, 3600)
+    mins, secs = divmod(rem, 60)
+    if days:
+        return f"{days}d{hours}h"
+    if hours:
+        return f"{hours}h{mins}m"
+    if mins:
+        return f"{mins}m{secs}s"
+    return f"{secs}s"
+
+
+def _run(cmd: list[str], timeout: float = 5.0) -> tuple[int, str]:
+    """Run a subprocess and capture combined output."""
+    try:
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            check=False,
+        )
+        return result.returncode, (result.stdout or "") + (result.stderr or "")
+    except FileNotFoundError as e:
+        return 127, str(e)
+    except subprocess.TimeoutExpired:
+        return 124, f"timeout after {timeout}s"
+
+
+def _parse_systemctl_show(output: str) -> dict[str, str]:
+    """Parse `systemctl show -p Key1,Key2 unit` KEY=VALUE lines."""
+    props: dict[str, str] = {}
+    for line in output.splitlines():
+        if "=" in line:
+            k, v = line.split("=", 1)
+            props[k.strip()] = v.strip()
+    return props
+
+
+def _parse_systemd_timestamp(value: str) -> Optional[datetime]:
+    """Parse a systemd timestamp like 'Sun 2026-04-19 16:08:26 PDT'.
+
+    Returns None for empty / sentinel values ("n/a", "0", "").
+    """
+    if not value or value in {"n/a", "0"}:
+        return None
+    # Format example: "Sun 2026-04-19 16:08:26 PDT"
+    for fmt in (
+        "%a %Y-%m-%d %H:%M:%S %Z",
+        "%Y-%m-%d %H:%M:%S %Z",
+        "%a %Y-%m-%d %H:%M:%S",
+        "%Y-%m-%d %H:%M:%S",
+    ):
+        try:
+            dt = datetime.strptime(value, fmt)
+            return dt.astimezone()
+        except ValueError:
+            continue
+    return None
+
+
+def _collect_recent_errors(unit: str, hours: int = 24, limit: int = 3) -> list[str]:
+    """Pull the last few distinct error/warning messages from the unit's journal.
+
+    Uses journalctl with priority <= err. Returns at most `limit` lines.
+    Silently returns [] if journalctl isn't available.
+    """
+    if shutil.which("journalctl") is None:
+        return []
+    code, out = _run(
+        [
+            "journalctl",
+            "-u",
+            unit,
+            f"--since={hours} hours ago",
+            "--no-pager",
+            "-q",
+            "-p",
+            "err",
+            "-o",
+            "short-iso",
+        ],
+        timeout=5.0,
+    )
+    if code != 0:
+        return []
+    lines = [ln.strip() for ln in out.splitlines() if ln.strip()]
+    # De-duplicate while preserving order, keep the last `limit` distinct.
+    seen: set[str] = set()
+    uniq: list[str] = []
+    for ln in reversed(lines):
+        # Strip leading timestamp + hostname + unit[pid]: to dedupe by message.
+        msg = ln.split(": ", 1)[-1] if ": " in ln else ln
+        if msg in seen:
+            continue
+        seen.add(msg)
+        uniq.append(ln)
+        if len(uniq) >= limit:
+            break
+    return list(reversed(uniq))
+
+
+def check_service(unit: str) -> ServiceStatus:
+    """Check one systemd unit and return a structured status."""
+    status = ServiceStatus(unit=unit)
+
+    if shutil.which("systemctl") is None:
+        status.error = "systemctl not found"
+        return status
+
+    code, out = _run(
+        [
+            "systemctl",
+            "show",
+            unit,
+            "-p",
+            "ActiveState,SubState,ActiveEnterTimestamp,NRestarts,"
+            "ExecMainStatus,Result,LoadState",
+        ]
+    )
+    if code != 0:
+        status.error = out.strip() or f"systemctl show exited {code}"
+        return status
+
+    props = _parse_systemctl_show(out)
+    if props.get("LoadState") in {"not-found", "masked"}:
+        status.error = f"unit not loaded (LoadState={props.get('LoadState')})"
+        status.active_state = "not-found"
+        return status
+
+    status.active_state = props.get("ActiveState", "unknown")
+    status.sub_state = props.get("SubState", "")
+    status.active_since = _parse_systemd_timestamp(
+        props.get("ActiveEnterTimestamp", "")
+    )
+    try:
+        status.n_restarts = int(props.get("NRestarts", "0"))
+    except ValueError:
+        status.n_restarts = 0
+    try:
+        status.exec_main_status = int(props.get("ExecMainStatus", "0"))
+    except ValueError:
+        status.exec_main_status = None
+    status.result = props.get("Result", "")
+
+    if not status.healthy or status.n_restarts > 0:
+        status.recent_errors = _collect_recent_errors(unit)
+
+    return status
+
+
+def check_all(units: tuple[str, ...] = DEFAULT_UNITS) -> list[ServiceStatus]:
+    """Check all services. Returned in the order given."""
+    return [check_service(u) for u in units]

labctl/web/__init__.py

@@ -0,0 +1,9 @@
+"""
+Web interface for lab controller.
+
+Provides REST API and web dashboard for managing lab resources.
+"""
+
+from labctl.web.app import create_app
+
+__all__ = ["create_app"]

labctl/web/activity_broadcaster.py

@@ -0,0 +1,204 @@
+"""
+Activity-stream broadcaster for the web server.
+
+A single background thread polls `audit_log` for new rows and fans
+them out to per-subscriber queues. SSE endpoints iterate their queue
+to stream events to clients.
+
+Polling the DB (rather than hooking `audit.emit()` directly) means
+events from CLI, MCP, and the monitor daemon — which run in separate
+processes — all reach the browser.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import queue
+import threading
+import time
+from dataclasses import dataclass
+from typing import Iterator, Optional
+
+from labctl.core.database import Database
+
+logger = logging.getLogger(__name__)
+
+POLL_INTERVAL_SECONDS = 0.5
+BATCH_LIMIT = 500
+SUBSCRIBER_QUEUE_MAXSIZE = 1000
+INITIAL_BACKFILL = 200  # Hydration events a new subscriber receives immediately.
+
+
+@dataclass
+class ActivityEvent:
+    """One event pushed to subscribers."""
+
+    id: int
+    logged_at: str
+    actor: str
+    source: str
+    action: str
+    entity_type: str
+    entity_id: Optional[int]
+    entity_name: Optional[str]
+    result: str
+    details: Optional[str]  # JSON string as stored
+    claim_id: Optional[int]
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "logged_at": self.logged_at,
+            "actor": self.actor,
+            "source": self.source,
+            "action": self.action,
+            "entity_type": self.entity_type,
+            "entity_id": self.entity_id,
+            "entity_name": self.entity_name,
+            "result": self.result,
+            "details": self.details,
+            "claim_id": self.claim_id,
+        }
+
+    def to_sse(self) -> str:
+        """Render as a single SSE frame."""
+        payload = json.dumps(self.to_dict(), separators=(",", ":"))
+        return f"id: {self.id}\nevent: activity\ndata: {payload}\n\n"
+
+
+def _row_to_event(row) -> ActivityEvent:
+    return ActivityEvent(
+        id=row["id"],
+        logged_at=row["logged_at"] or "",
+        actor=row["actor"] or "internal",
+        source=row["source"] or "internal",
+        action=row["action"] or "",
+        entity_type=row["entity_type"] or "",
+        entity_id=row["entity_id"],
+        entity_name=row["entity_name"],
+        result=row["result"] or "ok",
+        details=row["details"],
+        claim_id=row["claim_id"],
+    )
+
+
+class ActivityBroadcaster:
+    """Singleton broadcaster: polls the audit_log and feeds subscriber queues."""
+
+    def __init__(self, db: Database, poll_interval: float = POLL_INTERVAL_SECONDS):
+        self.db = db
+        self.poll_interval = poll_interval
+        self._last_id: int = 0
+        self._subscribers: set[queue.Queue] = set()
+        self._subscribers_lock = threading.Lock()
+        self._stop_event = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+
+    def start(self) -> None:
+        """Start the background poll thread. Safe to call multiple times."""
+        if self._thread and self._thread.is_alive():
+            return
+        # Seed the cursor with the current max id so we don't replay history.
+        row = self.db.execute_one("SELECT COALESCE(MAX(id), 0) AS m FROM audit_log")
+        self._last_id = row["m"] if row else 0
+        self._stop_event.clear()
+        self._thread = threading.Thread(
+            target=self._run, name="activity-broadcaster", daemon=True
+        )
+        self._thread.start()
+        logger.info(
+            "ActivityBroadcaster started (last_id=%d, interval=%.2fs)",
+            self._last_id,
+            self.poll_interval,
+        )
+
+    def stop(self) -> None:
+        """Signal the poll thread to stop. Does not join."""
+        self._stop_event.set()
+
+    def subscribe(self) -> queue.Queue:
+        """Register a new subscriber queue. Call unsubscribe() when done."""
+        q: queue.Queue = queue.Queue(maxsize=SUBSCRIBER_QUEUE_MAXSIZE)
+        with self._subscribers_lock:
+            self._subscribers.add(q)
+        return q
+
+    def unsubscribe(self, q: queue.Queue) -> None:
+        with self._subscribers_lock:
+            self._subscribers.discard(q)
+
+    def recent(self, limit: int = INITIAL_BACKFILL) -> list[ActivityEvent]:
+        """Return the most recent N events (oldest-first for replay)."""
+        rows = self.db.execute(
+            "SELECT * FROM audit_log ORDER BY id DESC LIMIT ?",
+            (int(limit),),
+        )
+        return [_row_to_event(r) for r in reversed(rows)]
+
+    def _run(self) -> None:
+        while not self._stop_event.is_set():
+            try:
+                self._poll_once()
+            except Exception as e:  # noqa: BLE001
+                logger.warning("ActivityBroadcaster poll failed: %s", e)
+            self._stop_event.wait(self.poll_interval)
+
+    def _poll_once(self) -> None:
+        rows = self.db.execute(
+            "SELECT * FROM audit_log WHERE id > ? ORDER BY id ASC LIMIT ?",
+            (self._last_id, BATCH_LIMIT),
+        )
+        if not rows:
+            return
+        events = [_row_to_event(r) for r in rows]
+        self._last_id = events[-1].id
+        self._fanout(events)
+
+    def _fanout(self, events: list[ActivityEvent]) -> None:
+        """Push events to every subscriber queue. Drop slow consumers."""
+        with self._subscribers_lock:
+            subscribers = list(self._subscribers)
+        for q in subscribers:
+            for event in events:
+                try:
+                    q.put_nowait(event)
+                except queue.Full:
+                    # Slow consumer: drop all events to catch up rather than
+                    # block others. The client will still have its initial
+                    # backfill + whatever lands next.
+                    try:
+                        while True:
+                            q.get_nowait()
+                    except queue.Empty:
+                        pass
+                    logger.info("ActivityBroadcaster: dropped slow subscriber queue")
+                    break  # don't keep adding to a drained queue this tick
+
+
+def iter_sse_frames(
+    broadcaster: ActivityBroadcaster,
+    backfill: int = INITIAL_BACKFILL,
+    heartbeat_seconds: float = 15.0,
+) -> Iterator[str]:
+    """Yield SSE frames: first a hydration batch, then live events.
+
+    Sends a comment-only heartbeat line every `heartbeat_seconds` so
+    intermediaries (nginx, browsers) don't drop an idle connection.
+    """
+    q = broadcaster.subscribe()
+    try:
+        yield "retry: 2000\n\n"
+        for event in broadcaster.recent(backfill):
+            yield event.to_sse()
+        last_beat = time.monotonic()
+        while True:
+            timeout = max(0.0, heartbeat_seconds - (time.monotonic() - last_beat))
+            try:
+                event = q.get(timeout=timeout or 0.1)
+                yield event.to_sse()
+            except queue.Empty:
+                yield ": keepalive\n\n"
+                last_beat = time.monotonic()
+    finally:
+        broadcaster.unsubscribe(q)