alissa-tools-github-devloop 0.8.46__py3-none-any.whl → 0.8.48__py3-none-any.whl

alissa/tools/github/devloop/config.py

@@ -132,6 +132,36 @@ ENV_REVERT_LABEL = "ALISSA_REVERT_LABEL"
 # worker's worktree as part of the session; the daemon never executes it.
 ENV_REGENERATE = "ALISSA_DEV_REGENERATE"
 
+# The hung-test reaper's window (issue #174): minutes a worker's test-runner
+# tree may sit at under `hungtests.MIN_PROGRESS_SECONDS` of CPU growth before
+# the daemon kills it. Env-only, read straight from the environment like
+# ENV_REGENERATE: not a config key, not a build ARG. Default 20; 0 turns the
+# reaper off; blank falls through to the default; anything that is not a
+# non-negative integer is a config error at boot, by name -- a typo silently
+# read as "off" would leave a 55 h hung suite exactly where it was.
+ENV_HUNG_TEST_MINUTES = "ALISSA_DEV_HUNG_TEST_MINUTES"
+DEFAULT_HUNG_TEST_MINUTES = 20
+
+
+def parse_hung_test_minutes(raw: str) -> int:
+    """The ALISSA_DEV_HUNG_TEST_MINUTES value as minutes (0 = off)."""
+    text = (raw or "").strip()
+    if not text:
+        return DEFAULT_HUNG_TEST_MINUTES
+    try:
+        minutes = int(text)
+    except ValueError:
+        raise ValueError(
+            f"{ENV_HUNG_TEST_MINUTES} must be a whole number of minutes "
+            f"(0 = off), got {raw!r}"
+        ) from None
+    if minutes < 0:
+        raise ValueError(
+            f"{ENV_HUNG_TEST_MINUTES} must be >= 0 (0 = off), got {minutes}"
+        )
+    return minutes
+
+
 # `owner/repo` for a regenerate entry: exactly one `/`, no whitespace, and no
 # `:` (the separator that ends it). Shape only, like the actor-id check.
 _REGENERATE_REPO_RE = re.compile(r"^[^\s/:]+/[^\s/:]+\Z")
@@ -857,6 +887,12 @@ class Config:
     # Read through `regenerate_for(owner, repo)`, never executed here.
     regenerate: tuple[RegenerateRule, ...] = ()
 
+    # The hung-test reaper's window in minutes (issue #174), from
+    # ALISSA_DEV_HUNG_TEST_MINUTES only (no config key, no CLI flag). A
+    # worker's `npm test` / `tsx --test` tree whose CPU grows by under 5 s
+    # across this window is killed; 0 turns the reaper off.
+    hung_test_minutes: int = DEFAULT_HUNG_TEST_MINUTES
+
     dry_run: bool = False
 
     # Derived at build time: `repos` casefolded for matching. GitHub names are
@@ -1118,6 +1154,10 @@ class Config:
         # The regenerable-paths map (issue #138): env-only, validated here so
         # a malformed entry fails the boot by name instead of a directive.
         regenerate = parse_regenerate(env_map.get(ENV_REGENERATE, ""))
+        # The hung-test window (issue #174): env-only, validated here.
+        hung_test_minutes = parse_hung_test_minutes(
+            env_map.get(ENV_HUNG_TEST_MINUTES, "")
+        )
 
         bow_owners = normalize_bow_owners(raw.get("bow_owners", cls.bow_owners))
 
@@ -1430,6 +1470,7 @@ class Config:
             ),
             disk_warn_percent=warn_percent,
             regenerate=regenerate,
+            hung_test_minutes=hung_test_minutes,
             dry_run=bool(raw.get("dry_run", cls.dry_run)),
         )
 

alissa/tools/github/devloop/hungtests.py

@@ -0,0 +1,481 @@
+"""The hung-test reaper's mechanics (issue #174).
+
+On 2026-10-18 the studio devloop container ran at load ~15 while four worker
+`npm test` runs sat hung, each at about 10 CPU-seconds in total -- one of them
+for 55 hours, its lane long merged. Their test workers had died (esbuild's
+"The service was stopped", EAGAIN with the box out of processes), the
+`tsx --test` runner then waited forever, and the sessions waited on it in an
+`until grep -q '^EXIT=' suite.log` loop. Nothing in the fleet noticed: a
+session waiting on a log reads ALIVE to every check the daemon has.
+
+What lives here is the janitor's split again -- only the MECHANICS: which
+process trees are worker test runs (`find_test_trees`), whether one has made
+CPU progress across a window (`CpuWatch`), how to stop one (`kill_tree`), and
+how to tell its session (`message_session`). The POLICY -- which hung or
+orphaned tree to kill, and when -- lives in `loop.py` next to the janitor
+whose predicates it shares.
+
+The properties everything here is written around:
+
+* Nothing outside a task workspace is ever a candidate. A tree is admitted
+  only when its cwd or one of its args sits inside a watched hub's task
+  workspace, judged by the janitor's own grammar (`parse_task_workspace`), so
+  `main/`, `.source/`, an operator's shell and the daemon itself are never
+  CONSIDERED -- the property the tests pin, rather than "checked and spared".
+* CPU progress, not wall-clock, is the hang signal. A full suite that runs for
+  an hour is fine; a runner that burned under `MIN_PROGRESS_SECONDS` of CPU
+  over the whole window is waiting on something that is not coming back.
+* Nothing raises past its own concern. A process that exits mid-read, an
+  unreadable `/proc` entry, a signal to a pid that is already gone -- all are
+  skipped, because this runs inside a poll pass.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import signal
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable, Iterable, Mapping, Sequence
+
+from .janitor import TaskWorkspace, parse_task_workspace
+from .proc import CommandError, run
+
+log = logging.getLogger(__name__)
+
+# Where the process table is read. A module attribute (read at call time,
+# never bound as a default) so the tests point it at a fake tree.
+PROC_ROOT = Path("/proc")
+
+# A tree whose summed CPU time grows by less than this across the window is
+# hung. The incident's trees sat at ~10 CPU-seconds TOTAL over hours; a live
+# runner burns this much in seconds.
+MIN_PROGRESS_SECONDS = 5.0
+
+# TERM, then KILL whatever is still alive this long after.
+KILL_GRACE_SECONDS = 10.0
+
+# How often the TERM→KILL wait re-reads the tree.
+_KILL_POLL_SECONDS = 0.5
+
+# The env a managed seat exports (alissa CLI): the tmux session it runs in and
+# the Alissa session id the session-message path addresses. Inherited by every
+# process the seat starts, detached ones included -- which is why they are
+# read off the tree rather than its ancestry: a `nohup npm test &` whose shell
+# exited is reparented to init and has no ancestor pane at all.
+ENV_TMUX_SESSION = "ALISSA_TMUX_SESSION"
+ENV_SESSION_ID = "ALISSA_SESSION_ID"
+
+# The prefix the CLI puts on a managed session's real tmux name; the roster
+# (`alissa tmux ls --json`'s `name`) carries it bare.
+TMUX_PREFIX = "ali-"
+
+# `npm test` and its aliases, plus `npm run test` / `npm run-script test`.
+_NPM_BINARIES = frozenset({"npm", "npm-cli.js"})
+_NPM_TEST = frozenset({"test", "t", "tst"})
+_NPM_RUN = frozenset({"run", "run-script", "rum", "urn"})
+
+
+@dataclass(frozen=True, slots=True)
+class Proc:
+    """One process, as `/proc/<pid>` described it at read time.
+
+    `cpu` is utime+stime PLUS cutime+cstime, in seconds: the reaped children's
+    time is the runner's progress too. A runner that executes one test file
+    after another reaps each child as it finishes; without the children's
+    share the tree's sum would DROP between samples while it works."""
+
+    pid: int
+    ppid: int
+    state: str
+    start: int
+    cpu: float
+    argv: "tuple[str, ...]"
+    cwd: "str | None"
+
+    @property
+    def alive(self) -> bool:
+        return self.state not in ("Z", "X")
+
+
+@dataclass(frozen=True)
+class TestTree:
+    """One worker test run: the topmost runner process and every descendant,
+    with the task workspace it was admitted under."""
+
+    root: Proc
+    members: "tuple[Proc, ...]"
+    workspace: TaskWorkspace
+    tmux_session: "str | None" = None
+    session_id: "str | None" = None
+    age: float = 0.0
+
+    @property
+    def key(self) -> "tuple[int, int]":
+        """(pid, start ticks): a pid alone is reused; the pair is not."""
+        return (self.root.pid, self.root.start)
+
+    @property
+    def cpu(self) -> float:
+        return sum(member.cpu for member in self.members)
+
+    @property
+    def session(self) -> "str | None":
+        """The owning session's roster name (no `ali-` prefix), or None."""
+        name = self.tmux_session
+        if not name:
+            return None
+        return name[len(TMUX_PREFIX):] if name.startswith(TMUX_PREFIX) else name
+
+    @property
+    def command(self) -> str:
+        return " ".join(self.root.argv)
+
+
+def _clock_ticks() -> int:
+    try:
+        return int(os.sysconf("SC_CLK_TCK")) or 100
+    except (ValueError, OSError, AttributeError):  # pragma: no cover
+        return 100
+
+
+def read_proc(pid: int, root: "Path | None" = None) -> "Proc | None":
+    """`/proc/<pid>` as a Proc, or None when it cannot be read (gone, a
+    kernel thread with no cmdline, permission)."""
+    base = (PROC_ROOT if root is None else root) / str(pid)
+    try:
+        stat = (base / "stat").read_text()
+        raw = (base / "cmdline").read_bytes()
+    except OSError:
+        return None
+    # The comm field is parenthesised and may itself hold spaces or `)`;
+    # everything after the LAST `)` is the fixed-position tail.
+    _, _, tail = stat.rpartition(")")
+    fields = tail.split()
+    try:
+        state, ppid = fields[0], int(fields[1])
+        ticks = sum(int(value) for value in fields[11:15])
+        start = int(fields[19])
+    except (IndexError, ValueError):
+        return None
+    argv = tuple(part.decode("utf-8", "replace") for part in raw.split(b"\0") if part)
+    if not argv:
+        return None
+    try:
+        cwd: "str | None" = os.readlink(base / "cwd")
+    except OSError:
+        cwd = None
+    return Proc(
+        pid=pid, ppid=ppid, state=state, start=start,
+        cpu=ticks / _clock_ticks(), argv=argv, cwd=cwd,
+    )
+
+
+def snapshot(root: "Path | None" = None) -> "dict[int, Proc]":
+    """Every readable process, by pid. An unreadable table yields nothing."""
+    base = PROC_ROOT if root is None else root
+    try:
+        names = [entry.name for entry in base.iterdir()]
+    except OSError as exc:
+        log.warning("hung tests: cannot list %s (%s) — skipped this pass", base, exc)
+        return {}
+    procs = {}
+    for name in names:
+        if not name.isdigit():
+            continue
+        proc = read_proc(int(name), base)
+        if proc is not None:
+            procs[proc.pid] = proc
+    return procs
+
+
+def uptime(root: "Path | None" = None) -> "float | None":
+    """Seconds since boot (`/proc/uptime`), or None."""
+    base = PROC_ROOT if root is None else root
+    try:
+        return float((base / "uptime").read_text().split()[0])
+    except (OSError, IndexError, ValueError):
+        return None
+
+
+def is_test_runner(argv: "Sequence[str]") -> bool:
+    """Is this command line a test runner: `npm test` (or `npm run test`),
+    or anything carrying a bare `--test` argument -- `node --test`,
+    `tsx --test`, `npx tsx --test`, and a `timeout … --test` wrapper around
+    one. Exact tokens only: a `bash -c '… --test …'` is one argument holding
+    the words, and the shell is not the runner."""
+    if "--test" in argv:
+        return True
+    for index, token in enumerate(argv):
+        if os.path.basename(token) not in _NPM_BINARIES:
+            continue
+        rest = [arg for arg in argv[index + 1:] if not arg.startswith("-")]
+        if rest and rest[0] in _NPM_TEST:
+            return True
+        if len(rest) >= 2 and rest[0] in _NPM_RUN and rest[1] == "test":
+            return True
+    return False
+
+
+def _path_in(text: str) -> "str | None":
+    if text.startswith("file://"):
+        text = text[len("file://"):]
+    return text if text.startswith("/") else None
+
+
+def workspace_of(
+    proc: Proc, hubs: "Sequence[Path]"
+) -> "TaskWorkspace | None":
+    """The task workspace `proc`'s cwd or one of its path args sits in, or
+    None. The hub's direct child that holds the path must parse as a task
+    workspace under the janitor's grammar -- `main/`, `.source/` and anything
+    else under a hub never qualify."""
+    candidates = [proc.cwd] if proc.cwd else []
+    candidates += [p for p in (_path_in(arg) for arg in proc.argv) if p]
+    for text in candidates:
+        path = Path(os.path.normpath(text))
+        for hub in hubs:
+            try:
+                relative = path.relative_to(hub)
+            except ValueError:
+                continue
+            if not relative.parts:
+                continue
+            workspace = parse_task_workspace(hub / relative.parts[0], hub)
+            if workspace is not None:
+                return workspace
+    return None
+
+
+def session_env(pid: int, root: "Path | None" = None) -> "tuple[str | None, str | None]":
+    """(ALISSA_TMUX_SESSION, ALISSA_SESSION_ID) from the process's env."""
+    base = (PROC_ROOT if root is None else root) / str(pid)
+    try:
+        raw = (base / "environ").read_bytes()
+    except OSError:
+        return None, None
+    found: "dict[str, str]" = {}
+    for part in raw.split(b"\0"):
+        name, sep, value = part.decode("utf-8", "replace").partition("=")
+        if sep and name in (ENV_TMUX_SESSION, ENV_SESSION_ID) and value:
+            found[name] = value
+    return found.get(ENV_TMUX_SESSION), found.get(ENV_SESSION_ID)
+
+
+def _children(procs: "Mapping[int, Proc]") -> "dict[int, list[int]]":
+    children: "dict[int, list[int]]" = {}
+    for proc in procs.values():
+        children.setdefault(proc.ppid, []).append(proc.pid)
+    return children
+
+
+def descendants(procs: "Mapping[int, Proc]", pid: int) -> "list[Proc]":
+    """`pid` and every process below it, root first."""
+    children = _children(procs)
+    order: "list[Proc]" = []
+    stack = [pid]
+    seen: "set[int]" = set()
+    while stack:
+        current = stack.pop()
+        if current in seen or current not in procs:
+            continue
+        seen.add(current)
+        order.append(procs[current])
+        stack.extend(sorted(children.get(current, ()), reverse=True))
+    return order
+
+
+def _has_runner_ancestor(procs: "Mapping[int, Proc]", proc: Proc) -> bool:
+    """Is any live ancestor of `proc` itself a runner? A `sh -c 'tsx --test'`
+    between two runners is one argument and not a runner, so the parent alone
+    cannot answer -- the whole chain up to init is read."""
+    seen = {proc.pid}
+    current = procs.get(proc.ppid)
+    while current is not None and current.pid not in seen:
+        if current.alive and is_test_runner(current.argv):
+            return True
+        seen.add(current.pid)
+        current = procs.get(current.ppid)
+    return False
+
+
+def find_test_trees(
+    procs: "Mapping[int, Proc]",
+    hubs: "Sequence[Path]",
+    root: "Path | None" = None,
+) -> "list[TestTree]":
+    """Every worker test run inside a task workspace of one of `hubs`.
+
+    A tree is rooted at its TOPMOST runner (a runner with no runner above it),
+    so `timeout 900 npx tsx --test` and the `npm exec` / `tsx` / `node --test`
+    chain under it are one tree, killed together. The workspace is read off
+    the first member, root first, whose cwd or args name one; a tree none of
+    whose members does is not ours and is never returned."""
+    resolved = []
+    for hub in hubs:
+        try:
+            resolved.append(Path(hub).resolve())
+        except OSError:
+            continue
+    boot = uptime(root)
+    ticks = _clock_ticks()
+    trees = []
+    for proc in sorted(procs.values(), key=lambda p: p.pid):
+        if not proc.alive or not is_test_runner(proc.argv):
+            continue
+        if _has_runner_ancestor(procs, proc):
+            continue
+        members = tuple(m for m in descendants(procs, proc.pid) if m.alive)
+        workspace = None
+        for member in members:
+            workspace = workspace_of(member, resolved)
+            if workspace is not None:
+                break
+        if workspace is None:
+            continue
+        tmux_session, session_id = session_env(proc.pid, root)
+        age = 0.0 if boot is None else max(0.0, boot - proc.start / ticks)
+        trees.append(TestTree(
+            root=proc, members=members, workspace=workspace,
+            tmux_session=tmux_session, session_id=session_id, age=age,
+        ))
+    return trees
+
+
+@dataclass
+class CpuWatch:
+    """The last CPU sample per tree, and the hang verdict across a window.
+
+    A tree is sampled the first time it is seen; once `window` seconds have
+    passed since its sample it is judged: hung when it is at least `window`
+    old and its CPU grew by under `MIN_PROGRESS_SECONDS`, otherwise the
+    sample moves forward and a new window starts. A tree that burns CPU is
+    therefore re-judged every window, and a tree younger than one never is.
+    Samples of trees that are no longer seen are dropped."""
+
+    window: float
+    clock: "Callable[[], float] | None" = None
+    samples: "dict[tuple[int, int], tuple[float, float]]" = field(default_factory=dict)
+
+    def observe(self, trees: "Iterable[TestTree]") -> "list[TestTree]":
+        now = (self.clock or monotonic)()
+        hung = []
+        seen = set()
+        for tree in trees:
+            seen.add(tree.key)
+            sample = self.samples.get(tree.key)
+            if sample is None:
+                self.samples[tree.key] = (now, tree.cpu)
+                continue
+            at, cpu = sample
+            if now - at < self.window:
+                continue
+            if tree.age >= self.window and tree.cpu - cpu < MIN_PROGRESS_SECONDS:
+                hung.append(tree)
+                continue
+            self.samples[tree.key] = (now, tree.cpu)
+        for key in list(self.samples):
+            if key not in seen:
+                del self.samples[key]
+        return hung
+
+
+def send_signal(pid: int, signum: int) -> None:
+    """`os.kill`, swallowing a pid that is already gone or not ours."""
+    try:
+        os.kill(pid, signum)
+    except (ProcessLookupError, PermissionError):
+        pass
+
+
+def sleep(seconds: float) -> None:
+    """`time.sleep`, behind a module seam the tests replace."""
+    time.sleep(seconds)
+
+
+def monotonic() -> float:
+    """`time.monotonic`, behind a module seam the tests replace."""
+    return time.monotonic()
+
+
+def _still_alive(member: Proc, root: "Path | None") -> bool:
+    now = read_proc(member.pid, root)
+    return now is not None and now.start == member.start and now.alive
+
+
+def kill_tree(
+    tree: TestTree,
+    *,
+    root: "Path | None" = None,
+    grace: float = KILL_GRACE_SECONDS,
+) -> "tuple[int, int]":
+    """TERM every member of `tree`, wait up to `grace` seconds, then KILL
+    whatever is still alive. Returns (termed, killed).
+
+    The member list is re-read first, so a child the runner spawned since the
+    snapshot is signalled too; every pid is matched on its start time before
+    each signal, so a pid the kernel reused for something else is never hit.
+    """
+    fresh = snapshot(root)
+    current = fresh.get(tree.root.pid)
+    if current is None or current.start != tree.root.start:
+        return 0, 0
+    members = [m for m in descendants(fresh, tree.root.pid) if m.alive]
+    for member in members:
+        send_signal(member.pid, signal.SIGTERM)
+    deadline = monotonic() + grace
+    survivors = [m for m in members if _still_alive(m, root)]
+    while survivors and monotonic() < deadline:
+        sleep(_KILL_POLL_SECONDS)
+        survivors = [m for m in survivors if _still_alive(m, root)]
+    for member in survivors:
+        send_signal(member.pid, signal.SIGKILL)
+    return len(members), len(survivors)
+
+
+def session_id_for(tmux_session: str, runner: "Callable[..., str]" = run) -> "str | None":
+    """The Alissa session id behind a managed tmux session, read off
+    `alissa session list --json`'s `tmuxSession`, or None."""
+    try:
+        out = runner(["alissa", "session", "list", "--json", "--limit", "50"], timeout=60)
+        data = json.loads(out)
+    except (CommandError, OSError, ValueError) as exc:
+        log.debug("hung tests: could not list sessions (%s)", exc)
+        return None
+    if isinstance(data, dict):
+        data = data.get("sessions", [])
+    wanted = {tmux_session, TMUX_PREFIX + tmux_session}
+    for entry in data if isinstance(data, list) else []:
+        if isinstance(entry, dict) and entry.get("tmuxSession") in wanted:
+            value = entry.get("sessionId")
+            if isinstance(value, str) and value:
+                return value
+    return None
+
+
+def message_session(
+    session_id: str, text: str, runner: "Callable[..., str]" = run
+) -> bool:
+    """Post `text` to one session through the session-message path
+    (`alissa session msg send -s <id>`). True when the CLI accepted it."""
+    try:
+        runner(["alissa", "session", "msg", "send", "-s", session_id, text], timeout=60)
+    except (CommandError, OSError) as exc:
+        log.warning("hung tests: could not message session %s (%s)", session_id, exc)
+        return False
+    return True
+
+
+def hung_message(tree: TestTree, minutes: int, reason: str) -> str:
+    """The one message a live session gets after its test run was stopped."""
+    return (
+        f"devloop: your test run in {tree.workspace.path.name} "
+        f"(`{tree.command}`, pid {tree.root.pid}) hung and was stopped "
+        f"({reason}). Do not rerun the full suite and never wait on one for "
+        f"more than {minutes} minutes: run only the test files your diff "
+        f"touches, each on its own, one file per command. CI on the PR runs "
+        f"the full suite."
+    )