alissa-tools-github-devloop 0.8.52__py3-none-any.whl → 0.8.53__py3-none-any.whl

alissa/tools/github/devloop/hungtests.py

@@ -125,16 +125,23 @@ class TestTree:
     @property
     def session(self) -> "str | None":
         """The owning session's roster name (no `ali-` prefix), or None."""
-        name = self.tmux_session
-        if not name:
-            return None
-        return name[len(TMUX_PREFIX):] if name.startswith(TMUX_PREFIX) else name
+        return roster_name(self.tmux_session)
 
     @property
     def command(self) -> str:
         return " ".join(self.root.argv)
 
 
+def roster_name(tmux_session: "str | None") -> "str | None":
+    """A seat env's `ALISSA_TMUX_SESSION` as the roster's name (no `ali-`
+    prefix), or None when the env names no session."""
+    if not tmux_session:
+        return None
+    if tmux_session.startswith(TMUX_PREFIX):
+        return tmux_session[len(TMUX_PREFIX):] or None
+    return tmux_session
+
+
 def _clock_ticks() -> int:
     try:
         return int(os.sysconf("SC_CLK_TCK")) or 100
@@ -424,6 +431,19 @@ def kill_tree(
     if current is None or current.start != tree.root.start:
         return 0, 0
     members = [m for m in descendants(fresh, tree.root.pid) if m.alive]
+    return kill_procs(members, root=root, grace=grace)
+
+
+def kill_procs(
+    members: "Sequence[Proc]",
+    *,
+    root: "Path | None" = None,
+    grace: float = KILL_GRACE_SECONDS,
+) -> "tuple[int, int]":
+    """TERM every member still alive under its recorded start time, wait up
+    to `grace` seconds, then KILL the survivors. Returns (termed, killed).
+    A member whose pid now belongs to another process is never signalled."""
+    members = [m for m in members if _still_alive(m, root)]
     for member in members:
         send_signal(member.pid, signal.SIGTERM)
     deadline = monotonic() + grace
@@ -479,3 +499,147 @@ def hung_message(tree: TestTree, minutes: int, reason: str) -> str:
         f"touches, each on its own, one file per command. CI on the PR runs "
         f"the full suite."
     )
+
+
+# -- the dead-session process reaper (issue #186) -----------------------------
+#
+# 2026-10-19: a crashed fix session's tmux pane sat at bash for 17 h while the
+# seat also carried orphan test runs of sessions long finished and ~20 idle
+# Claude Code executor helpers (`node /tmp/.tmpXXXX/local.cjs --ipc-path
+# ….executor.sock`, ~115 MB each). What follows finds what a DEAD session left
+# running; the policy -- which sessions count as dead, and when to sweep --
+# lives in loop.py.
+
+# The executor helper's script and the socket argument that identify it.
+EXECUTOR_SCRIPT = "local.cjs"
+EXECUTOR_SOCKET_SUFFIX = ".executor.sock"
+
+# Processes never signalled whatever their env says: the tmux server hosts
+# EVERY session on the seat (one `tmux new-session -d` server), so killing it
+# would take the live sessions down with the dead one.
+_NEVER_KILL = frozenset({"tmux", "tmux: server", "tini"})
+
+
+@dataclass(frozen=True)
+class Orphan:
+    """One process a dead session left behind: a member of a process tree
+    rooted in a task workspace (`workspace` set), or a parentless executor
+    helper (`workspace` None). `session` is the roster name its env names."""
+
+    proc: Proc
+    session: "str | None"
+    workspace: "TaskWorkspace | None" = None
+
+    @property
+    def command(self) -> str:
+        return " ".join(self.proc.argv)
+
+
+def is_executor(argv: "Sequence[str]") -> bool:
+    """Is this Claude Code's executor helper: a `local.cjs` script given an
+    `….executor.sock` IPC path. Exact tokens, like `is_test_runner`."""
+    return (
+        any(os.path.basename(arg) == EXECUTOR_SCRIPT for arg in argv)
+        and any(arg.endswith(EXECUTOR_SOCKET_SUFFIX) for arg in argv)
+    )
+
+
+def protected_pids(
+    procs: "Mapping[int, Proc]", self_pid: "int | None" = None
+) -> "set[int]":
+    """This process, every ancestor of it, and pid 1: the reaper never
+    signals itself or the chain that started it."""
+    me = os.getpid() if self_pid is None else self_pid
+    protected = {1, me}
+    current = procs.get(me)
+    while current is not None and current.ppid not in protected:
+        protected.add(current.ppid)
+        current = procs.get(current.ppid)
+    return protected
+
+
+def _spared(proc: Proc, protected: "set[int]") -> bool:
+    if not proc.alive or proc.pid in protected:
+        return True
+    head = os.path.basename(proc.argv[0]) if proc.argv else ""
+    return head in _NEVER_KILL or head.startswith("tmux")
+
+
+def find_session_orphans(
+    procs: "Mapping[int, Proc]",
+    hubs: "Sequence[Path]",
+    is_dead: "Callable[[str], bool]",
+    root: "Path | None" = None,
+    self_pid: "int | None" = None,
+) -> "list[Orphan]":
+    """Every process tree rooted in a task workspace of `hubs` whose owner
+    session `is_dead` -- and nothing else.
+
+    A ROOT is a process whose cwd or a path argument sits inside a task
+    workspace (`workspace_of`, the janitor's grammar: `main/`, `.source/` and
+    the hub itself never qualify) AND whose env names a session `is_dead`
+    answers True for. A process naming no session is never a root: it is not
+    a worker's (the seat exports `ALISSA_TMUX_SESSION` to every process a
+    session starts), so an operator's shell parked in a worktree is safe.
+    Below a root, every descendant goes too -- a test worker that moved its
+    cwd to `/tmp`, a child an env-scrubbing helper started -- unless its env
+    names a session that is NOT dead. The tmux server, the reaper itself and
+    its ancestors are never returned."""
+    resolved = []
+    for hub in hubs:
+        try:
+            resolved.append(Path(hub).resolve())
+        except OSError:
+            continue
+    protected = protected_pids(procs, self_pid)
+    sessions: "dict[int, str | None]" = {}
+
+    def session_of(pid: int) -> "str | None":
+        if pid not in sessions:
+            sessions[pid] = roster_name(session_env(pid, root)[0])
+        return sessions[pid]
+
+    found: "dict[int, Orphan]" = {}
+    for proc in sorted(procs.values(), key=lambda p: p.pid):
+        if proc.pid in found or _spared(proc, protected):
+            continue
+        workspace = workspace_of(proc, resolved)
+        if workspace is None:
+            continue
+        session = session_of(proc.pid)
+        if session is None or not is_dead(session):
+            continue
+        for member in descendants(procs, proc.pid):
+            if member.pid in found or _spared(member, protected):
+                continue
+            owner = session_of(member.pid)
+            if owner is not None and not is_dead(owner):
+                continue
+            found[member.pid] = Orphan(
+                proc=member, session=owner or session, workspace=workspace
+            )
+    return sorted(found.values(), key=lambda o: o.proc.pid)
+
+
+def find_parentless_executors(
+    procs: "Mapping[int, Proc]",
+    is_alive: "Callable[[str], bool]",
+    root: "Path | None" = None,
+    self_pid: "int | None" = None,
+) -> "list[Orphan]":
+    """Every executor helper (`is_executor`) whose parent is gone: re-parented
+    to init (pid 1) or to a pid the table no longer holds. One whose env names
+    a session `is_alive` answers True for is spared -- it may still be that
+    session's."""
+    protected = protected_pids(procs, self_pid)
+    found = []
+    for proc in sorted(procs.values(), key=lambda p: p.pid):
+        if _spared(proc, protected) or not is_executor(proc.argv):
+            continue
+        if proc.ppid != 1 and proc.ppid in procs:
+            continue
+        session = roster_name(session_env(proc.pid, root)[0])
+        if session is not None and is_alive(session):
+            continue
+        found.append(Orphan(proc=proc, session=session))
+    return found

alissa/tools/github/devloop/loop.py

@@ -449,6 +449,7 @@ from __future__ import annotations
 
 import calendar
 import dataclasses
+import itertools
 import json
 import logging
 import os
@@ -1851,6 +1852,14 @@ FIX_STALLED_COMMENT = (
     "close the PR to park the work."
 )
 
+# The tail every FIX STALLED ping carries (issue #186): what the session's
+# pane runs right now -- a bare shell means its claude is gone -- and the
+# promise that the ping repeats while the stall persists.
+FIX_STALLED_TAIL = (
+    " Pane foreground command: {pane}. This ping repeats every "
+    "{repeat_hours} h while the stall persists{ordinal}."
+)
+
 # The ALIVE twin of FIX_STALLED_COMMENT (issue #142): the fix edge no longer
 # respawns a stale round on the timer alone, so a session that is LISTED but
 # has posted no triage reply defers -- floored by this once-per-episode ping
@@ -2649,6 +2658,26 @@ ESCALATION_FIRST_RUN_DIALOG = "first-run-dialog"
 WEDGE_CLAUDE_EXITED = "wedged:claude-exited"
 ESCALATION_CLAUDE_EXITED = "claude-exited"
 
+# The fix edge's claude-exited episode (issue #186): the same wedge read on a
+# stale FIX round (fix_claude_exited_kind keys it per round and attempt). The
+# `fix-` prefix keeps it a PR-anchored kind for the loop-events emitter.
+ESCALATION_FIX_CLAUDE_EXITED = "fix-claude-exited"
+
+# How many consecutive polls a fix session's pane must read a bare shell
+# (`SHELL_COMMANDS`) before the fix edge reads the session DEAD (issue #186).
+# Two, not one: a single read can catch the pane between processes, and the
+# respawn it licenses lands on the same branch. A poll is one pass's roster
+# (`SessionLiveness.seq`), so re-reads inside one pass never count twice.
+SHELL_PANE_DEAD_POLLS = 2
+
+# The wording of the PR note a dead-pane restart leaves (issue #186).
+EXITED_RESTART_NOTE = "the previous session exited unexpectedly"
+
+# How often a FIX STALLED ping repeats while the stall persists (issue #186):
+# once per episode let a 17 h stall page the operator exactly once. See
+# stall_repeat_kind.
+FIX_STALLED_REPEAT_SECONDS = 2 * 60 * 60
+
 # The maintenance edge's cap-out kind prefix. The `maintain-` prefix keeps
 # rows from ever colliding with the issue edge's `cap` when an issue and a
 # PR share a number; the epoch suffix is maintain_cap_kind's job below.
@@ -3047,6 +3076,24 @@ def claude_exited_kind(attempt: int) -> str:
     return f"{ESCALATION_CLAUDE_EXITED}:a{attempt}"
 
 
+def fix_claude_exited_kind(round_: int, attempt: int) -> str:
+    """The ledger kind for ONE fix-round claude-exited episode (issue #186),
+    keyed per round AND attempt for fix_stalled_kind's episode reasoning."""
+    return f"{ESCALATION_FIX_CLAUDE_EXITED}:r{round_}a{attempt}"
+
+
+def stall_repeat_kind(kind: str, age: float, stalled_after: float) -> str:
+    """The ledger kind of the FIX STALLED ping due at `age` (issue #186).
+
+    The first ping of an episode keeps the episode's own kind (`kind`), so
+    every existing row, reader and test reads it unchanged; each further
+    FIX_STALLED_REPEAT_SECONDS past the stall floor is its own row,
+    `<kind>@<n>`. One row per window is the dedupe: a ping repeats every
+    2 h while the stall persists, and never twice inside one window."""
+    repeat = int(max(0.0, age - stalled_after) // FIX_STALLED_REPEAT_SECONDS)
+    return kind if repeat == 0 else f"{kind}@{repeat}"
+
+
 def deferral_activity_kind(session: str) -> str:
     """The activity-log kind that dedupes ONE capacity-deferral episode's line.
 
@@ -3420,6 +3467,14 @@ LIVE_STALLED_STATUS = {
 }
 
 
+# One number per POLL roster built (issue #186): a pass builds ONE, so its
+# `seq` is what "consecutive polls" counts in, without the edges having to be
+# told which pass they run in. One-off listings (`poll=False`: the resume
+# re-check, the process reaper's fresh listing) take no number, so two passes
+# in a row always hold adjacent numbers.
+_ROSTER_SEQ = itertools.count(1)
+
+
 class SessionLiveness:
     """The pass's tmux roster -- the oracle behind `session_liveness`.
 
@@ -3449,8 +3504,10 @@ class SessionLiveness:
         self,
         sessions: "list[dict] | None" = None,
         reaped: "set[str] | None" = None,
+        poll: bool = True,
     ):
         self.listed = sessions is not None
+        self.seq = next(_ROSTER_SEQ) if poll else 0
         # The raw rows, kept for the fleet-vitals snapshot (issue #116) so
         # the pass's ONE listing is also the roster it reports -- minus the
         # names the sweep killed, which are not sessions any more. None when
@@ -4556,6 +4613,13 @@ class DevWatcher:
         self._hung_watch: "hungtests.CpuWatch | None" = None
         self._hung_tests_at: "float | None" = None
         self._hung_reaped: "set[tuple[int, int]]" = set()
+        # The dead-pane read (issue #186): per session, the roster seq of the
+        # last poll its pane read a bare shell and how many polls in a row
+        # did. Any other read drops the entry; a poll that skips the session
+        # breaks the run, and poll_once prunes entries no poll can extend.
+        self._shell_streaks: "dict[str, tuple[int, int]]" = {}
+        # The process reaper's own clock (janitor cadence, issue #186).
+        self._orphans_at: "float | None" = None
 
     # -- per-issue decision ------------------------------------------------
 
@@ -5734,7 +5798,7 @@ class DevWatcher:
                 exc,
             )
             return False
-        if SessionLiveness(sessions).of(session) is Liveness.DEAD:
+        if SessionLiveness(sessions, poll=False).of(session) is Liveness.DEAD:
             return True
         log.info(
             "resume: %s is back in the session listing — standing the resume "
@@ -5883,10 +5947,10 @@ class DevWatcher:
         Dry-run classifies and logs but kills nothing and answers False
         (the defer holds); so does a kill that fails, retried next poll.
 
-        Not done here: the process groups the session's claude started
-        that outlived it (a test run re-parented to init). Nothing on the
-        daemon's side ties such a group to a session once its parent is
-        gone, so they are left to the operator (see the README)."""
+        Once the kill lands, the process trees the session left running in
+        its task workspace are reaped (`reap_session_processes`, issue
+        #186): the seat env every one of them inherited names the session,
+        however they were re-parented."""
         command = self.alissa.pane_command(session)
         if command not in SHELL_COMMANDS:
             return False, None
@@ -5970,6 +6034,7 @@ class DevWatcher:
                 WEDGE_CLAUDE_EXITED, session, exc,
             )
             return False, None
+        self.reap_session_processes(session)
         if not self.state.activity_logged(issue.full_name, issue.number, kind):
             landed = self._append_activity(
                 issue.owner,
@@ -5988,6 +6053,152 @@ class DevWatcher:
                 self.state.record_activity(issue.full_name, issue.number, kind)
         return True, resume_pr
 
+    def _shell_pane(
+        self, session: str, roster: SessionLiveness
+    ) -> "tuple[str, int]":
+        """The session's pane foreground command, and for how many polls in
+        a row it has read a bare shell (`SHELL_COMMANDS`) -- 0 for anything
+        else, "" (tmux could not answer) included, which is absence of
+        evidence and resets nothing it did not see (issue #186).
+
+        A poll is one roster (`roster.seq`): two reads inside one pass count
+        once, so "2 polls" always means two passes, never one pass that
+        happened to ask twice. And the polls must be ADJACENT: a pass that
+        never read this pane (round not stale yet, PR not in the search, edge
+        off) breaks the run, so two momentary shell reads hours apart never
+        join into a DEAD read."""
+        command = self.alissa.pane_command(session)
+        seen_at, polls = self._shell_streaks.get(session, (None, 0))
+        if command not in SHELL_COMMANDS:
+            if not command and seen_at in (roster.seq - 1, roster.seq):
+                # Polled but unreadable: carries the run, adds nothing.
+                self._shell_streaks[session] = (roster.seq, polls)
+            else:
+                self._shell_streaks.pop(session, None)
+            return command, 0
+        if seen_at == roster.seq - 1:
+            polls += 1
+        elif seen_at != roster.seq:
+            polls = 1
+        self._shell_streaks[session] = (roster.seq, polls)
+        return command, polls
+
+    def _prune_shell_streaks(self, roster: SessionLiveness) -> None:
+        """Drop the dead-pane runs this pass's poll can no longer extend --
+        last read before the previous poll -- so sessions that vanished
+        without a kill do not pile up over a long-lived daemon."""
+        for session, (seen_at, _) in list(self._shell_streaks.items()):
+            if seen_at < roster.seq - 1:
+                del self._shell_streaks[session]
+
+    def _fix_claude_exited(
+        self,
+        pr: PullRequest,
+        round_: int,
+        attempts: int,
+        session: str,
+        command: str,
+        polls: int,
+        *,
+        author_activity: bool,
+    ) -> bool:
+        """Issue #186: the fix edge's read of a stale round whose session is
+        LISTED but whose pane is a bare shell -- True once the session is
+        killed and the caller may read it DEAD.
+
+        The 2026-10-19 incident (studio PR #2518): a fix session's claude
+        exited mid-tool-call, the pane fell back to bash, and the round
+        logged "reads alive — a live session owns the round" every poll for
+        17 h. Now SHELL_PANE_DEAD_POLLS consecutive shell reads make the
+        session dead: it is killed (`alissa tmux kill`, the row's own
+        session, never a sweep), the processes it left in its workspace are
+        reaped, and one activity line on the PR says the previous session
+        exited unexpectedly and what the round does next -- which the
+        existing stale-attempt matrix decides once the caller reads DEAD: a
+        respawn without triage replies, a resume behind them (when
+        resume_enabled), the cap-out page when attempt_cap is spent.
+
+        Under the poll threshold, for a usage-limited session (never killed,
+        issue #144), under dry-run and when the kill fails (retried next
+        poll), the answer is False and every defer holds."""
+        if self.state.session_limited(session):
+            return False
+        if polls < SHELL_PANE_DEAD_POLLS:
+            log.info(
+                "%s: fix session %s (%s round %d attempt %d) reads alive but "
+                "its pane is at the shell (%s), poll %d of %d — read dead "
+                "only if it still is next poll",
+                WEDGE_CLAUDE_EXITED, session, pr.pr_slug, round_, attempts,
+                command, polls, SHELL_PANE_DEAD_POLLS,
+            )
+            return False
+        kind = fix_claude_exited_kind(round_, attempts)
+        billed = self.state.fix_attempt_count(
+            pr.full_name, pr.number, round_, billable=True
+        )
+        if author_activity and not self.config.resume_enabled:
+            outcome = (
+                "the round keeps its defer (resume_enabled is off), now "
+                "reading dead"
+            )
+        elif billed >= self.config.attempt_cap:
+            outcome = (
+                f"the round has spent attempt_cap {self.config.attempt_cap}, "
+                f"so the operator is paged instead of a restart"
+            )
+        else:
+            outcome = (
+                f"restarting the round as attempt {attempts + 1}"
+                + (" (resuming behind its triage replies)"
+                   if author_activity else "")
+            )
+        if self.state.escalated(pr.full_name, pr.number, kind):
+            log.info(
+                "%s: fix session %s (%s round %d attempt %d) still has no "
+                "claude in its pane — retrying the kill",
+                WEDGE_CLAUDE_EXITED, session, pr.pr_slug, round_, attempts,
+            )
+        else:
+            log.warning(
+                "%s: fix session %s (%s round %d attempt %d) reads alive but "
+                "its pane has been at the shell (%s) for %d polls — its "
+                "claude exited. Killing it; %s",
+                WEDGE_CLAUDE_EXITED, session, pr.pr_slug, round_, attempts,
+                command, polls, outcome,
+            )
+            self.state.record_escalation(pr.full_name, pr.number, kind)
+        if self.config.dry_run:
+            log.info("[dry-run] would kill %s (claude exited); %s", session, outcome)
+            return False
+        try:
+            self.alissa.kill_session(session)
+        except CommandError as exc:
+            log.warning(
+                "%s: could not kill %s (%s) — kills are best-effort; the "
+                "round keeps its defer and the kill is retried next poll",
+                WEDGE_CLAUDE_EXITED, session, exc,
+            )
+            return False
+        self._shell_streaks.pop(session, None)
+        self.reap_session_processes(session)
+        if not self.state.activity_logged(pr.full_name, pr.number, kind):
+            landed = self._append_activity(
+                pr.owner,
+                pr.repo,
+                pr.number,
+                self._activity_line(
+                    session,
+                    "fix edge",
+                    f"{ACTIVITY_WEDGE_KILLED} — {WEDGE_CLAUDE_EXITED}: "
+                    f"{EXITED_RESTART_NOTE} (its pane was at `{command}` "
+                    f"for {polls} polls); killed, {outcome}",
+                ),
+                pr.pr_slug,
+            )
+            if landed:
+                self.state.record_activity(pr.full_name, pr.number, kind)
+        return True
+
     def _wedged_on_first_run_dialog(
         self, issue: Issue, attempts: int, session: str
     ) -> bool:
@@ -6749,6 +6960,7 @@ class DevWatcher:
         # says CONFIRMED dead.
         resume_dead_session: "str | None" = None
         confirmed_dead_session: "str | None" = None
+        exited_session: "str | None" = None
         if age is not None:
             author_activity = any(
                 r.author == pr.author and r.submitted_at > newest.submitted_at
@@ -6758,17 +6970,34 @@ class DevWatcher:
                 pr.full_name, number, round_
             )
             liveness = self.session_liveness(row_session, roster)
+            # Issue #186: LISTED is not alive. A session whose pane has read a
+            # bare shell for SHELL_PANE_DEAD_POLLS polls lost its claude; once
+            # it is killed it reads DEAD, and the matrix below respawns or
+            # resumes the round exactly as for a session gone from the list.
+            pane = ""
+            if liveness is Liveness.ALIVE and row_session and roster is not None:
+                pane, polls = self._shell_pane(row_session, roster)
+                if polls and self._fix_claude_exited(
+                    pr, round_, attempts, row_session, pane, polls,
+                    author_activity=author_activity,
+                ):
+                    liveness = Liveness.DEAD
+                    exited_session = row_session
+            stalled_after = (
+                STALLED_DEFER_MULTIPLE * self.config.stale_minutes * 60
+            )
             if not author_activity:
                 if liveness is Liveness.ALIVE:
-                    stalled_after = (
-                        STALLED_DEFER_MULTIPLE * self.config.stale_minutes * 60
+                    live_kind = stall_repeat_kind(
+                        fix_live_stalled_kind(round_, attempts), age,
+                        stalled_after,
                     )
                     if age >= stalled_after and not self.state.escalated(
-                        pr.full_name, number,
-                        fix_live_stalled_kind(round_, attempts),
+                        pr.full_name, number, live_kind,
                     ):
                         self._escalate_fix_live_stalled(
-                            pr, round_, attempts, age, row_session or ""
+                            pr, round_, attempts, age, row_session or "",
+                            kind=live_kind, pane=pane,
                         )
                     fix_live_timer = self._stall_timer(
                         pr.full_name, number,
@@ -6809,13 +7038,28 @@ class DevWatcher:
                     # resume is an ORDINARY attempt of this round.
                     resume_dead_session = row_session
                 else:
-                    stalled_after = (
-                        STALLED_DEFER_MULTIPLE * self.config.stale_minutes * 60
+                    # Repeats only while the session may still be there: a
+                    # DEAD one (resume off) waits on a human with nothing
+                    # left to read, and its seat may park after the first
+                    # ping -- a repeat would then be the very page a wake
+                    # must never manufacture (parking §2.9).
+                    stall_kind = (
+                        fix_stalled_kind(round_, attempts)
+                        if liveness is Liveness.DEAD
+                        else stall_repeat_kind(
+                            fix_stalled_kind(round_, attempts), age,
+                            stalled_after,
+                        )
                     )
                     if age >= stalled_after and not self.state.escalated(
-                        pr.full_name, number, fix_stalled_kind(round_, attempts)
+                        pr.full_name, number, stall_kind
                     ):
-                        self._escalate_fix_stalled(pr, round_, attempts, age)
+                        if not pane and liveness is Liveness.ALIVE and row_session:
+                            pane = self.alissa.pane_command(row_session)
+                        self._escalate_fix_stalled(
+                            pr, round_, attempts, age, kind=stall_kind,
+                            pane=pane, session=row_session,
+                        )
                     # Parking (issue #153), the issue edge's rule: owed until
                     # the stall ping is out, and past it while the session
                     # might still be there; a dead one waits on a human.
@@ -6877,14 +7121,25 @@ class DevWatcher:
         if age is not None and resume_dead_session is not None:
             log.warning(
                 "%s fix attempt %d (round %d) has been in flight %.0f min "
-                "behind its own triage replies and its session %s is absent "
-                "from the tmux listing (CONFIRMED dead, not presumed) — "
+                "behind its own triage replies and its session %s %s — "
                 "re-enqueuing as attempt %d to RESUME the round",
                 pr.pr_slug,
                 attempts,
                 round_,
                 age / 60,
                 resume_dead_session,
+                "exited (pane back at the shell; killed)"
+                if exited_session is not None
+                else "is absent from the tmux listing (CONFIRMED dead, not "
+                "presumed)",
+                attempts + 1,
+            )
+        elif age is not None and exited_session is not None:
+            log.warning(
+                "%s fix attempt %d (round %d) has been in flight %.0f min "
+                "with no re-request and its session %s exited (pane back at "
+                "the shell; killed) — restarting the round as attempt %d",
+                pr.pr_slug, attempts, round_, age / 60, exited_session,
                 attempts + 1,
             )
         elif age is not None:
@@ -6912,6 +7167,7 @@ class DevWatcher:
             reenqueued=age is not None,
             dead_session=resume_dead_session,
             confirmed_dead=confirmed_dead_session,
+            exited=exited_session is not None,
         )
 
     def _reviewer_reviews(
@@ -8343,8 +8599,29 @@ class DevWatcher:
             )
         self.state.record_escalation(pr.full_name, pr.number, fix_cap_kind(round_))
 
+    @staticmethod
+    def _fix_stalled_tail(kind: str, pane: str, session: "str | None") -> str:
+        """FIX_STALLED_TAIL for one ping: `pane` as read, or why there is no
+        reading; and which repeat this is (`kind`'s `@<n>` suffix)."""
+        if pane:
+            shown = f"`{pane}`"
+            if pane in SHELL_COMMANDS:
+                shown += " (a bare shell: the session's claude has exited)"
+        elif session:
+            shown = f"unreadable for `{session}`"
+        else:
+            shown = "unknown (no session recorded for this attempt)"
+        _, sep, repeat = kind.rpartition("@")
+        ordinal = f" (repeat {repeat})" if sep and repeat.isdigit() else ""
+        return FIX_STALLED_TAIL.format(
+            pane=shown, repeat_hours=FIX_STALLED_REPEAT_SECONDS // 3600,
+            ordinal=ordinal,
+        )
+
     def _escalate_fix_stalled(
-        self, pr: PullRequest, round_: int, attempts: int, age: float
+        self, pr: PullRequest, round_: int, attempts: int, age: float,
+        *, kind: "str | None" = None, pane: str = "",
+        session: "str | None" = None,
     ) -> None:
         """Operator ping when the author-activity deferral itself runs long
         (_escalate_stalled's shape): the author's own triage replies are the
@@ -8357,17 +8634,20 @@ class DevWatcher:
         exactly-once delivery contract as _escalate_stalled -- a transient
         failure retries next poll. The decision stays IN_FLIGHT either way
         -- this comments, it never respawns."""
+        kind = kind or fix_stalled_kind(round_, attempts)
         body = FIX_STALLED_COMMENT.format(
             attempts=attempts, round=round_, minutes=int(age / 60)
-        )
+        ) + self._fix_stalled_tail(kind, pane, session)
         log.warning(
             "FIX STALLED %s fix attempt %d (round %d) has been deferred "
-            "%.0f min behind the author's own triage replies — escalating "
-            "to operator (once per episode)",
+            "%.0f min behind the author's own triage replies (pane: %s) — "
+            "escalating to operator (repeats every %d h while it persists)",
             pr.pr_slug,
             attempts,
             round_,
             age / 60,
+            pane or "unread",
+            FIX_STALLED_REPEAT_SECONDS // 3600,
         )
 
         if self.config.dry_run:
@@ -8384,28 +8664,29 @@ class DevWatcher:
                 exc,
             )
             return
-        self.state.record_escalation(
-            pr.full_name, pr.number, fix_stalled_kind(round_, attempts)
-        )
+        self.state.record_escalation(pr.full_name, pr.number, kind)
 
     def _escalate_fix_live_stalled(
         self, pr: PullRequest, round_: int, attempts: int, age: float,
-        session: str,
+        session: str, *, kind: "str | None" = None, pane: str = "",
     ) -> None:
         """_escalate_fix_stalled's ALIVE twin (issue #142): the liveness
         guard on the no-triage-reply branch defers while the row's session is
         listed, and this ping is its floor. Same delivery contract: the row
         is recorded only AFTER the comment posts; the decision stays
         IN_FLIGHT either way."""
+        kind = kind or fix_live_stalled_kind(round_, attempts)
         body = FIX_LIVE_STALLED_COMMENT.format(
             attempts=attempts, round=round_, minutes=int(age / 60),
             session=session,
-        )
+        ) + self._fix_stalled_tail(kind, pane, session)
         log.warning(
             "FIX STALLED %s fix attempt %d (round %d) has been deferred "
             "%.0f min behind its still-listed session %s with no triage "
-            "reply — escalating to operator (once per episode)",
+            "reply (pane: %s) — escalating to operator (repeats every %d h "
+            "while it persists)",
             pr.pr_slug, attempts, round_, age / 60, session,
+            pane or "unread", FIX_STALLED_REPEAT_SECONDS // 3600,
         )
         if self.config.dry_run:
             log.info("[dry-run] would comment on %s:\n%s", pr.pr_slug, body)
@@ -8419,9 +8700,7 @@ class DevWatcher:
                 pr.pr_slug, exc,
             )
             return
-        self.state.record_escalation(
-            pr.full_name, pr.number, fix_live_stalled_kind(round_, attempts)
-        )
+        self.state.record_escalation(pr.full_name, pr.number, kind)
 
     def _hand_over_round(
         self,
@@ -8743,6 +9022,7 @@ class DevWatcher:
         reenqueued: bool = False,
         dead_session: "str | None" = None,
         confirmed_dead: "str | None" = None,
+        exited: bool = False,
     ) -> Decision:
         """The fix edge's spawn, behind the drain gate (`_drain_gated`)."""
         return self._drain_gated(
@@ -8750,7 +9030,7 @@ class DevWatcher:
             lambda: self._spawn_fix_claimed(
                 pr, round_, attempt, review, budget,
                 reenqueued=reenqueued, dead_session=dead_session,
-                confirmed_dead=confirmed_dead,
+                confirmed_dead=confirmed_dead, exited=exited,
             ),
             round_=round_,
         )
@@ -8766,6 +9046,7 @@ class DevWatcher:
         reenqueued: bool = False,
         dead_session: "str | None" = None,
         confirmed_dead: "str | None" = None,
+        exited: bool = False,
     ) -> Decision:
         skipped = self._foreign_head_skip(pr, attempt)
         if skipped is not None:
@@ -8886,7 +9167,14 @@ class DevWatcher:
         #
         # The session name carries the round and attempt; the context marks a
         # retry, and names the confirmation when a resume is what unblocked it.
-        if dead_session is not None:
+        if exited:
+            # Issue #186: the session was listed, but its claude was gone.
+            context = (
+                f"{'resumed' if dead_session is not None else 're-enqueued'} "
+                f"— {EXITED_RESTART_NOTE} (its pane was back at the shell; "
+                f"killed); restarting the round"
+            )
+        elif dead_session is not None:
             context = (
                 "resumed — previous session confirmed dead (absent from the "
                 "tmux listing); finishing the round on the same branch"
@@ -12095,6 +12383,146 @@ class DevWatcher:
         if hungtests.message_session(session_id, text):
             log.info("hung tests: told %s its test run was stopped", tree.session)
 
+    # -- the dead-session process reaper (issue #186) ----------------------
+
+    def _watched_hubs(self) -> "list[Path]":
+        hubs = []
+        for owner, repo in self._watched_repos():
+            hub = self.config.hub_for(owner, repo)
+            if hub.is_dir():
+                hubs.append(hub.resolve())
+        return hubs
+
+    def reap_session_processes(self, session: str) -> int:
+        """Reap what ONE session the daemon just killed left running in its
+        task workspace: every process tree rooted there whose seat env names
+        `session` (`hungtests.find_session_orphans`). Returns how many
+        processes were signalled. Never raises."""
+        try:
+            hubs = self._watched_hubs()
+            if not hubs:
+                return 0
+            orphans = hungtests.find_session_orphans(
+                hungtests.snapshot(), hubs, lambda name: name == session
+            )
+            return self._reap_orphans(orphans, f"its session {session} was killed")
+        except (CommandError, OSError) as exc:
+            log.error("process reaper: %s could not be reaped (%s)", session, exc)
+            return 0
+
+    def run_process_reaper(self, roster: "SessionLiveness | None") -> int:
+        """Reap the processes dead sessions left behind (issue #186), on the
+        janitor's cadence (`janitor_interval_minutes`, its own clock).
+
+        2026-10-19: orphan `npm test` runs of sessions whose PRs had merged
+        17-18 h earlier, and ~20 idle executor helpers (`node /tmp/.tmpXXXX/
+        local.cjs --ipc-path ….executor.sock`, ~115 MB each), on a seat at
+        load 16-25. Two sweeps, both needing a SUCCESSFUL listing (no
+        listing is no evidence, and nothing is touched):
+
+        * a process tree rooted in a watched hub's task workspace whose seat
+          env names a session the listing does not carry -- killed, gone, or
+          reaped after its lane's PR merged or closed. A tree of a session
+          still listed is never touched, whatever its lane's state;
+        * an executor helper whose parent is gone, unless its env names a
+          listed session.
+
+        The pass's roster is listed at the START of the pass, and the edges
+        enqueue sessions after it, so on its own it would read a session
+        spawned this pass DEAD and kill its fresh claude. The sweep
+        therefore lists again right AFTER the /proc snapshot: a session that
+        owns a snapshotted process existed before the snapshot, so it is in
+        that listing unless it really ended. A name is DEAD only when BOTH
+        listings lack it; if the fresh listing fails, nothing is touched.
+        The fresh listing is taken only when the pass roster finds something
+        to reap, so a quiet seat keeps its one listing per pass.
+
+        Everything reaped is logged. Nothing in here may take a poll down."""
+        if roster is None or not roster.listed:
+            return 0
+        interval = self.config.janitor_interval_seconds
+        now = time.monotonic()
+        if (
+            self._orphans_at is not None
+            and interval > 0
+            and now - self._orphans_at < interval
+        ):
+            return 0
+        self._orphans_at = now
+        try:
+            procs = hungtests.snapshot()
+            hubs = self._watched_hubs()
+
+            def sweep(
+                dead: "Callable[[str], bool]",
+            ) -> "tuple[list[hungtests.Orphan], list[hungtests.Orphan]]":
+                trees = (
+                    hungtests.find_session_orphans(procs, hubs, dead)
+                    if hubs else []
+                )
+                executors = hungtests.find_parentless_executors(
+                    procs, lambda name: not dead(name)
+                )
+                return trees, executors
+
+            trees, executors = sweep(
+                lambda name: roster.of(name) is Liveness.DEAD
+            )
+            if not trees and not executors:
+                return 0
+            # Listed AFTER the snapshot, never before (see the docstring).
+            fresh = SessionLiveness(self.alissa.list_sessions(), poll=False)
+            trees, executors = sweep(
+                lambda name: roster.of(name) is Liveness.DEAD
+                and fresh.of(name) is Liveness.DEAD
+            )
+            return self._reap_orphans(
+                trees, "its session is gone"
+            ) + self._reap_orphans(executors, "its parent is gone")
+        except (CommandError, OSError) as exc:
+            log.error("process reaper: sweep failed (%s) — skipped this pass", exc)
+            return 0
+
+    def _reap_orphans(
+        self, orphans: "list[hungtests.Orphan]", reason: str
+    ) -> int:
+        """Log, then TERM (KILL after the grace) one batch of orphans;
+        one WARNING per workspace (or for the executors) naming every pid
+        and command. Returns how many processes were signalled."""
+        if not orphans:
+            return 0
+        groups: "dict[str, list[hungtests.Orphan]]" = {}
+        for orphan in orphans:
+            where = (
+                orphan.workspace.slug if orphan.workspace is not None
+                else "parentless executor helpers"
+            )
+            groups.setdefault(where, []).append(orphan)
+        for where, members in groups.items():
+            listing = "; ".join(
+                f"pid {o.proc.pid} `{o.command[:160]}`"
+                + (f" ({o.session})" if o.session else "")
+                for o in members
+            )
+            if self.config.dry_run:
+                log.info(
+                    "[dry-run] process reaper would stop %d process(es) in "
+                    "%s — %s: %s", len(members), where, reason, listing,
+                )
+            else:
+                log.warning(
+                    "process reaper: stopping %d process(es) in %s — %s: %s",
+                    len(members), where, reason, listing,
+                )
+        if self.config.dry_run:
+            return 0
+        termed, killed = hungtests.kill_procs([o.proc for o in orphans])
+        if termed:
+            log.info(
+                "process reaper: TERM to %d process(es), KILL to %d", termed, killed
+            )
+        return termed
+
     def _janitor_due(self) -> bool:
         """Has `janitor_interval_minutes` elapsed since the last expensive
         half? The first pass of a process always runs (nothing has been swept
@@ -12785,6 +13213,7 @@ class DevWatcher:
         included."""
         started = time.monotonic()
         sweep, budget, roster = self._sweep_and_budget()
+        self._prune_shell_streaks(roster)
 
         # The allowlist is re-derived BETWEEN the sweep and the searches: the
         # sweep is deliberately search-independent and must not wait on a feed
@@ -12833,6 +13262,10 @@ class DevWatcher:
         # worker's test run that stopped making CPU progress, or outlived its
         # lane, is stopped and its live session told once.
         self.run_hung_tests(roster)
+        # The process reaper (issue #186), on the same cadence: what dead
+        # sessions left running in their workspaces, and executor helpers
+        # whose parent is gone.
+        self.run_process_reaper(roster)
 
         counts = Counter(decision.action for _, decision in results)
         self._write_snapshot(

alissa/tools/github/devloop/loop_events.py

@@ -157,8 +157,10 @@ DEFER_DRAINED = "drained"  # the seat is drained for parking (issue #153)
 # added to the loop later reaches Studio without touching this table.
 _STALLED_RE = re.compile(r"^stalled:a(\d+)$")
 _LIVE_STALLED_RE = re.compile(r"^live-stalled:a(\d+)$")
-_FIX_STALLED_RE = re.compile(r"^fix-stalled:r(\d+)a(\d+)$")
-_FIX_LIVE_STALLED_RE = re.compile(r"^fix-live-stalled:r(\d+)a(\d+)$")
+# The fix stalls repeat every 2 h while they persist (issue #186): repeat n
+# is its own row, `<kind>@<n>`, enriched like the first.
+_FIX_STALLED_RE = re.compile(r"^fix-stalled:r(\d+)a(\d+)(?:@\d+)?$")
+_FIX_LIVE_STALLED_RE = re.compile(r"^fix-live-stalled:r(\d+)a(\d+)(?:@\d+)?$")
 _FIX_CAP_RE = re.compile(r"^fix-cap:r(\d+)$")
 _MAINTAIN_CAP_RE = re.compile(r"^maintain-cap(:e\d+)?$")
 _REVERT_REFUSED_RE = re.compile(r"^revert-refused:e(\d+)$")

alissa/tools/github/devloop/version

@@ -1 +1 @@
-0.8.52
+0.8.53

{alissa_tools_github_devloop-0.8.52.dist-info → alissa_tools_github_devloop-0.8.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alissa-tools-github-devloop
-Version: 0.8.52
+Version: 0.8.53
 Summary: ALISSA-TOOLS-GITHUB-DEVLOOP
 Home-page: https://alissa.app
 Author: Fahera

{alissa_tools_github_devloop-0.8.52.dist-info → alissa_tools_github_devloop-0.8.53.dist-info}/RECORD

@@ -7,16 +7,16 @@ alissa/tools/github/devloop/config.py,sha256=ZgGb7DQR3hOOfCXnRIjmG52m3A2SfQHFX94
 alissa/tools/github/devloop/fleet_vitals.py,sha256=5mWnRkJh8u9soSplud3UMxkelT6EqAiz-ivX8UmfIqk,25477
 alissa/tools/github/devloop/ghclient.py,sha256=tYUHwmoy9CBBt-jSOz9nuAl8lpMvm1LH4A-wHx25bss,71842
 alissa/tools/github/devloop/hubs.py,sha256=yjnsnSHLG7GNNP8X2hILeAMV6P5n4JGc_IaAxpSP_YQ,18974
-alissa/tools/github/devloop/hungtests.py,sha256=LKBLg0fmswEhGz6GcPWoOx1bDJf0NCBd0UGJLl0K6Yo,18185
+alissa/tools/github/devloop/hungtests.py,sha256=TozIDydUe_Kn1m5EtGeGtYt_BrLqGznrNjikLYhxH-Q,24621
 alissa/tools/github/devloop/integration_check.py,sha256=JkFwz3JqOQBIvzszmiql-IMVGSH3lDers6mJUHh9zMM,17928
 alissa/tools/github/devloop/janitor.py,sha256=R8REoHBeY2cfW7fYJVhKr-F5z9Mjlc-vmv-z4KtvBZc,27360
-alissa/tools/github/devloop/loop.py,sha256=JuT3cWOjOZAz0mA-gPdfl-YnepkmWSf0jVX6-N2SmWU,670260
-alissa/tools/github/devloop/loop_events.py,sha256=pKtBl1HK8gYPB2gKiI29xUkx-JuW2A48KwRlbdcf_lo,37942
+alissa/tools/github/devloop/loop.py,sha256=hhB92wgot5H2KTCg3gARWos1mDK-05GV8brr9_CyYH4,690963
+alissa/tools/github/devloop/loop_events.py,sha256=9b7G2ak69Zm-oc_3IK8PplgOEdBydfER_ZFjKlCJJ5A,38093
 alissa/tools/github/devloop/parking.py,sha256=7h3Gg2XQ0eqlvn0M6K6NeEg9xbqnlUPrZNs9gRofzxc,12410
 alissa/tools/github/devloop/proc.py,sha256=nzfsxJSHFWRQcrXmfv588KazFysv8dVQrhMPH1q5eRk,3134
 alissa/tools/github/devloop/prompts.py,sha256=VzN6ec5eF2wwg7QSCRg52WLcxo-tGbGRgQbc4HSahUc,55169
 alissa/tools/github/devloop/state.py,sha256=uVwhbkogUTjWXXFq-COKzUeYrpuGbHzi05CjfEzAtPA,88668
-alissa/tools/github/devloop/version,sha256=zTdwnMsnc43u41Sz30AsmePfwZ75js_gUSxIgwqVa64,7
+alissa/tools/github/devloop/version,sha256=vNneLCSZvtUhO_wU9JFzULL8TpYn9ulHY4YWNTYykZU,7
 alissa/tools/github/devloop/version.py,sha256=znuuOaitXGrBpbskNy78_rlVpFdtmKmk_3k1ACxZrl0,1418
 alissa/tools/github/devloop/webui/__init__.py,sha256=vdXYoyGYzD7BrJ5lV6XdH8Bpw_pej8-RK0hjfxop7-8,1513
 alissa/tools/github/devloop/webui/__main__.py,sha256=wORyW5AkgsniK0TZcrOrvm_P7Q4Mr-fH-4S6qj9XcgA,6137
@@ -25,10 +25,10 @@ alissa/tools/github/devloop/webui/page.py,sha256=4OD3e01ByvVM4Wo04AZyFuyOYTFEMT2
 alissa/tools/github/devloop/webui/server.py,sha256=3aGYGesjUSrPrgcpfy-oYDefBNKnAj3lGfRzwsRkv_I,27031
 alissa/tools/github/devloop/webui/sources.py,sha256=Uw_2B5QGSmW6f66WfVbQb6yLLYmSv6uuF7HNJ5OSr_I,39314
 alissa/tools/github/devloop/webui/sysinfo.py,sha256=ptlogH9alFv0vZ-pQfyUaA3w-K7LQS3tQHGv0d8Tdlk,9649
-alissa_tools_github_devloop-0.8.52.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-alissa_tools_github_devloop-0.8.52.dist-info/licenses/NOTICE,sha256=FKstNs3uw_tQod4_1XcFLD1FUTJQXf4FC0oXJAOA_HE,260
-alissa_tools_github_devloop-0.8.52.dist-info/METADATA,sha256=ubhTFTwht-kKRAONJl7o-c8qhMlZfuLcHIojF0OtlT4,15973
-alissa_tools_github_devloop-0.8.52.dist-info/WHEEL,sha256=a-udL2m4V_6Yn8b-uKzWb12RjRxwwMo6ohYFM24tpaY,91
-alissa_tools_github_devloop-0.8.52.dist-info/entry_points.txt,sha256=8aC7Ajzx4bu9ZRR37a40IDfuStacvZP5KsAxY1dbFKU,145
-alissa_tools_github_devloop-0.8.52.dist-info/top_level.txt,sha256=DodjDg-l-TWQnlxG-Vuc3G5sB4O7cWGQwe6XrVx3u-A,7
-alissa_tools_github_devloop-0.8.52.dist-info/RECORD,,
+alissa_tools_github_devloop-0.8.53.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+alissa_tools_github_devloop-0.8.53.dist-info/licenses/NOTICE,sha256=FKstNs3uw_tQod4_1XcFLD1FUTJQXf4FC0oXJAOA_HE,260
+alissa_tools_github_devloop-0.8.53.dist-info/METADATA,sha256=VhWa2WDOnuqVugjFYmLjM-8FMYCtjViai7ECLPsW9qE,15973
+alissa_tools_github_devloop-0.8.53.dist-info/WHEEL,sha256=a-udL2m4V_6Yn8b-uKzWb12RjRxwwMo6ohYFM24tpaY,91
+alissa_tools_github_devloop-0.8.53.dist-info/entry_points.txt,sha256=8aC7Ajzx4bu9ZRR37a40IDfuStacvZP5KsAxY1dbFKU,145
+alissa_tools_github_devloop-0.8.53.dist-info/top_level.txt,sha256=DodjDg-l-TWQnlxG-Vuc3G5sB4O7cWGQwe6XrVx3u-A,7
+alissa_tools_github_devloop-0.8.53.dist-info/RECORD,,