alissa-tools-github-devloop 0.8.45__py3-none-any.whl → 0.8.47__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alissa/tools/github/devloop/config.py +41 -0
- alissa/tools/github/devloop/hungtests.py +481 -0
- alissa/tools/github/devloop/loop.py +220 -0
- alissa/tools/github/devloop/parking.py +1 -1
- alissa/tools/github/devloop/version +1 -1
- alissa/tools/github/devloop/webui/server.py +18 -4
- alissa/tools/github/devloop/webui/sources.py +12 -5
- {alissa_tools_github_devloop-0.8.45.dist-info → alissa_tools_github_devloop-0.8.47.dist-info}/METADATA +41 -2
- {alissa_tools_github_devloop-0.8.45.dist-info → alissa_tools_github_devloop-0.8.47.dist-info}/RECORD +14 -13
- {alissa_tools_github_devloop-0.8.45.dist-info → alissa_tools_github_devloop-0.8.47.dist-info}/WHEEL +0 -0
- {alissa_tools_github_devloop-0.8.45.dist-info → alissa_tools_github_devloop-0.8.47.dist-info}/entry_points.txt +0 -0
- {alissa_tools_github_devloop-0.8.45.dist-info → alissa_tools_github_devloop-0.8.47.dist-info}/licenses/LICENSE +0 -0
- {alissa_tools_github_devloop-0.8.45.dist-info → alissa_tools_github_devloop-0.8.47.dist-info}/licenses/NOTICE +0 -0
- {alissa_tools_github_devloop-0.8.45.dist-info → alissa_tools_github_devloop-0.8.47.dist-info}/top_level.txt +0 -0
|
@@ -132,6 +132,36 @@ ENV_REVERT_LABEL = "ALISSA_REVERT_LABEL"
|
|
|
132
132
|
# worker's worktree as part of the session; the daemon never executes it.
|
|
133
133
|
ENV_REGENERATE = "ALISSA_DEV_REGENERATE"
|
|
134
134
|
|
|
135
|
+
# The hung-test reaper's window (issue #174): minutes a worker's test-runner
|
|
136
|
+
# tree may sit at under `hungtests.MIN_PROGRESS_SECONDS` of CPU growth before
|
|
137
|
+
# the daemon kills it. Env-only, read straight from the environment like
|
|
138
|
+
# ENV_REGENERATE: not a config key, not a build ARG. Default 20; 0 turns the
|
|
139
|
+
# reaper off; blank falls through to the default; anything that is not a
|
|
140
|
+
# non-negative integer is a config error at boot, by name -- a typo silently
|
|
141
|
+
# read as "off" would leave a 55 h hung suite exactly where it was.
|
|
142
|
+
ENV_HUNG_TEST_MINUTES = "ALISSA_DEV_HUNG_TEST_MINUTES"
|
|
143
|
+
DEFAULT_HUNG_TEST_MINUTES = 20
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def parse_hung_test_minutes(raw: str) -> int:
|
|
147
|
+
"""The ALISSA_DEV_HUNG_TEST_MINUTES value as minutes (0 = off)."""
|
|
148
|
+
text = (raw or "").strip()
|
|
149
|
+
if not text:
|
|
150
|
+
return DEFAULT_HUNG_TEST_MINUTES
|
|
151
|
+
try:
|
|
152
|
+
minutes = int(text)
|
|
153
|
+
except ValueError:
|
|
154
|
+
raise ValueError(
|
|
155
|
+
f"{ENV_HUNG_TEST_MINUTES} must be a whole number of minutes "
|
|
156
|
+
f"(0 = off), got {raw!r}"
|
|
157
|
+
) from None
|
|
158
|
+
if minutes < 0:
|
|
159
|
+
raise ValueError(
|
|
160
|
+
f"{ENV_HUNG_TEST_MINUTES} must be >= 0 (0 = off), got {minutes}"
|
|
161
|
+
)
|
|
162
|
+
return minutes
|
|
163
|
+
|
|
164
|
+
|
|
135
165
|
# `owner/repo` for a regenerate entry: exactly one `/`, no whitespace, and no
|
|
136
166
|
# `:` (the separator that ends it). Shape only, like the actor-id check.
|
|
137
167
|
_REGENERATE_REPO_RE = re.compile(r"^[^\s/:]+/[^\s/:]+\Z")
|
|
@@ -857,6 +887,12 @@ class Config:
|
|
|
857
887
|
# Read through `regenerate_for(owner, repo)`, never executed here.
|
|
858
888
|
regenerate: tuple[RegenerateRule, ...] = ()
|
|
859
889
|
|
|
890
|
+
# The hung-test reaper's window in minutes (issue #174), from
|
|
891
|
+
# ALISSA_DEV_HUNG_TEST_MINUTES only (no config key, no CLI flag). A
|
|
892
|
+
# worker's `npm test` / `tsx --test` tree whose CPU grows by under 5 s
|
|
893
|
+
# across this window is killed; 0 turns the reaper off.
|
|
894
|
+
hung_test_minutes: int = DEFAULT_HUNG_TEST_MINUTES
|
|
895
|
+
|
|
860
896
|
dry_run: bool = False
|
|
861
897
|
|
|
862
898
|
# Derived at build time: `repos` casefolded for matching. GitHub names are
|
|
@@ -1118,6 +1154,10 @@ class Config:
|
|
|
1118
1154
|
# The regenerable-paths map (issue #138): env-only, validated here so
|
|
1119
1155
|
# a malformed entry fails the boot by name instead of a directive.
|
|
1120
1156
|
regenerate = parse_regenerate(env_map.get(ENV_REGENERATE, ""))
|
|
1157
|
+
# The hung-test window (issue #174): env-only, validated here.
|
|
1158
|
+
hung_test_minutes = parse_hung_test_minutes(
|
|
1159
|
+
env_map.get(ENV_HUNG_TEST_MINUTES, "")
|
|
1160
|
+
)
|
|
1121
1161
|
|
|
1122
1162
|
bow_owners = normalize_bow_owners(raw.get("bow_owners", cls.bow_owners))
|
|
1123
1163
|
|
|
@@ -1430,6 +1470,7 @@ class Config:
|
|
|
1430
1470
|
),
|
|
1431
1471
|
disk_warn_percent=warn_percent,
|
|
1432
1472
|
regenerate=regenerate,
|
|
1473
|
+
hung_test_minutes=hung_test_minutes,
|
|
1433
1474
|
dry_run=bool(raw.get("dry_run", cls.dry_run)),
|
|
1434
1475
|
)
|
|
1435
1476
|
|
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
"""The hung-test reaper's mechanics (issue #174).
|
|
2
|
+
|
|
3
|
+
On 2026-10-18 the studio devloop container ran at load ~15 while four worker
|
|
4
|
+
`npm test` runs sat hung, each at about 10 CPU-seconds in total -- one of them
|
|
5
|
+
for 55 hours, its lane long merged. Their test workers had died (esbuild's
|
|
6
|
+
"The service was stopped", EAGAIN with the box out of processes), the
|
|
7
|
+
`tsx --test` runner then waited forever, and the sessions waited on it in an
|
|
8
|
+
`until grep -q '^EXIT=' suite.log` loop. Nothing in the fleet noticed: a
|
|
9
|
+
session waiting on a log reads ALIVE to every check the daemon has.
|
|
10
|
+
|
|
11
|
+
What lives here is the janitor's split again -- only the MECHANICS: which
|
|
12
|
+
process trees are worker test runs (`find_test_trees`), whether one has made
|
|
13
|
+
CPU progress across a window (`CpuWatch`), how to stop one (`kill_tree`), and
|
|
14
|
+
how to tell its session (`message_session`). The POLICY -- which hung or
|
|
15
|
+
orphaned tree to kill, and when -- lives in `loop.py` next to the janitor
|
|
16
|
+
whose predicates it shares.
|
|
17
|
+
|
|
18
|
+
The properties everything here is written around:
|
|
19
|
+
|
|
20
|
+
* Nothing outside a task workspace is ever a candidate. A tree is admitted
|
|
21
|
+
only when its cwd or one of its args sits inside a watched hub's task
|
|
22
|
+
workspace, judged by the janitor's own grammar (`parse_task_workspace`), so
|
|
23
|
+
`main/`, `.source/`, an operator's shell and the daemon itself are never
|
|
24
|
+
CONSIDERED -- the property the tests pin, rather than "checked and spared".
|
|
25
|
+
* CPU progress, not wall-clock, is the hang signal. A full suite that runs for
|
|
26
|
+
an hour is fine; a runner that burned under `MIN_PROGRESS_SECONDS` of CPU
|
|
27
|
+
over the whole window is waiting on something that is not coming back.
|
|
28
|
+
* Nothing raises past its own concern. A process that exits mid-read, an
|
|
29
|
+
unreadable `/proc` entry, a signal to a pid that is already gone -- all are
|
|
30
|
+
skipped, because this runs inside a poll pass.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import json
|
|
36
|
+
import logging
|
|
37
|
+
import os
|
|
38
|
+
import signal
|
|
39
|
+
import time
|
|
40
|
+
from dataclasses import dataclass, field
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from typing import Callable, Iterable, Mapping, Sequence
|
|
43
|
+
|
|
44
|
+
from .janitor import TaskWorkspace, parse_task_workspace
|
|
45
|
+
from .proc import CommandError, run
|
|
46
|
+
|
|
47
|
+
log = logging.getLogger(__name__)
|
|
48
|
+
|
|
49
|
+
# Where the process table is read. A module attribute (read at call time,
|
|
50
|
+
# never bound as a default) so the tests point it at a fake tree.
|
|
51
|
+
PROC_ROOT = Path("/proc")
|
|
52
|
+
|
|
53
|
+
# A tree whose summed CPU time grows by less than this across the window is
|
|
54
|
+
# hung. The incident's trees sat at ~10 CPU-seconds TOTAL over hours; a live
|
|
55
|
+
# runner burns this much in seconds.
|
|
56
|
+
MIN_PROGRESS_SECONDS = 5.0
|
|
57
|
+
|
|
58
|
+
# TERM, then KILL whatever is still alive this long after.
|
|
59
|
+
KILL_GRACE_SECONDS = 10.0
|
|
60
|
+
|
|
61
|
+
# How often the TERM→KILL wait re-reads the tree.
|
|
62
|
+
_KILL_POLL_SECONDS = 0.5
|
|
63
|
+
|
|
64
|
+
# The env a managed seat exports (alissa CLI): the tmux session it runs in and
|
|
65
|
+
# the Alissa session id the session-message path addresses. Inherited by every
|
|
66
|
+
# process the seat starts, detached ones included -- which is why they are
|
|
67
|
+
# read off the tree rather than its ancestry: a `nohup npm test &` whose shell
|
|
68
|
+
# exited is reparented to init and has no ancestor pane at all.
|
|
69
|
+
ENV_TMUX_SESSION = "ALISSA_TMUX_SESSION"
|
|
70
|
+
ENV_SESSION_ID = "ALISSA_SESSION_ID"
|
|
71
|
+
|
|
72
|
+
# The prefix the CLI puts on a managed session's real tmux name; the roster
|
|
73
|
+
# (`alissa tmux ls --json`'s `name`) carries it bare.
|
|
74
|
+
TMUX_PREFIX = "ali-"
|
|
75
|
+
|
|
76
|
+
# `npm test` and its aliases, plus `npm run test` / `npm run-script test`.
|
|
77
|
+
_NPM_BINARIES = frozenset({"npm", "npm-cli.js"})
|
|
78
|
+
_NPM_TEST = frozenset({"test", "t", "tst"})
|
|
79
|
+
_NPM_RUN = frozenset({"run", "run-script", "rum", "urn"})
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass(frozen=True, slots=True)
|
|
83
|
+
class Proc:
|
|
84
|
+
"""One process, as `/proc/<pid>` described it at read time.
|
|
85
|
+
|
|
86
|
+
`cpu` is utime+stime PLUS cutime+cstime, in seconds: the reaped children's
|
|
87
|
+
time is the runner's progress too. A runner that executes one test file
|
|
88
|
+
after another reaps each child as it finishes; without the children's
|
|
89
|
+
share the tree's sum would DROP between samples while it works."""
|
|
90
|
+
|
|
91
|
+
pid: int
|
|
92
|
+
ppid: int
|
|
93
|
+
state: str
|
|
94
|
+
start: int
|
|
95
|
+
cpu: float
|
|
96
|
+
argv: "tuple[str, ...]"
|
|
97
|
+
cwd: "str | None"
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def alive(self) -> bool:
|
|
101
|
+
return self.state not in ("Z", "X")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass(frozen=True)
|
|
105
|
+
class TestTree:
|
|
106
|
+
"""One worker test run: the topmost runner process and every descendant,
|
|
107
|
+
with the task workspace it was admitted under."""
|
|
108
|
+
|
|
109
|
+
root: Proc
|
|
110
|
+
members: "tuple[Proc, ...]"
|
|
111
|
+
workspace: TaskWorkspace
|
|
112
|
+
tmux_session: "str | None" = None
|
|
113
|
+
session_id: "str | None" = None
|
|
114
|
+
age: float = 0.0
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def key(self) -> "tuple[int, int]":
|
|
118
|
+
"""(pid, start ticks): a pid alone is reused; the pair is not."""
|
|
119
|
+
return (self.root.pid, self.root.start)
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def cpu(self) -> float:
|
|
123
|
+
return sum(member.cpu for member in self.members)
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def session(self) -> "str | None":
|
|
127
|
+
"""The owning session's roster name (no `ali-` prefix), or None."""
|
|
128
|
+
name = self.tmux_session
|
|
129
|
+
if not name:
|
|
130
|
+
return None
|
|
131
|
+
return name[len(TMUX_PREFIX):] if name.startswith(TMUX_PREFIX) else name
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def command(self) -> str:
|
|
135
|
+
return " ".join(self.root.argv)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _clock_ticks() -> int:
|
|
139
|
+
try:
|
|
140
|
+
return int(os.sysconf("SC_CLK_TCK")) or 100
|
|
141
|
+
except (ValueError, OSError, AttributeError): # pragma: no cover
|
|
142
|
+
return 100
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def read_proc(pid: int, root: "Path | None" = None) -> "Proc | None":
|
|
146
|
+
"""`/proc/<pid>` as a Proc, or None when it cannot be read (gone, a
|
|
147
|
+
kernel thread with no cmdline, permission)."""
|
|
148
|
+
base = (PROC_ROOT if root is None else root) / str(pid)
|
|
149
|
+
try:
|
|
150
|
+
stat = (base / "stat").read_text()
|
|
151
|
+
raw = (base / "cmdline").read_bytes()
|
|
152
|
+
except OSError:
|
|
153
|
+
return None
|
|
154
|
+
# The comm field is parenthesised and may itself hold spaces or `)`;
|
|
155
|
+
# everything after the LAST `)` is the fixed-position tail.
|
|
156
|
+
_, _, tail = stat.rpartition(")")
|
|
157
|
+
fields = tail.split()
|
|
158
|
+
try:
|
|
159
|
+
state, ppid = fields[0], int(fields[1])
|
|
160
|
+
ticks = sum(int(value) for value in fields[11:15])
|
|
161
|
+
start = int(fields[19])
|
|
162
|
+
except (IndexError, ValueError):
|
|
163
|
+
return None
|
|
164
|
+
argv = tuple(part.decode("utf-8", "replace") for part in raw.split(b"\0") if part)
|
|
165
|
+
if not argv:
|
|
166
|
+
return None
|
|
167
|
+
try:
|
|
168
|
+
cwd: "str | None" = os.readlink(base / "cwd")
|
|
169
|
+
except OSError:
|
|
170
|
+
cwd = None
|
|
171
|
+
return Proc(
|
|
172
|
+
pid=pid, ppid=ppid, state=state, start=start,
|
|
173
|
+
cpu=ticks / _clock_ticks(), argv=argv, cwd=cwd,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def snapshot(root: "Path | None" = None) -> "dict[int, Proc]":
|
|
178
|
+
"""Every readable process, by pid. An unreadable table yields nothing."""
|
|
179
|
+
base = PROC_ROOT if root is None else root
|
|
180
|
+
try:
|
|
181
|
+
names = [entry.name for entry in base.iterdir()]
|
|
182
|
+
except OSError as exc:
|
|
183
|
+
log.warning("hung tests: cannot list %s (%s) — skipped this pass", base, exc)
|
|
184
|
+
return {}
|
|
185
|
+
procs = {}
|
|
186
|
+
for name in names:
|
|
187
|
+
if not name.isdigit():
|
|
188
|
+
continue
|
|
189
|
+
proc = read_proc(int(name), base)
|
|
190
|
+
if proc is not None:
|
|
191
|
+
procs[proc.pid] = proc
|
|
192
|
+
return procs
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def uptime(root: "Path | None" = None) -> "float | None":
|
|
196
|
+
"""Seconds since boot (`/proc/uptime`), or None."""
|
|
197
|
+
base = PROC_ROOT if root is None else root
|
|
198
|
+
try:
|
|
199
|
+
return float((base / "uptime").read_text().split()[0])
|
|
200
|
+
except (OSError, IndexError, ValueError):
|
|
201
|
+
return None
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def is_test_runner(argv: "Sequence[str]") -> bool:
|
|
205
|
+
"""Is this command line a test runner: `npm test` (or `npm run test`),
|
|
206
|
+
or anything carrying a bare `--test` argument -- `node --test`,
|
|
207
|
+
`tsx --test`, `npx tsx --test`, and a `timeout … --test` wrapper around
|
|
208
|
+
one. Exact tokens only: a `bash -c '… --test …'` is one argument holding
|
|
209
|
+
the words, and the shell is not the runner."""
|
|
210
|
+
if "--test" in argv:
|
|
211
|
+
return True
|
|
212
|
+
for index, token in enumerate(argv):
|
|
213
|
+
if os.path.basename(token) not in _NPM_BINARIES:
|
|
214
|
+
continue
|
|
215
|
+
rest = [arg for arg in argv[index + 1:] if not arg.startswith("-")]
|
|
216
|
+
if rest and rest[0] in _NPM_TEST:
|
|
217
|
+
return True
|
|
218
|
+
if len(rest) >= 2 and rest[0] in _NPM_RUN and rest[1] == "test":
|
|
219
|
+
return True
|
|
220
|
+
return False
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _path_in(text: str) -> "str | None":
|
|
224
|
+
if text.startswith("file://"):
|
|
225
|
+
text = text[len("file://"):]
|
|
226
|
+
return text if text.startswith("/") else None
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def workspace_of(
|
|
230
|
+
proc: Proc, hubs: "Sequence[Path]"
|
|
231
|
+
) -> "TaskWorkspace | None":
|
|
232
|
+
"""The task workspace `proc`'s cwd or one of its path args sits in, or
|
|
233
|
+
None. The hub's direct child that holds the path must parse as a task
|
|
234
|
+
workspace under the janitor's grammar -- `main/`, `.source/` and anything
|
|
235
|
+
else under a hub never qualify."""
|
|
236
|
+
candidates = [proc.cwd] if proc.cwd else []
|
|
237
|
+
candidates += [p for p in (_path_in(arg) for arg in proc.argv) if p]
|
|
238
|
+
for text in candidates:
|
|
239
|
+
path = Path(os.path.normpath(text))
|
|
240
|
+
for hub in hubs:
|
|
241
|
+
try:
|
|
242
|
+
relative = path.relative_to(hub)
|
|
243
|
+
except ValueError:
|
|
244
|
+
continue
|
|
245
|
+
if not relative.parts:
|
|
246
|
+
continue
|
|
247
|
+
workspace = parse_task_workspace(hub / relative.parts[0], hub)
|
|
248
|
+
if workspace is not None:
|
|
249
|
+
return workspace
|
|
250
|
+
return None
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def session_env(pid: int, root: "Path | None" = None) -> "tuple[str | None, str | None]":
|
|
254
|
+
"""(ALISSA_TMUX_SESSION, ALISSA_SESSION_ID) from the process's env."""
|
|
255
|
+
base = (PROC_ROOT if root is None else root) / str(pid)
|
|
256
|
+
try:
|
|
257
|
+
raw = (base / "environ").read_bytes()
|
|
258
|
+
except OSError:
|
|
259
|
+
return None, None
|
|
260
|
+
found: "dict[str, str]" = {}
|
|
261
|
+
for part in raw.split(b"\0"):
|
|
262
|
+
name, sep, value = part.decode("utf-8", "replace").partition("=")
|
|
263
|
+
if sep and name in (ENV_TMUX_SESSION, ENV_SESSION_ID) and value:
|
|
264
|
+
found[name] = value
|
|
265
|
+
return found.get(ENV_TMUX_SESSION), found.get(ENV_SESSION_ID)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _children(procs: "Mapping[int, Proc]") -> "dict[int, list[int]]":
|
|
269
|
+
children: "dict[int, list[int]]" = {}
|
|
270
|
+
for proc in procs.values():
|
|
271
|
+
children.setdefault(proc.ppid, []).append(proc.pid)
|
|
272
|
+
return children
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def descendants(procs: "Mapping[int, Proc]", pid: int) -> "list[Proc]":
|
|
276
|
+
"""`pid` and every process below it, root first."""
|
|
277
|
+
children = _children(procs)
|
|
278
|
+
order: "list[Proc]" = []
|
|
279
|
+
stack = [pid]
|
|
280
|
+
seen: "set[int]" = set()
|
|
281
|
+
while stack:
|
|
282
|
+
current = stack.pop()
|
|
283
|
+
if current in seen or current not in procs:
|
|
284
|
+
continue
|
|
285
|
+
seen.add(current)
|
|
286
|
+
order.append(procs[current])
|
|
287
|
+
stack.extend(sorted(children.get(current, ()), reverse=True))
|
|
288
|
+
return order
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _has_runner_ancestor(procs: "Mapping[int, Proc]", proc: Proc) -> bool:
|
|
292
|
+
"""Is any live ancestor of `proc` itself a runner? A `sh -c 'tsx --test'`
|
|
293
|
+
between two runners is one argument and not a runner, so the parent alone
|
|
294
|
+
cannot answer -- the whole chain up to init is read."""
|
|
295
|
+
seen = {proc.pid}
|
|
296
|
+
current = procs.get(proc.ppid)
|
|
297
|
+
while current is not None and current.pid not in seen:
|
|
298
|
+
if current.alive and is_test_runner(current.argv):
|
|
299
|
+
return True
|
|
300
|
+
seen.add(current.pid)
|
|
301
|
+
current = procs.get(current.ppid)
|
|
302
|
+
return False
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def find_test_trees(
|
|
306
|
+
procs: "Mapping[int, Proc]",
|
|
307
|
+
hubs: "Sequence[Path]",
|
|
308
|
+
root: "Path | None" = None,
|
|
309
|
+
) -> "list[TestTree]":
|
|
310
|
+
"""Every worker test run inside a task workspace of one of `hubs`.
|
|
311
|
+
|
|
312
|
+
A tree is rooted at its TOPMOST runner (a runner with no runner above it),
|
|
313
|
+
so `timeout 900 npx tsx --test` and the `npm exec` / `tsx` / `node --test`
|
|
314
|
+
chain under it are one tree, killed together. The workspace is read off
|
|
315
|
+
the first member, root first, whose cwd or args name one; a tree none of
|
|
316
|
+
whose members does is not ours and is never returned."""
|
|
317
|
+
resolved = []
|
|
318
|
+
for hub in hubs:
|
|
319
|
+
try:
|
|
320
|
+
resolved.append(Path(hub).resolve())
|
|
321
|
+
except OSError:
|
|
322
|
+
continue
|
|
323
|
+
boot = uptime(root)
|
|
324
|
+
ticks = _clock_ticks()
|
|
325
|
+
trees = []
|
|
326
|
+
for proc in sorted(procs.values(), key=lambda p: p.pid):
|
|
327
|
+
if not proc.alive or not is_test_runner(proc.argv):
|
|
328
|
+
continue
|
|
329
|
+
if _has_runner_ancestor(procs, proc):
|
|
330
|
+
continue
|
|
331
|
+
members = tuple(m for m in descendants(procs, proc.pid) if m.alive)
|
|
332
|
+
workspace = None
|
|
333
|
+
for member in members:
|
|
334
|
+
workspace = workspace_of(member, resolved)
|
|
335
|
+
if workspace is not None:
|
|
336
|
+
break
|
|
337
|
+
if workspace is None:
|
|
338
|
+
continue
|
|
339
|
+
tmux_session, session_id = session_env(proc.pid, root)
|
|
340
|
+
age = 0.0 if boot is None else max(0.0, boot - proc.start / ticks)
|
|
341
|
+
trees.append(TestTree(
|
|
342
|
+
root=proc, members=members, workspace=workspace,
|
|
343
|
+
tmux_session=tmux_session, session_id=session_id, age=age,
|
|
344
|
+
))
|
|
345
|
+
return trees
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
@dataclass
|
|
349
|
+
class CpuWatch:
|
|
350
|
+
"""The last CPU sample per tree, and the hang verdict across a window.
|
|
351
|
+
|
|
352
|
+
A tree is sampled the first time it is seen; once `window` seconds have
|
|
353
|
+
passed since its sample it is judged: hung when it is at least `window`
|
|
354
|
+
old and its CPU grew by under `MIN_PROGRESS_SECONDS`, otherwise the
|
|
355
|
+
sample moves forward and a new window starts. A tree that burns CPU is
|
|
356
|
+
therefore re-judged every window, and a tree younger than one never is.
|
|
357
|
+
Samples of trees that are no longer seen are dropped."""
|
|
358
|
+
|
|
359
|
+
window: float
|
|
360
|
+
clock: "Callable[[], float] | None" = None
|
|
361
|
+
samples: "dict[tuple[int, int], tuple[float, float]]" = field(default_factory=dict)
|
|
362
|
+
|
|
363
|
+
def observe(self, trees: "Iterable[TestTree]") -> "list[TestTree]":
|
|
364
|
+
now = (self.clock or monotonic)()
|
|
365
|
+
hung = []
|
|
366
|
+
seen = set()
|
|
367
|
+
for tree in trees:
|
|
368
|
+
seen.add(tree.key)
|
|
369
|
+
sample = self.samples.get(tree.key)
|
|
370
|
+
if sample is None:
|
|
371
|
+
self.samples[tree.key] = (now, tree.cpu)
|
|
372
|
+
continue
|
|
373
|
+
at, cpu = sample
|
|
374
|
+
if now - at < self.window:
|
|
375
|
+
continue
|
|
376
|
+
if tree.age >= self.window and tree.cpu - cpu < MIN_PROGRESS_SECONDS:
|
|
377
|
+
hung.append(tree)
|
|
378
|
+
continue
|
|
379
|
+
self.samples[tree.key] = (now, tree.cpu)
|
|
380
|
+
for key in list(self.samples):
|
|
381
|
+
if key not in seen:
|
|
382
|
+
del self.samples[key]
|
|
383
|
+
return hung
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def send_signal(pid: int, signum: int) -> None:
|
|
387
|
+
"""`os.kill`, swallowing a pid that is already gone or not ours."""
|
|
388
|
+
try:
|
|
389
|
+
os.kill(pid, signum)
|
|
390
|
+
except (ProcessLookupError, PermissionError):
|
|
391
|
+
pass
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def sleep(seconds: float) -> None:
|
|
395
|
+
"""`time.sleep`, behind a module seam the tests replace."""
|
|
396
|
+
time.sleep(seconds)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def monotonic() -> float:
|
|
400
|
+
"""`time.monotonic`, behind a module seam the tests replace."""
|
|
401
|
+
return time.monotonic()
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _still_alive(member: Proc, root: "Path | None") -> bool:
|
|
405
|
+
now = read_proc(member.pid, root)
|
|
406
|
+
return now is not None and now.start == member.start and now.alive
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def kill_tree(
|
|
410
|
+
tree: TestTree,
|
|
411
|
+
*,
|
|
412
|
+
root: "Path | None" = None,
|
|
413
|
+
grace: float = KILL_GRACE_SECONDS,
|
|
414
|
+
) -> "tuple[int, int]":
|
|
415
|
+
"""TERM every member of `tree`, wait up to `grace` seconds, then KILL
|
|
416
|
+
whatever is still alive. Returns (termed, killed).
|
|
417
|
+
|
|
418
|
+
The member list is re-read first, so a child the runner spawned since the
|
|
419
|
+
snapshot is signalled too; every pid is matched on its start time before
|
|
420
|
+
each signal, so a pid the kernel reused for something else is never hit.
|
|
421
|
+
"""
|
|
422
|
+
fresh = snapshot(root)
|
|
423
|
+
current = fresh.get(tree.root.pid)
|
|
424
|
+
if current is None or current.start != tree.root.start:
|
|
425
|
+
return 0, 0
|
|
426
|
+
members = [m for m in descendants(fresh, tree.root.pid) if m.alive]
|
|
427
|
+
for member in members:
|
|
428
|
+
send_signal(member.pid, signal.SIGTERM)
|
|
429
|
+
deadline = monotonic() + grace
|
|
430
|
+
survivors = [m for m in members if _still_alive(m, root)]
|
|
431
|
+
while survivors and monotonic() < deadline:
|
|
432
|
+
sleep(_KILL_POLL_SECONDS)
|
|
433
|
+
survivors = [m for m in survivors if _still_alive(m, root)]
|
|
434
|
+
for member in survivors:
|
|
435
|
+
send_signal(member.pid, signal.SIGKILL)
|
|
436
|
+
return len(members), len(survivors)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def session_id_for(tmux_session: str, runner: "Callable[..., str]" = run) -> "str | None":
|
|
440
|
+
"""The Alissa session id behind a managed tmux session, read off
|
|
441
|
+
`alissa session list --json`'s `tmuxSession`, or None."""
|
|
442
|
+
try:
|
|
443
|
+
out = runner(["alissa", "session", "list", "--json", "--limit", "50"], timeout=60)
|
|
444
|
+
data = json.loads(out)
|
|
445
|
+
except (CommandError, OSError, ValueError) as exc:
|
|
446
|
+
log.debug("hung tests: could not list sessions (%s)", exc)
|
|
447
|
+
return None
|
|
448
|
+
if isinstance(data, dict):
|
|
449
|
+
data = data.get("sessions", [])
|
|
450
|
+
wanted = {tmux_session, TMUX_PREFIX + tmux_session}
|
|
451
|
+
for entry in data if isinstance(data, list) else []:
|
|
452
|
+
if isinstance(entry, dict) and entry.get("tmuxSession") in wanted:
|
|
453
|
+
value = entry.get("sessionId")
|
|
454
|
+
if isinstance(value, str) and value:
|
|
455
|
+
return value
|
|
456
|
+
return None
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def message_session(
|
|
460
|
+
session_id: str, text: str, runner: "Callable[..., str]" = run
|
|
461
|
+
) -> bool:
|
|
462
|
+
"""Post `text` to one session through the session-message path
|
|
463
|
+
(`alissa session msg send -s <id>`). True when the CLI accepted it."""
|
|
464
|
+
try:
|
|
465
|
+
runner(["alissa", "session", "msg", "send", "-s", session_id, text], timeout=60)
|
|
466
|
+
except (CommandError, OSError) as exc:
|
|
467
|
+
log.warning("hung tests: could not message session %s (%s)", session_id, exc)
|
|
468
|
+
return False
|
|
469
|
+
return True
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def hung_message(tree: TestTree, minutes: int, reason: str) -> str:
|
|
473
|
+
"""The one message a live session gets after its test run was stopped."""
|
|
474
|
+
return (
|
|
475
|
+
f"devloop: your test run in {tree.workspace.path.name} "
|
|
476
|
+
f"(`{tree.command}`, pid {tree.root.pid}) hung and was stopped "
|
|
477
|
+
f"({reason}). Do not rerun the full suite and never wait on one for "
|
|
478
|
+
f"more than {minutes} minutes: run only the test files your diff "
|
|
479
|
+
f"touches, each on its own, one file per command. CI on the PR runs "
|
|
480
|
+
f"the full suite."
|
|
481
|
+
)
|
|
@@ -461,6 +461,7 @@ from enum import Enum
|
|
|
461
461
|
from pathlib import Path
|
|
462
462
|
from typing import Callable, Sequence
|
|
463
463
|
|
|
464
|
+
from . import hungtests
|
|
464
465
|
from . import janitor
|
|
465
466
|
from . import fleet_vitals as fleet_vitals_mod
|
|
466
467
|
from . import loop_events as loop_events_mod
|
|
@@ -776,6 +777,21 @@ _SHELL_RULE = (
|
|
|
776
777
|
"change the command; do not retry it. "
|
|
777
778
|
)
|
|
778
779
|
|
|
780
|
+
# The test-run rule (issue #174). On 2026-10-18 four worker `npm test` runs
|
|
781
|
+
# hung -- their test workers dead, the runner waiting forever -- and their
|
|
782
|
+
# sessions waited on them, one for 55 h, in an `until grep -q '^EXIT='` loop.
|
|
783
|
+
# The daemon now stops such a run (`DevWatcher.run_hung_tests`); this rule is
|
|
784
|
+
# the half that keeps a worker from starting one it would wait on. In EVERY
|
|
785
|
+
# directive that carries _SHELL_RULE, right after it, because every one of
|
|
786
|
+
# them runs tests in the same box. The 20 is the issue's own number, not the
|
|
787
|
+
# configured window: the directives are byte-pinned templates.
|
|
788
|
+
_TEST_RULE = (
|
|
789
|
+
"Never wait on a full-suite test run for more than 20 minutes: prefer "
|
|
790
|
+
"targeted test files — only the files your diff touches, each on its "
|
|
791
|
+
"own — because CI on the PR is the full run (the daemon stops a worker "
|
|
792
|
+
"test run that makes no CPU progress for 20 minutes). "
|
|
793
|
+
)
|
|
794
|
+
|
|
779
795
|
# The regenerate-on-conflict rule (issue #138), rendered into every worker
|
|
780
796
|
# directive that can end up merging main into its branch -- develop, resume,
|
|
781
797
|
# fix, fix-resume and maintain -- and ONLY for a repo with entries in
|
|
@@ -833,6 +849,7 @@ DEV_DIRECTIVE = (
|
|
|
833
849
|
"main). "
|
|
834
850
|
+ _BRANCH_RULE
|
|
835
851
|
+ _SHELL_RULE
|
|
852
|
+
+ _TEST_RULE
|
|
836
853
|
+ "Implement to the issue's acceptance criteria and VERIFY locally (run the "
|
|
837
854
|
"repo's test/style/type scripts) before publishing. {regenerate_rule}"
|
|
838
855
|
"Open a DRAFT PR whose "
|
|
@@ -899,6 +916,7 @@ RESUME_DIRECTIVE = (
|
|
|
899
916
|
"(the origin task AND your implementation task) — restoring them if the "
|
|
900
917
|
"dead session did not. "
|
|
901
918
|
+ _SHELL_RULE
|
|
919
|
+
+ _TEST_RULE
|
|
902
920
|
+ _RESUME_DELIVERY_NOTE
|
|
903
921
|
+ " "
|
|
904
922
|
"If the PR already carries review findings, the ORDER is load-bearing: "
|
|
@@ -958,6 +976,7 @@ DISCOVERY_DIRECTIVE = (
|
|
|
958
976
|
"main). "
|
|
959
977
|
+ _BRANCH_RULE
|
|
960
978
|
+ _SHELL_RULE
|
|
979
|
+
+ _TEST_RULE
|
|
961
980
|
+ "READ the code at the pinned SHA and nowhere else, read-only: "
|
|
962
981
|
"`{read_repo}` at `{read_sha}` (the issue's `Alissa-Reads:` trailer). "
|
|
963
982
|
"If this workspace holds a hub of {read_repo} (a sibling of your hub "
|
|
@@ -1064,6 +1083,7 @@ FIX_DIRECTIVE = (
|
|
|
1064
1083
|
"main). "
|
|
1065
1084
|
+ _BRANCH_RULE
|
|
1066
1085
|
+ _SHELL_RULE
|
|
1086
|
+
+ _TEST_RULE
|
|
1067
1087
|
+ "FIRST verify the triage of every prior review finding per CR8: exactly "
|
|
1068
1088
|
"one reasoned `[triage:*]` reply on each finding's thread — post any that "
|
|
1069
1089
|
"are missing. Then fix the pursued findings on the same branch and VERIFY "
|
|
@@ -1119,6 +1139,7 @@ FIX_RESUME_DIRECTIVE = (
|
|
|
1119
1139
|
"rename, never a clone, never main, and NEVER a second PR. "
|
|
1120
1140
|
+ _BRANCH_RULE
|
|
1121
1141
|
+ _SHELL_RULE
|
|
1142
|
+
+ _TEST_RULE
|
|
1122
1143
|
+ "The dead session may have posted SOME of its CR8 triage replies already: "
|
|
1123
1144
|
"read every finding's thread and post exactly the ones that are missing — "
|
|
1124
1145
|
"one reasoned `[triage:*]` reply per finding, NEVER a duplicate on a "
|
|
@@ -1167,6 +1188,7 @@ MAINTAIN_DIRECTIVE = (
|
|
|
1167
1188
|
"main). "
|
|
1168
1189
|
+ _BRANCH_RULE
|
|
1169
1190
|
+ _SHELL_RULE
|
|
1191
|
+
+ _TEST_RULE
|
|
1170
1192
|
+ "FIRST the idempotent no-op check: if the branch is already conflict-free "
|
|
1171
1193
|
"AND current vs origin/main, remove the `{label}` label from the PR "
|
|
1172
1194
|
"(`gh pr edit {number} --remove-label \"{label}\"`) and exit — push "
|
|
@@ -1227,6 +1249,7 @@ REVERT_DIRECTIVE = (
|
|
|
1227
1249
|
"the revert below. "
|
|
1228
1250
|
+ _BRANCH_RULE
|
|
1229
1251
|
+ _SHELL_RULE
|
|
1252
|
+
+ _TEST_RULE
|
|
1230
1253
|
+ "Revert by merge shape. `git cat-file -p {merge_sha}` lists the "
|
|
1231
1254
|
"parents. TWO `parent` lines mean a merge commit: run `git revert "
|
|
1232
1255
|
"--no-edit -m 1 {merge_sha}`. ONE parent means a squash OR a rebase "
|
|
@@ -3874,6 +3897,13 @@ class DevWatcher:
|
|
|
3874
3897
|
# keys and re-capturing the pane (issue #127); injectable so a test
|
|
3875
3898
|
# never sleeps.
|
|
3876
3899
|
self._sleep: "Callable[[float], None]" = time.sleep
|
|
3900
|
+
# The hung-test reaper's per-process state (issue #174): the last CPU
|
|
3901
|
+
# sample per test tree, when its sweep last ran (the janitor's
|
|
3902
|
+
# cadence, on its own clock), and the trees it already stopped --
|
|
3903
|
+
# each is stopped, and its session told, once.
|
|
3904
|
+
self._hung_watch: "hungtests.CpuWatch | None" = None
|
|
3905
|
+
self._hung_tests_at: "float | None" = None
|
|
3906
|
+
self._hung_reaped: "set[tuple[int, int]]" = set()
|
|
3877
3907
|
|
|
3878
3908
|
# -- per-issue decision ------------------------------------------------
|
|
3879
3909
|
|
|
@@ -11176,6 +11206,192 @@ class DevWatcher:
|
|
|
11176
11206
|
self._log_disk(reclaim)
|
|
11177
11207
|
return reclaim
|
|
11178
11208
|
|
|
11209
|
+
# -- the hung-test reaper (issue #174) ---------------------------------
|
|
11210
|
+
|
|
11211
|
+
def run_hung_tests(self, roster: "SessionLiveness | None") -> int:
|
|
11212
|
+
"""Stop worker test runs that hung, or that outlived their lane, and
|
|
11213
|
+
tell a live session once. Returns how many trees were stopped.
|
|
11214
|
+
|
|
11215
|
+
The 2026-10-18 incident: four worker `npm test` trees, their test
|
|
11216
|
+
workers dead, sat at ~10 CPU-seconds for 52 min to 55 h while their
|
|
11217
|
+
sessions polled a log for an `EXIT=` line that was never coming.
|
|
11218
|
+
Every check the daemon has reads such a session ALIVE, so nothing
|
|
11219
|
+
else will ever free it.
|
|
11220
|
+
|
|
11221
|
+
Runs on the janitor's cadence (`janitor_interval_minutes`, on its own
|
|
11222
|
+
clock) after the janitor, and is off when `hung_test_minutes` is 0.
|
|
11223
|
+
Mechanics are `hungtests`' -- which trees are worker test runs inside
|
|
11224
|
+
a task workspace, and which grew by under 5 s of CPU across the
|
|
11225
|
+
window; the verdict is `_hung_test_verdict`. Nothing in here may
|
|
11226
|
+
take a poll down."""
|
|
11227
|
+
minutes = self.config.hung_test_minutes
|
|
11228
|
+
if minutes <= 0:
|
|
11229
|
+
return 0
|
|
11230
|
+
window = float(minutes * 60)
|
|
11231
|
+
if self._hung_watch is None or self._hung_watch.window != window:
|
|
11232
|
+
self._hung_watch = hungtests.CpuWatch(window=window)
|
|
11233
|
+
interval = self.config.janitor_interval_seconds
|
|
11234
|
+
now = time.monotonic()
|
|
11235
|
+
if (
|
|
11236
|
+
self._hung_tests_at is not None
|
|
11237
|
+
and interval > 0
|
|
11238
|
+
and now - self._hung_tests_at < interval
|
|
11239
|
+
):
|
|
11240
|
+
return 0
|
|
11241
|
+
self._hung_tests_at = now
|
|
11242
|
+
try:
|
|
11243
|
+
return self._reap_hung_tests(roster)
|
|
11244
|
+
except (CommandError, OSError) as exc:
|
|
11245
|
+
log.error("hung tests: sweep failed (%s) — skipped this pass", exc)
|
|
11246
|
+
return 0
|
|
11247
|
+
|
|
11248
|
+
def _reap_hung_tests(self, roster: "SessionLiveness | None") -> int:
|
|
11249
|
+
"""One sweep: every test tree inside a watched hub's task workspace,
|
|
11250
|
+
judged and, when the verdict says so, stopped."""
|
|
11251
|
+
assert self._hung_watch is not None
|
|
11252
|
+
hubs: "dict[Path, tuple[str, str]]" = {}
|
|
11253
|
+
for owner, repo in self._watched_repos():
|
|
11254
|
+
hub = self.config.hub_for(owner, repo)
|
|
11255
|
+
if hub.is_dir():
|
|
11256
|
+
hubs[hub.resolve()] = (owner, repo)
|
|
11257
|
+
if not hubs:
|
|
11258
|
+
return 0
|
|
11259
|
+
trees = hungtests.find_test_trees(hungtests.snapshot(), list(hubs))
|
|
11260
|
+
hung = {tree.key for tree in self._hung_watch.observe(trees)}
|
|
11261
|
+
# A stopped tree that is still listed (a member stuck in D state)
|
|
11262
|
+
# stays remembered, so it is never stopped -- or its session told --
|
|
11263
|
+
# twice; one that is gone is forgotten.
|
|
11264
|
+
self._hung_reaped &= {tree.key for tree in trees}
|
|
11265
|
+
stopped = 0
|
|
11266
|
+
for tree in trees:
|
|
11267
|
+
if tree.key in self._hung_reaped:
|
|
11268
|
+
continue
|
|
11269
|
+
owner, repo = hubs[tree.workspace.hub]
|
|
11270
|
+
verdict = self._hung_test_verdict(
|
|
11271
|
+
tree, owner, repo, roster, hung=tree.key in hung
|
|
11272
|
+
)
|
|
11273
|
+
if verdict is None:
|
|
11274
|
+
continue
|
|
11275
|
+
stopped += self._stop_hung_tree(tree, verdict, roster)
|
|
11276
|
+
return stopped
|
|
11277
|
+
|
|
11278
|
+
def _hung_test_verdict(
|
|
11279
|
+
self,
|
|
11280
|
+
tree: "hungtests.TestTree",
|
|
11281
|
+
owner: str,
|
|
11282
|
+
repo: str,
|
|
11283
|
+
roster: "SessionLiveness | None",
|
|
11284
|
+
*,
|
|
11285
|
+
hung: bool,
|
|
11286
|
+
) -> "str | None":
|
|
11287
|
+
"""Why `tree` must be stopped, or None to leave it running.
|
|
11288
|
+
|
|
11289
|
+
Cheapest first:
|
|
11290
|
+
|
|
11291
|
+
* its session is GONE -- the seat env the tree inherited
|
|
11292
|
+
(`ALISSA_TMUX_SESSION`) names a session a successful listing does
|
|
11293
|
+
not carry. Stopped at once: nothing is waiting on the run;
|
|
11294
|
+
* it is HUNG -- under 5 s of CPU growth across the window;
|
|
11295
|
+
* its anchor is TERMINAL -- the janitor's own test: the branch has
|
|
11296
|
+
pull requests, none open, and the newest one's state is terminal
|
|
11297
|
+
per `anchor_terminal`. Stopped at once: the lane is over;
|
|
11298
|
+
* for a tree whose env names no session, the janitor's liveness
|
|
11299
|
+
read of the branch's PR (`_orphan_liveness`) is DEAD.
|
|
11300
|
+
|
|
11301
|
+
A listing that failed, or a GitHub read that failed, is never
|
|
11302
|
+
evidence of death: those gates leave the tree to the CPU window."""
|
|
11303
|
+
session = tree.session
|
|
11304
|
+
if (
|
|
11305
|
+
session is not None
|
|
11306
|
+
and roster is not None
|
|
11307
|
+
and roster.of(session) is Liveness.DEAD
|
|
11308
|
+
):
|
|
11309
|
+
return f"its session {session} is gone"
|
|
11310
|
+
if hung:
|
|
11311
|
+
return (
|
|
11312
|
+
f"hung — under {hungtests.MIN_PROGRESS_SECONDS:g} s of CPU in "
|
|
11313
|
+
f"{self.config.hung_test_minutes} min"
|
|
11314
|
+
)
|
|
11315
|
+
try:
|
|
11316
|
+
pulls = self.github.branch_pulls(owner, repo, tree.workspace.branch)
|
|
11317
|
+
except (RateLimited, CommandError) as exc:
|
|
11318
|
+
log.debug(
|
|
11319
|
+
"hung tests: %s anchor could not be read (%s) — left to the "
|
|
11320
|
+
"CPU window", tree.workspace.slug, exc,
|
|
11321
|
+
)
|
|
11322
|
+
return None
|
|
11323
|
+
if not pulls:
|
|
11324
|
+
return None
|
|
11325
|
+
if not any(pull.state == "open" for pull in pulls):
|
|
11326
|
+
newest = max(pulls, key=lambda pull: pull.number)
|
|
11327
|
+
reason = anchor_terminal(newest.state, is_issue=False)
|
|
11328
|
+
if reason is not None:
|
|
11329
|
+
return f"its anchor #{newest.number} is terminal ({reason})"
|
|
11330
|
+
if session is None:
|
|
11331
|
+
open_pulls = [pull for pull in pulls if pull.state == "open"]
|
|
11332
|
+
newest = max(open_pulls or pulls, key=lambda pull: pull.number)
|
|
11333
|
+
liveness, _ = self._orphan_liveness(newest, roster)
|
|
11334
|
+
if liveness is Liveness.DEAD:
|
|
11335
|
+
return f"no worker session for #{newest.number} is listed"
|
|
11336
|
+
return None
|
|
11337
|
+
|
|
11338
|
+
def _stop_hung_tree(
|
|
11339
|
+
self,
|
|
11340
|
+
tree: "hungtests.TestTree",
|
|
11341
|
+
reason: str,
|
|
11342
|
+
roster: "SessionLiveness | None",
|
|
11343
|
+
) -> int:
|
|
11344
|
+
"""TERM the whole tree, KILL what is left after 10 s, and post ONE
|
|
11345
|
+
message to its session when that session is live."""
|
|
11346
|
+
self._hung_reaped.add(tree.key)
|
|
11347
|
+
slug = tree.workspace.slug
|
|
11348
|
+
if self.config.dry_run:
|
|
11349
|
+
log.info(
|
|
11350
|
+
"[dry-run] would stop the test run %s in %s (pid %d): %s",
|
|
11351
|
+
tree.command, slug, tree.root.pid, reason,
|
|
11352
|
+
)
|
|
11353
|
+
return 0
|
|
11354
|
+
termed, killed = hungtests.kill_tree(tree)
|
|
11355
|
+
if self._hung_watch is not None:
|
|
11356
|
+
self._hung_watch.samples.pop(tree.key, None)
|
|
11357
|
+
if termed == 0:
|
|
11358
|
+
log.info(
|
|
11359
|
+
"hung tests: %s (pid %d) exited before it could be stopped",
|
|
11360
|
+
slug, tree.root.pid,
|
|
11361
|
+
)
|
|
11362
|
+
return 0
|
|
11363
|
+
log.warning(
|
|
11364
|
+
"hung tests: stopped `%s` in %s (pid %d, %.1f CPU-s) — %s; TERM "
|
|
11365
|
+
"to %d process(es), KILL to %d",
|
|
11366
|
+
tree.command, slug, tree.root.pid, tree.cpu, reason, termed, killed,
|
|
11367
|
+
)
|
|
11368
|
+
session = tree.session
|
|
11369
|
+
if (
|
|
11370
|
+
session is not None
|
|
11371
|
+
and roster is not None
|
|
11372
|
+
and roster.of(session) is Liveness.ALIVE
|
|
11373
|
+
):
|
|
11374
|
+
self._message_hung_session(tree, reason)
|
|
11375
|
+
return 1
|
|
11376
|
+
|
|
11377
|
+
def _message_hung_session(
|
|
11378
|
+
self, tree: "hungtests.TestTree", reason: str
|
|
11379
|
+
) -> None:
|
|
11380
|
+
"""The session-message half: one message, through `alissa session
|
|
11381
|
+
msg send`, to the session the tree's env names."""
|
|
11382
|
+
session_id = tree.session_id
|
|
11383
|
+
if not session_id and tree.tmux_session:
|
|
11384
|
+
session_id = hungtests.session_id_for(tree.tmux_session)
|
|
11385
|
+
if not session_id:
|
|
11386
|
+
log.warning(
|
|
11387
|
+
"hung tests: stopped a test run of %s but its Alissa session "
|
|
11388
|
+
"id is unknown — no message sent", tree.session,
|
|
11389
|
+
)
|
|
11390
|
+
return
|
|
11391
|
+
text = hungtests.hung_message(tree, self.config.hung_test_minutes, reason)
|
|
11392
|
+
if hungtests.message_session(session_id, text):
|
|
11393
|
+
log.info("hung tests: told %s its test run was stopped", tree.session)
|
|
11394
|
+
|
|
11179
11395
|
def _janitor_due(self) -> bool:
|
|
11180
11396
|
"""Has `janitor_interval_minutes` elapsed since the last expensive
|
|
11181
11397
|
half? The first pass of a process always runs (nothing has been swept
|
|
@@ -11910,6 +12126,10 @@ class DevWatcher:
|
|
|
11910
12126
|
# and the rate budget: nothing above consumes the disk it frees, and
|
|
11911
12127
|
# its telemetry line reads best next to the summary below it.
|
|
11912
12128
|
self.run_janitor(roster)
|
|
12129
|
+
# The hung-test reaper (issue #174) on the janitor's cadence: a
|
|
12130
|
+
# worker's test run that stopped making CPU progress, or outlived its
|
|
12131
|
+
# lane, is stopped and its live session told once.
|
|
12132
|
+
self.run_hung_tests(roster)
|
|
11913
12133
|
|
|
11914
12134
|
counts = Counter(decision.action for _, decision in results)
|
|
11915
12135
|
self._write_snapshot(
|
|
@@ -25,7 +25,7 @@ serves it on ``GET /api/state`` and runs the drain handshake):
|
|
|
25
25
|
null as a refusal.
|
|
26
26
|
* **The drain.** A flag with a TTL in `state.db` (`State.set_drain`), cleared
|
|
27
27
|
when the daemon boots. While it holds, every spawn path stands down before
|
|
28
|
-
its first side effect (`DevWatcher.
|
|
28
|
+
its first side effect (`DevWatcher._drain_gated`).
|
|
29
29
|
"""
|
|
30
30
|
|
|
31
31
|
from __future__ import annotations
|
|
@@ -1 +1 @@
|
|
|
1
|
-
0.8.
|
|
1
|
+
0.8.47
|
|
@@ -234,7 +234,16 @@ class App:
|
|
|
234
234
|
down and the answer is `drained: false` with the reasons. When the
|
|
235
235
|
drain holds, a vitals snapshot carrying `idle.drainedAt` is pushed
|
|
236
236
|
BEFORE the answer, because that snapshot is what Studio re-checks
|
|
237
|
-
before it removes anything; the answer says how the push went.
|
|
237
|
+
before it removes anything; the answer says how the push went.
|
|
238
|
+
|
|
239
|
+
A refused drain lowers the flag, not just its own raise: the clear is
|
|
240
|
+
unscoped (`State.clear_drain`). A refused RE-drain therefore also
|
|
241
|
+
lowers an earlier drain that held and had answered `drained: true`,
|
|
242
|
+
and so does a second, overlapping `/action/drain` that is refused.
|
|
243
|
+
That is intended (seat-parking §2.4: when the block breaks D2 "the
|
|
244
|
+
seat clears the flag itself and answers `drained: false`"): a seat
|
|
245
|
+
that is no longer idle must not stay drained, whoever raised the
|
|
246
|
+
flag."""
|
|
238
247
|
if ttl_s is None:
|
|
239
248
|
ttl = parking.DRAIN_TTL_DEFAULT
|
|
240
249
|
elif (
|
|
@@ -265,7 +274,7 @@ class App:
|
|
|
265
274
|
except sqlite3.Error as exc:
|
|
266
275
|
self._audit("drain", {"ok": False, "error": str(exc)})
|
|
267
276
|
return 503, {"drained": False, "error": "state_unavailable"}
|
|
268
|
-
listing = self.sources.
|
|
277
|
+
listing = self.sources.listing()
|
|
269
278
|
idle = self.sources.idle(listing=listing)
|
|
270
279
|
refusals = parking.idle_refusals(
|
|
271
280
|
idle, poll_interval=self.sources.config.poll_interval, now=now,
|
|
@@ -305,13 +314,18 @@ class App:
|
|
|
305
314
|
def undrain(self) -> "tuple[int, dict]":
|
|
306
315
|
"""`POST /action/undrain`: lower the drain flag -- orcloop's step 4
|
|
307
316
|
on any failure of a park. Idempotent: undraining a seat that is not
|
|
308
|
-
drained answers `cleared: false`. (status, payload), audited.
|
|
317
|
+
drained answers `cleared: false`. (status, payload), audited.
|
|
318
|
+
|
|
319
|
+
A ledger that cannot be written answers 503 with `drained: null`:
|
|
320
|
+
nobody knows then whether the flag is up (there may be no drain at
|
|
321
|
+
all), and unknown is never a value (design D2). The 503 already says
|
|
322
|
+
the undrain did not land; the TTL still bounds any flag left up."""
|
|
309
323
|
try:
|
|
310
324
|
with State(self.sources.config.state_db) as st:
|
|
311
325
|
cleared = st.clear_drain()
|
|
312
326
|
except sqlite3.Error as exc:
|
|
313
327
|
self._audit("undrain", {"ok": False, "error": str(exc)})
|
|
314
|
-
return 503, {"drained":
|
|
328
|
+
return 503, {"drained": None, "error": "state_unavailable"}
|
|
315
329
|
self._audit("undrain", {"ok": True, "cleared": cleared})
|
|
316
330
|
return 200, {"drained": False, "cleared": cleared}
|
|
317
331
|
|
|
@@ -384,6 +384,15 @@ class Sources:
|
|
|
384
384
|
except ValueError:
|
|
385
385
|
return None
|
|
386
386
|
|
|
387
|
+
def listing(self) -> "object":
|
|
388
|
+
"""The parsed ``alissa tmux ls --json`` listing, or None when it could
|
|
389
|
+
not be read (the command failed, printed nothing, or printed something
|
|
390
|
+
that is not JSON). The console's one reader of the tmux roster: the
|
|
391
|
+
sessions panel, the idle block, the dashboard and the drain handshake
|
|
392
|
+
all read it here. The value is whatever the CLI printed -- callers
|
|
393
|
+
check it is a list before trusting its shape."""
|
|
394
|
+
return self._safe_json(["alissa", "tmux", "ls", "--json"])
|
|
395
|
+
|
|
387
396
|
def sessions(self, listing: "list[dict] | None" = None) -> "list[dict]":
|
|
388
397
|
"""The managed sessions with their edge, age and `/proc` footprint.
|
|
389
398
|
|
|
@@ -393,9 +402,7 @@ class Sources:
|
|
|
393
402
|
console (no listing) lists for itself. Either way the rows go
|
|
394
403
|
through the same shaping, so the two never disagree about what a
|
|
395
404
|
session record looks like."""
|
|
396
|
-
raw = listing if listing is not None else (
|
|
397
|
-
self._safe_json(["alissa", "tmux", "ls", "--json"]) or []
|
|
398
|
-
)
|
|
405
|
+
raw = listing if listing is not None else (self.listing() or [])
|
|
399
406
|
if not isinstance(raw, list):
|
|
400
407
|
return []
|
|
401
408
|
now = int(self._wall())
|
|
@@ -463,7 +470,7 @@ class Sources:
|
|
|
463
470
|
snaps = self.snapshots(1)
|
|
464
471
|
latest = snaps[0] if snaps else None
|
|
465
472
|
if listing is _UNREAD:
|
|
466
|
-
listing = self.
|
|
473
|
+
listing = self.listing()
|
|
467
474
|
roster = listing if isinstance(listing, list) else None
|
|
468
475
|
snapshot = latest if isinstance(latest, dict) else None
|
|
469
476
|
queues = self._safe_json(["alissa", "tmux", "queue", "ls", "--json"])
|
|
@@ -678,7 +685,7 @@ class Sources:
|
|
|
678
685
|
latest = snaps[0] if snaps else None
|
|
679
686
|
ledgers = self.ledgers()
|
|
680
687
|
# One listing feeds both the sessions panel and the idle block.
|
|
681
|
-
listing = self.
|
|
688
|
+
listing = self.listing()
|
|
682
689
|
sessions = self.sessions(
|
|
683
690
|
listing=listing if isinstance(listing, list) else []
|
|
684
691
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: alissa-tools-github-devloop
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.47
|
|
4
4
|
Summary: ALISSA-TOOLS-GITHUB-DEVLOOP
|
|
5
5
|
Home-page: https://alissa.app
|
|
6
6
|
Author: Fahera
|
|
@@ -244,7 +244,46 @@ that alone. The daemon kills the row's own session (`alissa tmux kill
|
|
|
244
244
|
A pane tmux cannot read keeps the defer, a usage-limited session is never
|
|
245
245
|
killed (issue #144), and dry-run only logs. The root README's *Claude
|
|
246
246
|
exited, the session stayed* section has the details. Processes a dead claude started that
|
|
247
|
-
outlived it (re-parented to init) are not reaped — a known follow-up
|
|
247
|
+
outlived it (re-parented to init) are not reaped — a known follow-up; test
|
|
248
|
+
runs are the exception (see *Hung test runs* below).
|
|
249
|
+
|
|
250
|
+
## Hung test runs (issue #174)
|
|
251
|
+
|
|
252
|
+
A worker's test run whose test workers died (esbuild's "The service was
|
|
253
|
+
stopped", EAGAIN on a box out of processes) leaves its `tsx --test` runner
|
|
254
|
+
waiting forever, and a session polling a log for the run's exit waits with
|
|
255
|
+
it — on 2026-10-18 one had waited 55 h. The daemon now stops such runs.
|
|
256
|
+
|
|
257
|
+
- **What is a test run.** A process tree rooted at its topmost `npm test`
|
|
258
|
+
(`npm t`, `npm run test`) or `--test` runner (`node --test`, `tsx --test`,
|
|
259
|
+
`npx tsx --test`, a `timeout … --test` wrapper), read from `/proc`, whose
|
|
260
|
+
cwd or args sit inside a watched hub's task workspace — the janitor's
|
|
261
|
+
`TASK-<n>-<DESC>` grammar. `main/`, `.source/` and anything outside a hub
|
|
262
|
+
are never considered.
|
|
263
|
+
- **Hung.** The tree's summed CPU time (utime+stime of every member, plus
|
|
264
|
+
the reaped children's cutime+cstime) grew by under 5 s across a window of
|
|
265
|
+
`ALISSA_DEV_HUNG_TEST_MINUTES` (default **20**; **0 = off**; runtime env
|
|
266
|
+
only — not a config key, not a build ARG; a value that is not a
|
|
267
|
+
non-negative integer fails the boot by name). A tree younger than the
|
|
268
|
+
window, or one that keeps burning CPU, is left alone.
|
|
269
|
+
- **Stopped at once**, without the window: a tree whose session (the
|
|
270
|
+
`ALISSA_TMUX_SESSION` the seat exported, inherited by detached runs too)
|
|
271
|
+
is gone from a successful `alissa tmux ls`, or whose branch's pull
|
|
272
|
+
requests are all closed with a terminal newest anchor (the janitor's
|
|
273
|
+
`anchor_terminal`); for a tree carrying no seat env, the janitor's
|
|
274
|
+
liveness read of the branch's PR answering DEAD. A listing or a GitHub
|
|
275
|
+
read that failed is never evidence of death.
|
|
276
|
+
- **How.** TERM to every member, KILL to whatever is alive 10 s later; each
|
|
277
|
+
pid is matched on its start time, so a reused pid is never signalled.
|
|
278
|
+
- **Told once.** When the tree's session is live it gets one message
|
|
279
|
+
(`alissa session msg send`) per tree, never repeated: the run hung and
|
|
280
|
+
was stopped; run only the test files the diff touches, each on its own;
|
|
281
|
+
CI on the PR runs the full suite. Every worker directive says the same up
|
|
282
|
+
front: never wait on a full-suite run for more than 20 minutes.
|
|
283
|
+
|
|
284
|
+
Runs at the end of every poll on the janitor's cadence
|
|
285
|
+
(`janitor_interval_minutes`, its own clock, independent of
|
|
286
|
+
`janitor_enabled`); dry-run only logs.
|
|
248
287
|
|
|
249
288
|
## License
|
|
250
289
|
|
{alissa_tools_github_devloop-0.8.45.dist-info → alissa_tools_github_devloop-0.8.47.dist-info}/RECORD
RENAMED
|
@@ -3,30 +3,31 @@ alissa/tools/github/devloop/__main__.py,sha256=5p_SjwpOvdyTA8pPr54gS-RP99itkydVc
|
|
|
3
3
|
alissa/tools/github/devloop/alissa.py,sha256=TJU4eSZc2MvP35HewCwBKvhPAel32vKjtt3ApSPDLrM,18064
|
|
4
4
|
alissa/tools/github/devloop/alissa_client.py,sha256=wmpXYWf9ThcLeYnz6gGeDz7gj5BpYjduuVASMc-QOOY,14873
|
|
5
5
|
alissa/tools/github/devloop/bows.py,sha256=pzDNMRk_S78FX7vMNYMlXHxW6VPHPo1Xd2EzFMgVCUM,17325
|
|
6
|
-
alissa/tools/github/devloop/config.py,sha256=
|
|
6
|
+
alissa/tools/github/devloop/config.py,sha256=ZgGb7DQR3hOOfCXnRIjmG52m3A2SfQHFX94xEJcBzXo,76861
|
|
7
7
|
alissa/tools/github/devloop/fleet_vitals.py,sha256=5mWnRkJh8u9soSplud3UMxkelT6EqAiz-ivX8UmfIqk,25477
|
|
8
8
|
alissa/tools/github/devloop/ghclient.py,sha256=tYUHwmoy9CBBt-jSOz9nuAl8lpMvm1LH4A-wHx25bss,71842
|
|
9
9
|
alissa/tools/github/devloop/hubs.py,sha256=yjnsnSHLG7GNNP8X2hILeAMV6P5n4JGc_IaAxpSP_YQ,18974
|
|
10
|
+
alissa/tools/github/devloop/hungtests.py,sha256=LKBLg0fmswEhGz6GcPWoOx1bDJf0NCBd0UGJLl0K6Yo,18185
|
|
10
11
|
alissa/tools/github/devloop/janitor.py,sha256=R8REoHBeY2cfW7fYJVhKr-F5z9Mjlc-vmv-z4KtvBZc,27360
|
|
11
|
-
alissa/tools/github/devloop/loop.py,sha256=
|
|
12
|
+
alissa/tools/github/devloop/loop.py,sha256=5ZYrsB8JGBECNnmb8xd8Zb70QsFcXQ9Z0vy_vMUFwGw,633862
|
|
12
13
|
alissa/tools/github/devloop/loop_events.py,sha256=pKtBl1HK8gYPB2gKiI29xUkx-JuW2A48KwRlbdcf_lo,37942
|
|
13
|
-
alissa/tools/github/devloop/parking.py,sha256=
|
|
14
|
+
alissa/tools/github/devloop/parking.py,sha256=7h3Gg2XQ0eqlvn0M6K6NeEg9xbqnlUPrZNs9gRofzxc,12410
|
|
14
15
|
alissa/tools/github/devloop/proc.py,sha256=nzfsxJSHFWRQcrXmfv588KazFysv8dVQrhMPH1q5eRk,3134
|
|
15
16
|
alissa/tools/github/devloop/prompts.py,sha256=VzN6ec5eF2wwg7QSCRg52WLcxo-tGbGRgQbc4HSahUc,55169
|
|
16
17
|
alissa/tools/github/devloop/state.py,sha256=uVwhbkogUTjWXXFq-COKzUeYrpuGbHzi05CjfEzAtPA,88668
|
|
17
|
-
alissa/tools/github/devloop/version,sha256=
|
|
18
|
+
alissa/tools/github/devloop/version,sha256=VWqpVUhR_phcVHDPOXJg-3ydEhyvkS-oMkQnmypBdUs,7
|
|
18
19
|
alissa/tools/github/devloop/version.py,sha256=znuuOaitXGrBpbskNy78_rlVpFdtmKmk_3k1ACxZrl0,1418
|
|
19
20
|
alissa/tools/github/devloop/webui/__init__.py,sha256=vdXYoyGYzD7BrJ5lV6XdH8Bpw_pej8-RK0hjfxop7-8,1513
|
|
20
21
|
alissa/tools/github/devloop/webui/__main__.py,sha256=wORyW5AkgsniK0TZcrOrvm_P7Q4Mr-fH-4S6qj9XcgA,6137
|
|
21
22
|
alissa/tools/github/devloop/webui/auth.py,sha256=g9YEDao9nKt-cBEENp7vFqwJ-T8olN6uLiCzVIN8yhA,7890
|
|
22
23
|
alissa/tools/github/devloop/webui/page.py,sha256=4OD3e01ByvVM4Wo04AZyFuyOYTFEMT2lagktZAvyCKk,30862
|
|
23
|
-
alissa/tools/github/devloop/webui/server.py,sha256=
|
|
24
|
-
alissa/tools/github/devloop/webui/sources.py,sha256=
|
|
24
|
+
alissa/tools/github/devloop/webui/server.py,sha256=3aGYGesjUSrPrgcpfy-oYDefBNKnAj3lGfRzwsRkv_I,27031
|
|
25
|
+
alissa/tools/github/devloop/webui/sources.py,sha256=Uw_2B5QGSmW6f66WfVbQb6yLLYmSv6uuF7HNJ5OSr_I,39314
|
|
25
26
|
alissa/tools/github/devloop/webui/sysinfo.py,sha256=ptlogH9alFv0vZ-pQfyUaA3w-K7LQS3tQHGv0d8Tdlk,9649
|
|
26
|
-
alissa_tools_github_devloop-0.8.
|
|
27
|
-
alissa_tools_github_devloop-0.8.
|
|
28
|
-
alissa_tools_github_devloop-0.8.
|
|
29
|
-
alissa_tools_github_devloop-0.8.
|
|
30
|
-
alissa_tools_github_devloop-0.8.
|
|
31
|
-
alissa_tools_github_devloop-0.8.
|
|
32
|
-
alissa_tools_github_devloop-0.8.
|
|
27
|
+
alissa_tools_github_devloop-0.8.47.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
28
|
+
alissa_tools_github_devloop-0.8.47.dist-info/licenses/NOTICE,sha256=FKstNs3uw_tQod4_1XcFLD1FUTJQXf4FC0oXJAOA_HE,260
|
|
29
|
+
alissa_tools_github_devloop-0.8.47.dist-info/METADATA,sha256=ttfErUIcNrjotDmbztORzK-ERI0VjiK9SzoVp5t3fzY,15973
|
|
30
|
+
alissa_tools_github_devloop-0.8.47.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
31
|
+
alissa_tools_github_devloop-0.8.47.dist-info/entry_points.txt,sha256=8aC7Ajzx4bu9ZRR37a40IDfuStacvZP5KsAxY1dbFKU,145
|
|
32
|
+
alissa_tools_github_devloop-0.8.47.dist-info/top_level.txt,sha256=DodjDg-l-TWQnlxG-Vuc3G5sB4O7cWGQwe6XrVx3u-A,7
|
|
33
|
+
alissa_tools_github_devloop-0.8.47.dist-info/RECORD,,
|
{alissa_tools_github_devloop-0.8.45.dist-info → alissa_tools_github_devloop-0.8.47.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|