alissa-tools-github-devloop 0.8.39__py3-none-any.whl → 0.8.41__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alissa/tools/github/devloop/config.py +36 -0
- alissa/tools/github/devloop/loop.py +208 -19
- alissa/tools/github/devloop/proc.py +35 -0
- alissa/tools/github/devloop/version +1 -1
- {alissa_tools_github_devloop-0.8.39.dist-info → alissa_tools_github_devloop-0.8.41.dist-info}/METADATA +1 -1
- {alissa_tools_github_devloop-0.8.39.dist-info → alissa_tools_github_devloop-0.8.41.dist-info}/RECORD +11 -11
- {alissa_tools_github_devloop-0.8.39.dist-info → alissa_tools_github_devloop-0.8.41.dist-info}/WHEEL +0 -0
- {alissa_tools_github_devloop-0.8.39.dist-info → alissa_tools_github_devloop-0.8.41.dist-info}/entry_points.txt +0 -0
- {alissa_tools_github_devloop-0.8.39.dist-info → alissa_tools_github_devloop-0.8.41.dist-info}/licenses/LICENSE +0 -0
- {alissa_tools_github_devloop-0.8.39.dist-info → alissa_tools_github_devloop-0.8.41.dist-info}/licenses/NOTICE +0 -0
- {alissa_tools_github_devloop-0.8.39.dist-info → alissa_tools_github_devloop-0.8.41.dist-info}/top_level.txt +0 -0
|
@@ -284,6 +284,8 @@ CONFIG_KEYS = (
|
|
|
284
284
|
"attempt_cap",
|
|
285
285
|
"stale_minutes",
|
|
286
286
|
"max_sessions",
|
|
287
|
+
"spawn_headroom_threads",
|
|
288
|
+
"spawn_headroom_memory_pct",
|
|
287
289
|
"reap_grace_minutes",
|
|
288
290
|
"foreign_session_policy",
|
|
289
291
|
"foreign_session_allow_prefixes",
|
|
@@ -535,6 +537,19 @@ class Config:
|
|
|
535
537
|
# listing, so enabling it costs no extra GitHub or CLI calls.
|
|
536
538
|
max_sessions: int = 0
|
|
537
539
|
|
|
540
|
+
# Headroom-aware admission (issue #162): a session count does not bound
|
|
541
|
+
# what the sessions RUN -- an idle claude holds ~16 threads, a test suite
|
|
542
|
+
# or a Chromium screenshot 100-200+ -- and the container's cgroup v2
|
|
543
|
+
# `pids.max` counts THREADS. When `pids.max - pids.current` falls under
|
|
544
|
+
# `spawn_headroom_threads`, or the free share of `memory.max` under
|
|
545
|
+
# `spawn_headroom_memory_pct`, a pass spawns nothing new: every spawn
|
|
546
|
+
# (resumes and fix rounds included) defers on the capacity valve and is
|
|
547
|
+
# re-decided next poll. 0 turns either guard off; an unlimited (`max`) or
|
|
548
|
+
# unreadable cgroup file turns its guard off too. The 2026-10-13 outage
|
|
549
|
+
# (a `gh` fork failing EAGAIN at ~1000 threads) set the defaults.
|
|
550
|
+
spawn_headroom_threads: int = 300
|
|
551
|
+
spawn_headroom_memory_pct: int = 15
|
|
552
|
+
|
|
538
553
|
# The reaper's grace: how long a worker session must have been QUIET (no
|
|
539
554
|
# tmux activity) before a terminal anchor licenses killing it. The point
|
|
540
555
|
# is the window right after an anchor goes terminal -- a PR merges while
|
|
@@ -1154,6 +1169,25 @@ class Config:
|
|
|
1154
1169
|
f"max_sessions must be >= 0 (0 = unlimited), got {max_sessions}"
|
|
1155
1170
|
)
|
|
1156
1171
|
|
|
1172
|
+
headroom_threads = int(
|
|
1173
|
+
raw.get("spawn_headroom_threads", cls.spawn_headroom_threads)
|
|
1174
|
+
)
|
|
1175
|
+
if headroom_threads < 0:
|
|
1176
|
+
raise ValueError(
|
|
1177
|
+
f"spawn_headroom_threads must be >= 0 (0 = off), got "
|
|
1178
|
+
f"{headroom_threads}"
|
|
1179
|
+
)
|
|
1180
|
+
headroom_memory = int(
|
|
1181
|
+
raw.get("spawn_headroom_memory_pct", cls.spawn_headroom_memory_pct)
|
|
1182
|
+
)
|
|
1183
|
+
# Capped at 90: a reserve of the whole limit would hold every spawn
|
|
1184
|
+
# for ever on any container that has started anything at all.
|
|
1185
|
+
if not 0 <= headroom_memory <= 90:
|
|
1186
|
+
raise ValueError(
|
|
1187
|
+
f"spawn_headroom_memory_pct must be between 0 and 90 (0 = "
|
|
1188
|
+
f"off), got {headroom_memory}"
|
|
1189
|
+
)
|
|
1190
|
+
|
|
1157
1191
|
# The reaper's grace, validated like the two below it -- same 0
|
|
1158
1192
|
# sentinel, and a negative value is the only invalid one (it would
|
|
1159
1193
|
# make every listed session instantly reapable, which is the one
|
|
@@ -1340,6 +1374,8 @@ class Config:
|
|
|
1340
1374
|
attempt_cap=cap,
|
|
1341
1375
|
stale_minutes=stale,
|
|
1342
1376
|
max_sessions=max_sessions,
|
|
1377
|
+
spawn_headroom_threads=headroom_threads,
|
|
1378
|
+
spawn_headroom_memory_pct=headroom_memory,
|
|
1343
1379
|
reap_grace_minutes=reap_grace,
|
|
1344
1380
|
foreign_session_policy=foreign_policy,
|
|
1345
1381
|
foreign_session_allow_prefixes=normalize_session_prefixes(
|
|
@@ -464,6 +464,7 @@ from . import janitor
|
|
|
464
464
|
from . import fleet_vitals as fleet_vitals_mod
|
|
465
465
|
from . import loop_events as loop_events_mod
|
|
466
466
|
from . import parking as parking_mod
|
|
467
|
+
from . import proc as proc_mod
|
|
467
468
|
from . import prompts as prompts_mod
|
|
468
469
|
from .alissa import MANAGED_PREFIX, SHELL_COMMANDS, Alissa
|
|
469
470
|
from .alissa_client import AlissaClient
|
|
@@ -2689,6 +2690,112 @@ class DiskReport:
|
|
|
2689
2690
|
offenders: "tuple[tuple[str, int], ...]" = ()
|
|
2690
2691
|
|
|
2691
2692
|
|
|
2693
|
+
# The container's cgroup v2 mount (issue #162). A module global, read at each
|
|
2694
|
+
# call, so a test can point every watcher at a fake cgroup directory.
|
|
2695
|
+
CGROUP_ROOT = Path("/sys/fs/cgroup")
|
|
2696
|
+
|
|
2697
|
+
|
|
2698
|
+
def _cgroup_value(path: Path) -> "int | str | None":
|
|
2699
|
+
"""One cgroup v2 counter file: its integer, the string "max" (no limit),
|
|
2700
|
+
or None when it is absent, unreadable, or malformed."""
|
|
2701
|
+
try:
|
|
2702
|
+
raw = path.read_text().split()[0]
|
|
2703
|
+
except (OSError, ValueError, IndexError):
|
|
2704
|
+
return None
|
|
2705
|
+
if raw == "max":
|
|
2706
|
+
return "max"
|
|
2707
|
+
try:
|
|
2708
|
+
return int(raw)
|
|
2709
|
+
except ValueError:
|
|
2710
|
+
return None
|
|
2711
|
+
|
|
2712
|
+
|
|
2713
|
+
def _cgroup_stat(path: Path, field: str) -> int:
|
|
2714
|
+
"""One `key value` line of a cgroup `memory.stat`; 0 when unreadable --
|
|
2715
|
+
it only ever ADDS headroom (reclaimable page cache), so a missing stat
|
|
2716
|
+
reads the conservative way."""
|
|
2717
|
+
try:
|
|
2718
|
+
for line in path.read_text().splitlines():
|
|
2719
|
+
key, _, value = line.partition(" ")
|
|
2720
|
+
if key == field:
|
|
2721
|
+
return int(value.strip())
|
|
2722
|
+
except (OSError, ValueError):
|
|
2723
|
+
return 0
|
|
2724
|
+
return 0
|
|
2725
|
+
|
|
2726
|
+
|
|
2727
|
+
@dataclass(frozen=True)
|
|
2728
|
+
class Headroom:
|
|
2729
|
+
"""One pass's cgroup headroom reading (issue #162).
|
|
2730
|
+
|
|
2731
|
+
`low` names every guard whose reserve is breached, in the poll-summary
|
|
2732
|
+
wording (`threads 812/1000, reserve 300`), and is empty when the pass may
|
|
2733
|
+
spawn. `unreadable` names every ENABLED guard whose files could not be
|
|
2734
|
+
read: that guard is off this pass (failing open, like the max_sessions
|
|
2735
|
+
budget on a listing failure -- a missing file must not freeze spawning)
|
|
2736
|
+
and the watcher warns about it once."""
|
|
2737
|
+
|
|
2738
|
+
low: "tuple[str, ...]" = ()
|
|
2739
|
+
unreadable: "tuple[str, ...]" = ()
|
|
2740
|
+
|
|
2741
|
+
@property
|
|
2742
|
+
def reason(self) -> "str | None":
|
|
2743
|
+
"""`headroom (threads 812/1000, reserve 300)`, or None when no
|
|
2744
|
+
reserve is breached."""
|
|
2745
|
+
if not self.low:
|
|
2746
|
+
return None
|
|
2747
|
+
return f"headroom ({'; '.join(self.low)})"
|
|
2748
|
+
|
|
2749
|
+
|
|
2750
|
+
def read_headroom(
|
|
2751
|
+
threads_reserve: int,
|
|
2752
|
+
memory_pct: int,
|
|
2753
|
+
cgroup_root: "Path | None" = None,
|
|
2754
|
+
) -> Headroom:
|
|
2755
|
+
"""Read cgroup v2 `pids.current`/`pids.max` and `memory.current`/
|
|
2756
|
+
`memory.max` and judge them against the two reserves. A reserve of 0 is
|
|
2757
|
+
that guard off (its files are not even read); a `max` limit is unlimited,
|
|
2758
|
+
so that guard is off too. `pids.max` counts THREADS, not processes --
|
|
2759
|
+
which is why a session count alone never bounded it. Free memory is the
|
|
2760
|
+
limit minus what is charged, with the inactive page cache the kernel
|
|
2761
|
+
would drop under pressure (`memory.stat inactive_file`) counted as free:
|
|
2762
|
+
a container that just walked a few big checkouts is not out of memory."""
|
|
2763
|
+
root = CGROUP_ROOT if cgroup_root is None else cgroup_root
|
|
2764
|
+
low: "list[str]" = []
|
|
2765
|
+
unreadable: "list[str]" = []
|
|
2766
|
+
if threads_reserve > 0:
|
|
2767
|
+
current = _cgroup_value(root / "pids.current")
|
|
2768
|
+
limit = _cgroup_value(root / "pids.max")
|
|
2769
|
+
if not isinstance(current, int) or limit is None:
|
|
2770
|
+
unreadable.append("threads")
|
|
2771
|
+
elif isinstance(limit, int) and limit - current < threads_reserve:
|
|
2772
|
+
low.append(f"threads {current}/{limit}, reserve {threads_reserve}")
|
|
2773
|
+
if memory_pct > 0:
|
|
2774
|
+
current = _cgroup_value(root / "memory.current")
|
|
2775
|
+
limit = _cgroup_value(root / "memory.max")
|
|
2776
|
+
if not isinstance(current, int) or limit is None:
|
|
2777
|
+
unreadable.append("memory")
|
|
2778
|
+
elif isinstance(limit, int) and limit > 0:
|
|
2779
|
+
charged = max(0, current - _cgroup_stat(root / "memory.stat", "inactive_file"))
|
|
2780
|
+
free = max(0, limit - charged)
|
|
2781
|
+
if free * 100 < memory_pct * limit:
|
|
2782
|
+
low.append(
|
|
2783
|
+
f"memory {free * 100 // limit}% free of "
|
|
2784
|
+
f"{limit / (1 << 30):.1f} GiB, reserve {memory_pct}%"
|
|
2785
|
+
)
|
|
2786
|
+
return Headroom(low=tuple(low), unreadable=tuple(unreadable))
|
|
2787
|
+
|
|
2788
|
+
|
|
2789
|
+
def cgroup_threads(cgroup_root: "Path | None" = None) -> "str | None":
|
|
2790
|
+
"""`pids 998/1000` for a log line, or None when unreadable."""
|
|
2791
|
+
root = CGROUP_ROOT if cgroup_root is None else cgroup_root
|
|
2792
|
+
current = _cgroup_value(root / "pids.current")
|
|
2793
|
+
limit = _cgroup_value(root / "pids.max")
|
|
2794
|
+
if not isinstance(current, int) or limit is None:
|
|
2795
|
+
return None
|
|
2796
|
+
return f"pids {current}/{limit}"
|
|
2797
|
+
|
|
2798
|
+
|
|
2692
2799
|
class SessionBudget:
|
|
2693
2800
|
"""The per-pass concurrency budget behind `max_sessions`.
|
|
2694
2801
|
|
|
@@ -2727,10 +2834,18 @@ class SessionBudget:
|
|
|
2727
2834
|
hold: "str | None" = None,
|
|
2728
2835
|
probe: bool = False,
|
|
2729
2836
|
limited: int = 0,
|
|
2837
|
+
headroom: "str | None" = None,
|
|
2730
2838
|
):
|
|
2731
2839
|
self.limit = limit
|
|
2732
2840
|
self.active = active
|
|
2733
2841
|
self.reserved = 0
|
|
2842
|
+
# The cgroup headroom guard (issue #162): while the container's
|
|
2843
|
+
# thread or memory reserve is breached, every reserve() is refused --
|
|
2844
|
+
# resumes, fix rounds and the usage-limit probe included, since each
|
|
2845
|
+
# is a new session with the same footprint. It defers on the
|
|
2846
|
+
# capacity valve (claims nothing, re-decided next poll); `headroom`
|
|
2847
|
+
# is the reason, `headroom (threads 812/1000, reserve 300)`.
|
|
2848
|
+
self.headroom = headroom
|
|
2734
2849
|
# The prompt responder's spawn hold (issue #127): while an account-
|
|
2735
2850
|
# level notice (an expired login, a hit limit, no credits) is on a
|
|
2736
2851
|
# worker's screen, NEW sessions would only burn attempts against a
|
|
@@ -2763,7 +2878,10 @@ class SessionBudget:
|
|
|
2763
2878
|
def reserve(self) -> bool:
|
|
2764
2879
|
"""Claim one slot if the cap allows. Unlimited always grants -- unless
|
|
2765
2880
|
the responder's hold stands, which refuses every spawn but the one
|
|
2766
|
-
probe a usage-limit hold admits (issue #144).
|
|
2881
|
+
probe a usage-limit hold admits (issue #144). A breached cgroup
|
|
2882
|
+
headroom reserve refuses everything, the probe too (issue #162)."""
|
|
2883
|
+
if self.headroom:
|
|
2884
|
+
return False
|
|
2767
2885
|
if self.hold:
|
|
2768
2886
|
if not self.probe or self._probe_taken:
|
|
2769
2887
|
return False
|
|
@@ -2797,13 +2915,20 @@ class SessionBudget:
|
|
|
2797
2915
|
def defer_token(self) -> "str | None":
|
|
2798
2916
|
"""The loop-events defer token a DEFERRED decision off this budget
|
|
2799
2917
|
carries: the hold's own token while it stands, else None (the
|
|
2800
|
-
emitter reads None as the capacity valve).
|
|
2918
|
+
emitter reads None as the capacity valve). A headroom deferral is
|
|
2919
|
+
the capacity valve too (issue #162), even under a hold: the guard
|
|
2920
|
+
refused it first."""
|
|
2921
|
+
if self.headroom:
|
|
2922
|
+
return None
|
|
2801
2923
|
return loop_events_mod.DEFER_PROMPT_HOLD if self.hold else None
|
|
2802
2924
|
|
|
2803
2925
|
def deferral(self) -> str:
|
|
2804
2926
|
"""The DEFERRED decision's reason -- `at capacity k/k`, the operator's
|
|
2805
2927
|
signal that the cap (not per-item state) is what held the spawn --
|
|
2806
|
-
or the responder's hold, which names the notice standing in the way
|
|
2928
|
+
or the responder's hold, which names the notice standing in the way,
|
|
2929
|
+
or the cgroup headroom reserve the container is under (issue #162)."""
|
|
2930
|
+
if self.headroom:
|
|
2931
|
+
return f"{self.headroom}; retrying next poll"
|
|
2807
2932
|
if self.hold:
|
|
2808
2933
|
return f"spawns held by the prompt responder — {self.hold}"
|
|
2809
2934
|
return f"at capacity {self.used}/{self.limit}; retrying next poll"
|
|
@@ -3408,6 +3533,10 @@ class DevWatcher:
|
|
|
3408
3533
|
# The reaper's warn-once latch for a CLI without `tmux kill` (kills
|
|
3409
3534
|
# are best-effort; the condition is per-process, not per-session).
|
|
3410
3535
|
self._kill_warned = False
|
|
3536
|
+
# The headroom guard's warn-once latch (issue #162): the guards whose
|
|
3537
|
+
# cgroup files could not be read. Per-process like the one above --
|
|
3538
|
+
# an unreadable cgroup does not become readable between polls.
|
|
3539
|
+
self._headroom_warned: "set[str]" = set()
|
|
3411
3540
|
# The BOW-derived allowlist (`repos_source: "bows"`), or None in
|
|
3412
3541
|
# static mode -- in which nothing below this line ever runs and the
|
|
3413
3542
|
# allowlist is `config.repos`, unchanged.
|
|
@@ -10234,8 +10363,14 @@ class DevWatcher:
|
|
|
10234
10363
|
roster fails CLOSED in the same breath -- an unlisted roster answers
|
|
10235
10364
|
INDETERMINATE, so a hiccup can never manufacture a resume.
|
|
10236
10365
|
`max_sessions=0` also yields an unlimited budget -- the default, so
|
|
10237
|
-
nothing changes until an operator opts in.
|
|
10366
|
+
nothing changes until an operator opts in.
|
|
10367
|
+
|
|
10368
|
+
The cgroup headroom guard (issue #162) is read here too, from files
|
|
10369
|
+
rather than the listing, so it stands through a listing failure: a
|
|
10370
|
+
container out of threads is exactly the one whose `tmux ls` fork
|
|
10371
|
+
fails, and failing the budget open THEN would spawn into the wall."""
|
|
10238
10372
|
limit = self.config.max_sessions or None
|
|
10373
|
+
headroom = self._read_headroom()
|
|
10239
10374
|
try:
|
|
10240
10375
|
sessions = self.alissa.list_sessions()
|
|
10241
10376
|
except CommandError as exc:
|
|
@@ -10250,7 +10385,7 @@ class DevWatcher:
|
|
|
10250
10385
|
limit_hold, _probe = self._limit_gate(None)
|
|
10251
10386
|
return (
|
|
10252
10387
|
Sweep(reaped=0, names=set(), hold=limit_hold),
|
|
10253
|
-
SessionBudget(None, 0, hold=limit_hold),
|
|
10388
|
+
SessionBudget(None, 0, hold=limit_hold, headroom=headroom),
|
|
10254
10389
|
SessionLiveness(None),
|
|
10255
10390
|
)
|
|
10256
10391
|
sweep = self._reap_sessions(sessions)
|
|
@@ -10295,10 +10430,60 @@ class DevWatcher:
|
|
|
10295
10430
|
)
|
|
10296
10431
|
return (
|
|
10297
10432
|
sweep,
|
|
10298
|
-
SessionBudget(
|
|
10433
|
+
SessionBudget(
|
|
10434
|
+
limit, active, hold=sweep.hold, probe=probe, limited=limited,
|
|
10435
|
+
headroom=headroom,
|
|
10436
|
+
),
|
|
10299
10437
|
SessionLiveness(sessions, gone),
|
|
10300
10438
|
)
|
|
10301
10439
|
|
|
10440
|
+
def _read_headroom(self) -> "str | None":
|
|
10441
|
+
"""The pass's headroom deferral reason, or None when the pass may
|
|
10442
|
+
spawn (issue #162). Logs the breach at WARNING (the operator's signal
|
|
10443
|
+
that the container, not the session cap, is what holds the fleet),
|
|
10444
|
+
and an unreadable guard ONCE per process -- that guard is off."""
|
|
10445
|
+
reading = read_headroom(
|
|
10446
|
+
self.config.spawn_headroom_threads,
|
|
10447
|
+
self.config.spawn_headroom_memory_pct,
|
|
10448
|
+
)
|
|
10449
|
+
for guard in reading.unreadable:
|
|
10450
|
+
if guard in self._headroom_warned:
|
|
10451
|
+
continue
|
|
10452
|
+
self._headroom_warned.add(guard)
|
|
10453
|
+
log.warning(
|
|
10454
|
+
"headroom: the cgroup v2 %s files under %s cannot be read — "
|
|
10455
|
+
"the %s headroom guard is OFF (spawns are not held on it)",
|
|
10456
|
+
guard, CGROUP_ROOT,
|
|
10457
|
+
"spawn_headroom_threads" if guard == "threads"
|
|
10458
|
+
else "spawn_headroom_memory_pct",
|
|
10459
|
+
)
|
|
10460
|
+
if reading.reason:
|
|
10461
|
+
log.warning(
|
|
10462
|
+
"%s: HOLDING new worker spawns this pass (resumes and fix "
|
|
10463
|
+
"rounds included); live sessions keep running and the next "
|
|
10464
|
+
"poll re-reads the cgroup", reading.reason,
|
|
10465
|
+
)
|
|
10466
|
+
return reading.reason
|
|
10467
|
+
|
|
10468
|
+
def _warn_start_failures(self) -> None:
|
|
10469
|
+
"""ONE warning per poll for every command that could not even start
|
|
10470
|
+
(issue #162): the fork or exec failed -- EAGAIN at the cgroup's
|
|
10471
|
+
pids.max on 2026-10-13 -- and `proc.run` raised it as the
|
|
10472
|
+
CommandError its caller already handles. Names the errno(s) and the
|
|
10473
|
+
thread budget when the cgroup reads."""
|
|
10474
|
+
failures = proc_mod.take_start_failures()
|
|
10475
|
+
if not failures:
|
|
10476
|
+
return
|
|
10477
|
+
names = ", ".join(sorted(Counter(failures)))
|
|
10478
|
+
threads = cgroup_threads()
|
|
10479
|
+
log.warning(
|
|
10480
|
+
"%d command(s) could not start this poll (%s)%s — the container "
|
|
10481
|
+
"is likely at its process/thread or memory limit; the poll's "
|
|
10482
|
+
"failure paths handled each one",
|
|
10483
|
+
len(failures), names,
|
|
10484
|
+
f"; cgroup {threads}" if threads else "",
|
|
10485
|
+
)
|
|
10486
|
+
|
|
10302
10487
|
def _check_session_cap(self, active: int, sweep: Sweep) -> None:
|
|
10303
10488
|
"""Page-worthy log when the machine is STILL over `max_sessions` after
|
|
10304
10489
|
the sweep.
|
|
@@ -11259,7 +11444,7 @@ class DevWatcher:
|
|
|
11259
11444
|
"poll summary: %d candidate(s) — %d spawned, %d in-flight, "
|
|
11260
11445
|
"%d skipped, %d escalated, %d capped, %d deferred; "
|
|
11261
11446
|
"%d worker session(s) reaped, %d foreign; %d parked on a prompt, "
|
|
11262
|
-
"%d answered%s; vitals: %s",
|
|
11447
|
+
"%d answered%s%s; vitals: %s",
|
|
11263
11448
|
len(results),
|
|
11264
11449
|
counts[Action.SPAWNED],
|
|
11265
11450
|
counts[Action.IN_FLIGHT],
|
|
@@ -11272,6 +11457,7 @@ class DevWatcher:
|
|
|
11272
11457
|
len(sweep.waiting),
|
|
11273
11458
|
sweep.answered,
|
|
11274
11459
|
" (spawns HELD)" if sweep.hold else "",
|
|
11460
|
+
f"; deferred: {budget.headroom}" if budget.headroom else "",
|
|
11275
11461
|
vitals,
|
|
11276
11462
|
)
|
|
11277
11463
|
return results
|
|
@@ -11280,21 +11466,19 @@ class DevWatcher:
|
|
|
11280
11466
|
"""The pass's fleet-vitals push (issue #116): one console snapshot
|
|
11281
11467
|
to Studio, built from the pass's own roster and the ledgers the pass
|
|
11282
11468
|
just wrote. Returns the outcome token the summary quotes --
|
|
11283
|
-
`skipped` when the feature is off
|
|
11284
|
-
|
|
11285
|
-
|
|
11286
|
-
|
|
11469
|
+
`skipped` when the feature is off, else the pusher's `pushed` /
|
|
11470
|
+
`failed`. Runs AFTER the loop-events push and never raises.
|
|
11471
|
+
|
|
11472
|
+
dry_run still pushes (issue: managed fleet first heartbeat). Vitals
|
|
11473
|
+
are the seat's liveness report to Studio, not an action on any
|
|
11474
|
+
repository: the managed provisioner deploys devloop dry for the
|
|
11475
|
+
smoke's first phase and waits for its first heartbeat, which a dry
|
|
11476
|
+
pass that sent nothing could never give. revloop already pushes its
|
|
11477
|
+
vitals under dry_run; this makes the seats agree."""
|
|
11287
11478
|
if self.fleet_vitals is None:
|
|
11288
11479
|
return fleet_vitals_mod.SKIPPED
|
|
11289
11480
|
if self.config.dry_run:
|
|
11290
|
-
log.
|
|
11291
|
-
"[dry-run] would push fleet vitals (seat %s, %s roster "
|
|
11292
|
-
"session(s) → POST %s%s)",
|
|
11293
|
-
fleet_vitals_mod.SEAT,
|
|
11294
|
-
len(roster.entries) if roster.entries is not None else "unlisted",
|
|
11295
|
-
self.config.alissa_endpoint, fleet_vitals_mod.ENDPOINT_PATH,
|
|
11296
|
-
)
|
|
11297
|
-
return fleet_vitals_mod.SKIPPED
|
|
11481
|
+
log.debug("[dry-run] pushing fleet vitals anyway: liveness is not a side effect on a repository")
|
|
11298
11482
|
return self.fleet_vitals.push_pass(roster.entries)
|
|
11299
11483
|
|
|
11300
11484
|
def _vitals_config(self) -> dict:
|
|
@@ -11703,6 +11887,11 @@ class DevWatcher:
|
|
|
11703
11887
|
"not a GitHub or auth failure; retrying in %ds",
|
|
11704
11888
|
exc, backoff,
|
|
11705
11889
|
)
|
|
11890
|
+
finally:
|
|
11891
|
+
# After whichever branch ran, before the sleep: a fork
|
|
11892
|
+
# failure is usually caught deep in the pass, so a
|
|
11893
|
+
# "successful" poll is where it most often surfaces.
|
|
11894
|
+
self._warn_start_failures()
|
|
11706
11895
|
time.sleep(backoff)
|
|
11707
11896
|
except KeyboardInterrupt:
|
|
11708
11897
|
log.info("stopping")
|
|
@@ -3,6 +3,7 @@ their own auth handling; we never touch tokens ourselves."""
|
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
|
+
import errno
|
|
6
7
|
import json
|
|
7
8
|
import logging
|
|
8
9
|
import os
|
|
@@ -20,6 +21,29 @@ class CommandError(RuntimeError):
|
|
|
20
21
|
super().__init__(f"{argv[0]} exited {returncode}: {stderr.strip()[:400]}")
|
|
21
22
|
|
|
22
23
|
|
|
24
|
+
# The errno names of every command that could not even START (fork/exec
|
|
25
|
+
# failed) since the last take_start_failures(). Issue #162: at the container's
|
|
26
|
+
# cgroup pids.max a fork raises BlockingIOError (EAGAIN), which run() now
|
|
27
|
+
# turns into a CommandError so each caller's existing failure path handles it;
|
|
28
|
+
# the daemon loop drains this list once per poll to log ONE warning naming
|
|
29
|
+
# the errno and the thread budget, instead of one per failed command.
|
|
30
|
+
_start_failures: "list[str]" = []
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _errno_name(exc: OSError) -> str:
|
|
34
|
+
if exc.errno is None:
|
|
35
|
+
return type(exc).__name__
|
|
36
|
+
return errno.errorcode.get(exc.errno, str(exc.errno))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def take_start_failures() -> "list[str]":
|
|
40
|
+
"""The errno names of the commands that failed to start since the last
|
|
41
|
+
call, oldest first; clears the list."""
|
|
42
|
+
taken = list(_start_failures)
|
|
43
|
+
_start_failures.clear()
|
|
44
|
+
return taken
|
|
45
|
+
|
|
46
|
+
|
|
23
47
|
def run(
|
|
24
48
|
argv: Sequence[str],
|
|
25
49
|
*,
|
|
@@ -39,6 +63,17 @@ def run(
|
|
|
39
63
|
)
|
|
40
64
|
except subprocess.TimeoutExpired as exc:
|
|
41
65
|
raise CommandError(argv, -1, f"timed out after {timeout}s") from exc
|
|
66
|
+
except OSError as exc:
|
|
67
|
+
# The command never ran: the fork or the exec failed (EAGAIN at the
|
|
68
|
+
# cgroup's pids.max, ENOMEM, a missing binary). Raised as the
|
|
69
|
+
# CommandError every caller already handles -- the 2026-10-13 outage
|
|
70
|
+
# was a BlockingIOError escaping run_forever, which only catches
|
|
71
|
+
# CommandError, and taking the whole daemon down with it.
|
|
72
|
+
name = _errno_name(exc)
|
|
73
|
+
_start_failures.append(name)
|
|
74
|
+
raise CommandError(
|
|
75
|
+
argv, -1, f"could not start {argv[0]}: {name}: {exc.strerror or exc}"
|
|
76
|
+
) from exc
|
|
42
77
|
|
|
43
78
|
if check and proc.returncode != 0:
|
|
44
79
|
raise CommandError(argv, proc.returncode, proc.stderr)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
0.8.
|
|
1
|
+
0.8.41
|
{alissa_tools_github_devloop-0.8.39.dist-info → alissa_tools_github_devloop-0.8.41.dist-info}/RECORD
RENAMED
|
@@ -3,18 +3,18 @@ alissa/tools/github/devloop/__main__.py,sha256=5p_SjwpOvdyTA8pPr54gS-RP99itkydVc
|
|
|
3
3
|
alissa/tools/github/devloop/alissa.py,sha256=TJU4eSZc2MvP35HewCwBKvhPAel32vKjtt3ApSPDLrM,18064
|
|
4
4
|
alissa/tools/github/devloop/alissa_client.py,sha256=wmpXYWf9ThcLeYnz6gGeDz7gj5BpYjduuVASMc-QOOY,14873
|
|
5
5
|
alissa/tools/github/devloop/bows.py,sha256=pzDNMRk_S78FX7vMNYMlXHxW6VPHPo1Xd2EzFMgVCUM,17325
|
|
6
|
-
alissa/tools/github/devloop/config.py,sha256=
|
|
6
|
+
alissa/tools/github/devloop/config.py,sha256=j6W0Wrz7smFdFdSs3g5S6xadcxfFS5YEhQDL64yAuJk,75109
|
|
7
7
|
alissa/tools/github/devloop/fleet_vitals.py,sha256=Hl2xshweh7A9R0N7fyYRg2WzHN4aNefVlt0TiXCv8zk,24979
|
|
8
8
|
alissa/tools/github/devloop/ghclient.py,sha256=tYUHwmoy9CBBt-jSOz9nuAl8lpMvm1LH4A-wHx25bss,71842
|
|
9
9
|
alissa/tools/github/devloop/hubs.py,sha256=yjnsnSHLG7GNNP8X2hILeAMV6P5n4JGc_IaAxpSP_YQ,18974
|
|
10
10
|
alissa/tools/github/devloop/janitor.py,sha256=R8REoHBeY2cfW7fYJVhKr-F5z9Mjlc-vmv-z4KtvBZc,27360
|
|
11
|
-
alissa/tools/github/devloop/loop.py,sha256=
|
|
11
|
+
alissa/tools/github/devloop/loop.py,sha256=BFU9XgBTColvKwgiczxhWPgACn1LGiszV1HDxTIqaqM,599231
|
|
12
12
|
alissa/tools/github/devloop/loop_events.py,sha256=tmso0OALqY2yAPW2Fo6ae0sYetQIdIeIYooJ2Sc2cc4,37921
|
|
13
13
|
alissa/tools/github/devloop/parking.py,sha256=2Sys1bh_hG8qbq7cO9RxW1F1kAu3dCWHnvwg_iAFVvY,7987
|
|
14
|
-
alissa/tools/github/devloop/proc.py,sha256=
|
|
14
|
+
alissa/tools/github/devloop/proc.py,sha256=nzfsxJSHFWRQcrXmfv588KazFysv8dVQrhMPH1q5eRk,3134
|
|
15
15
|
alissa/tools/github/devloop/prompts.py,sha256=VzN6ec5eF2wwg7QSCRg52WLcxo-tGbGRgQbc4HSahUc,55169
|
|
16
16
|
alissa/tools/github/devloop/state.py,sha256=uVwhbkogUTjWXXFq-COKzUeYrpuGbHzi05CjfEzAtPA,88668
|
|
17
|
-
alissa/tools/github/devloop/version,sha256=
|
|
17
|
+
alissa/tools/github/devloop/version,sha256=QOcLK9uM_DEwqdJtbll5EUYMrrUbOLIguSS-pnllhCs,7
|
|
18
18
|
alissa/tools/github/devloop/version.py,sha256=znuuOaitXGrBpbskNy78_rlVpFdtmKmk_3k1ACxZrl0,1418
|
|
19
19
|
alissa/tools/github/devloop/webui/__init__.py,sha256=vdXYoyGYzD7BrJ5lV6XdH8Bpw_pej8-RK0hjfxop7-8,1513
|
|
20
20
|
alissa/tools/github/devloop/webui/__main__.py,sha256=wORyW5AkgsniK0TZcrOrvm_P7Q4Mr-fH-4S6qj9XcgA,6137
|
|
@@ -23,10 +23,10 @@ alissa/tools/github/devloop/webui/page.py,sha256=4OD3e01ByvVM4Wo04AZyFuyOYTFEMT2
|
|
|
23
23
|
alissa/tools/github/devloop/webui/server.py,sha256=YuUmcng5qdWqSurcuyxH3FKKkBBfrpDtvtZLidJOGiU,26226
|
|
24
24
|
alissa/tools/github/devloop/webui/sources.py,sha256=Ud8357-Ag4qC39V3mlepqnArui9p1fTJ058bAdN29Jc,36730
|
|
25
25
|
alissa/tools/github/devloop/webui/sysinfo.py,sha256=ptlogH9alFv0vZ-pQfyUaA3w-K7LQS3tQHGv0d8Tdlk,9649
|
|
26
|
-
alissa_tools_github_devloop-0.8.
|
|
27
|
-
alissa_tools_github_devloop-0.8.
|
|
28
|
-
alissa_tools_github_devloop-0.8.
|
|
29
|
-
alissa_tools_github_devloop-0.8.
|
|
30
|
-
alissa_tools_github_devloop-0.8.
|
|
31
|
-
alissa_tools_github_devloop-0.8.
|
|
32
|
-
alissa_tools_github_devloop-0.8.
|
|
26
|
+
alissa_tools_github_devloop-0.8.41.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
27
|
+
alissa_tools_github_devloop-0.8.41.dist-info/licenses/NOTICE,sha256=FKstNs3uw_tQod4_1XcFLD1FUTJQXf4FC0oXJAOA_HE,260
|
|
28
|
+
alissa_tools_github_devloop-0.8.41.dist-info/METADATA,sha256=Ngui7ERzDLYBQHzZ1Q6mdtFV1mnek3FUeFnwqk5YhG8,8093
|
|
29
|
+
alissa_tools_github_devloop-0.8.41.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
30
|
+
alissa_tools_github_devloop-0.8.41.dist-info/entry_points.txt,sha256=8aC7Ajzx4bu9ZRR37a40IDfuStacvZP5KsAxY1dbFKU,145
|
|
31
|
+
alissa_tools_github_devloop-0.8.41.dist-info/top_level.txt,sha256=DodjDg-l-TWQnlxG-Vuc3G5sB4O7cWGQwe6XrVx3u-A,7
|
|
32
|
+
alissa_tools_github_devloop-0.8.41.dist-info/RECORD,,
|
{alissa_tools_github_devloop-0.8.39.dist-info → alissa_tools_github_devloop-0.8.41.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|