cli-agent-runner 0.2.14__tar.gz → 0.2.16__tar.gz

{cli_agent_runner-0.2.14 → cli_agent_runner-0.2.16}/.vulture-whitelist.py

@@ -148,6 +148,12 @@ supervisor_stale_threshold_s
 mem_avail_min_mb
 disk_warning_pct
 disk_critical_pct
+swap_sout_noise_floor_mb
+mem_free_low_mb
+psi_full_avg10_critical
+psi_some_avg10_warning
+mem_critical_consecutive_samples
+in_round_mem_terminate
 
 # agent_runner.config.models.PhaseOverride
 round_timeout_s

{cli_agent_runner-0.2.14 → cli_agent_runner-0.2.16}/CHANGELOG.md

@@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.16] - 2026-09-05
+
+### Added
+- `[monitor.host_health]` gains PSI knobs — `psi_full_avg10_critical` (default 60.0) and `psi_some_avg10_warning` (default 5.0) — plus `mem_critical_consecutive_samples` (default 3) and an `in_round_mem_terminate` off switch for the mid-round memory floor.
+- `serve` probes this process's cgroup v2 memory budget at startup (`host_cgroup_memory_limit`); when `memory.max` and `memory.swap.max` are both finite, the mid-round floor defers to kernel cgroup-OOM (`mem_pressure_deferred_to_cgroup`) instead of terminating the round itself.
+- `mem_loop` now escalates across restarts: recurring 3 times inside a rolling 2h window stops the service for real (`mem_loop_persistent`, exit `70`) instead of respawning into the same loop forever.
+- New exit-0 no-progress breaker: 5 consecutive fast, clean rounds with no recorded usage (a CLI that exits 0 without ever reaching the model) now stop `serve` (`stalled_no_progress`, exit `75`) instead of spinning invisibly.
+
+### Fixed
+- PSI-full critical was a single ≥1% momentary sample — a hiccup, not a coma — on every PSI-capable host. It now requires a sustained ≥60% avg10 held for `mem_critical_consecutive_samples` (default 3, ~30s) consecutive checks.
+- The mid-round swap-out check is now per-tick instead of cumulative since round-start, so a host that recovers mid-round is no longer permanently flagged for the rest of the round.
+- The pre-round admission gate now defers only on `critical` pressure, not on any pressure (a mere `warning`, e.g. ordinary swap churn, no longer delays the next round).
+
+### Notes
+- New events: `host_cgroup_memory_limit`, `mem_pressure_deferred_to_cgroup`, `mem_loop_persistent`, `stalled_no_progress`, `round_mem_critical_sample`; `round_mem_terminated` gained `consecutive`/`context` fields. Not breaking: no `agent-runner migrate` needed, `peek --json` schema is unchanged.
+- Re-render the systemd unit (`agent-runner install`) after upgrading: the new unit adds exit `70` to `RestartPreventExitStatus` so `mem_loop_persistent` stops the service instead of restarting into the same loop.
+
+See `docs/migrations/0.2.md`.
+
+## [0.2.15] - 2026-09-04
+
+### Added
+- `[monitor.host_health]` gains two config floors — `swap_sout_noise_floor_mb` (default 32) and `mem_free_low_mb` (default 16) — making the 0.2.14 mem-pressure ladder's thresholds operator-tunable; defaults are byte-identical to the prior hardcoded behavior.
+- `serve` now gives up after `MEM_LOOP_THRESHOLD` (5) consecutive mem-terminated rounds instead of retrying forever: emits a new `mem_loop` event and exits `71` — deliberately excluded from the systemd unit's `RestartPreventExitStatus` so the service restarts fresh (break-then-restart).
+
+### Fixed
+- The mem-pressure warning no longer claims "active paging" it can't confirm; it now says only that swap-out crossed the configured noise floor.
+
+### Notes
+- New event kind: `mem_loop`. Not breaking: no `agent-runner migrate` needed this release — nothing that used to load is rejected. `peek --json` schema is unchanged.
+
+See `docs/migrations/0.2.md`.
+
 ## [0.2.14] - 2026-09-03
 
 ### Breaking

{cli_agent_runner-0.2.14 → cli_agent_runner-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cli-agent-runner
-Version: 0.2.14
+Version: 0.2.16
 Summary: Restart-on-exit supervisor for autonomous CLI agents
 Project-URL: Homepage, https://github.com/wan9yu/cli-agent-runner
 Project-URL: Documentation, https://github.com/wan9yu/cli-agent-runner#readme
@@ -94,7 +94,7 @@ Verb reference: [`docs/commands.md`](docs/commands.md).
 
 ## Defenses (built in)
 
-13 named defenses, structured as data — see `agent-runner peek --select defenses`.
+15 named defenses, structured as data — see `agent-runner peek --select defenses`.
 Each carries the historical incident it codifies and the invariant test that
 guards it. Highlights:
 

{cli_agent_runner-0.2.14 → cli_agent_runner-0.2.16}/README.md

@@ -58,7 +58,7 @@ Verb reference: [`docs/commands.md`](docs/commands.md).
 
 ## Defenses (built in)
 
-13 named defenses, structured as data — see `agent-runner peek --select defenses`.
+15 named defenses, structured as data — see `agent-runner peek --select defenses`.
 Each carries the historical incident it codifies and the invariant test that
 guards it. Highlights:
 

{cli_agent_runner-0.2.14 → cli_agent_runner-0.2.16}/agent_runner/_emit.py

@@ -19,12 +19,17 @@ __all__ = [
     "emit_anomaly_repetitive_tool",
     "emit_config_migrated",
     "emit_fresh_eyes_round_triggered",
+    "emit_host_cgroup_memory_limit",
     "emit_max_rounds_reached",
+    "emit_mem_loop",
+    "emit_mem_loop_persistent",
+    "emit_mem_pressure_deferred_to_cgroup",
     "emit_rate_limit_stop",
     "emit_round_deferred",
     "emit_round_grace_extended",
     "emit_round_grace_kill",
     "emit_round_logs_prune_deferred",
+    "emit_round_mem_critical_sample",
     "emit_round_mem_terminated",
     "emit_round_progress",
     "emit_round_resumed",
@@ -35,6 +40,7 @@ __all__ = [
     "emit_schedule_phase_skipped",
     "emit_schedule_resumed",
     "emit_stale_index_lock_cleared",
+    "emit_stalled_no_progress",
     "emit_stop_file_detected",
     "emit_transient_error_backoff_capped",
     "emit_transient_error_detected",
@@ -64,20 +70,103 @@ def emit_config_broken(log_dir: Path, *, reason: str) -> None:
     emit(log_dir, CONFIG_BROKEN, reason=reason)
 
 
-def emit_crash_loop(log_dir: Path, *, consecutive: int, exit_code: int, log_path: Path) -> None:
-    """Emit crash_loop (serve stopped after consecutive unknown short crashes).
-
-    Captures the failure reason — a redacted tail of the round log — so a
-    recurring unknown crash can later be classified into a transient bucket.
+def _emit_giveup(
+    log_dir: Path, kind: str, *, consecutive: int, exit_code: int, log_path: Path
+) -> None:
+    """Shared body for the give-up events (serve stopped/broke after consecutive
+    failures of one kind): capture a redacted tail of the round log as ``reason``
+    so the recurring failure can be inspected (or later classified into a
+    transient bucket), and emit ``kind`` with the standard give-up payload.
     """
     from agent_runner._redact import redact_secrets
-    from agent_runner.events import CRASH_LOOP, emit
+    from agent_runner.events import emit
 
     try:
         reason = redact_secrets(log_path.read_text(encoding="utf-8", errors="replace")[-2000:])
     except OSError:
         reason = ""
-    emit(log_dir, CRASH_LOOP, consecutive=consecutive, exit_code=exit_code, reason=reason)
+    emit(log_dir, kind, consecutive=consecutive, exit_code=exit_code, reason=reason)
+
+
+def emit_crash_loop(log_dir: Path, *, consecutive: int, exit_code: int, log_path: Path) -> None:
+    """Emit crash_loop (serve stopped after consecutive unknown short crashes).
+
+    Captures the failure reason — a redacted tail of the round log — so a
+    recurring unknown crash can later be classified into a transient bucket.
+    """
+    from agent_runner.events import CRASH_LOOP
+
+    _emit_giveup(
+        log_dir, CRASH_LOOP, consecutive=consecutive, exit_code=exit_code, log_path=log_path
+    )
+
+
+def emit_mem_loop(log_dir: Path, *, consecutive: int, exit_code: int, log_path: Path) -> None:
+    """Emit mem_loop (serve gave up after consecutive mid-round memory-pressure
+    terminations — the 0.2.15 coma-preventer's give-up cap). Distinct from
+    crash_loop: this is a break-then-restart, not a deliberate stop, so it is
+    deliberately absent from the unit's RestartPreventExitStatus.
+
+    Captures the failure reason — a redacted tail of the round log — same as
+    crash_loop, so an operator can see what the round was doing when the host
+    ran out of memory."""
+    from agent_runner.events import MEM_LOOP
+
+    _emit_giveup(log_dir, MEM_LOOP, consecutive=consecutive, exit_code=exit_code, log_path=log_path)
+
+
+def emit_mem_loop_persistent(
+    log_dir: Path, *, consecutive: int, exit_code: int, log_path: Path
+) -> None:
+    """Emit mem_loop_persistent (serve STOPS for real — 0.2.16 Task 5 cross-restart
+    convergence: mem_loop itself kept recurring across restarts within the
+    escalation window, so systemd is told to stop rather than respawn into the
+    identical loop forever). Distinct from mem_loop: this IS a deliberate stop,
+    like crash_loop/config_broken, so it belongs in the unit's
+    RestartPreventExitStatus.
+
+    ``consecutive`` counts mem_loop EPISODES within the persistence window
+    (up to ``_serve_policy._MEM_LOOP_PERSIST_THRESHOLD``) — NOT mid-round
+    terminations within one episode, which is mem_loop's own ``consecutive``
+    (up to ``MEM_LOOP_THRESHOLD``). Same redacted-log-tail reason capture as
+    crash_loop/mem_loop."""
+    from agent_runner.events import MEM_LOOP_PERSISTENT
+
+    _emit_giveup(
+        log_dir,
+        MEM_LOOP_PERSISTENT,
+        consecutive=consecutive,
+        exit_code=exit_code,
+        log_path=log_path,
+    )
+
+
+def emit_stalled_no_progress(
+    log_dir: Path, *, consecutive: int, exit_code: int, log_path: Path
+) -> None:
+    """Emit stalled_no_progress (serve gave up after consecutive clean-but-
+    no-progress rounds -- 0.2.16 Task 6). A round that exits 0 fast with no
+    ``agent_usage_recorded`` never reached the model (pi, and CLIs like it,
+    exit 0 on a provider failure) -- ``_round_ok = exit_code == 0``
+    (api_types.py) reads that as clean, so without this breaker it is a fast,
+    invisible, unclassified-failure spin, no different in kind from an
+    unknown short crash except for the exit code it hides behind.
+
+    Deliberately reuses ``CRASH_LOOP_EXIT`` (75), not a new exit code: this is
+    the SAME give-up verdict as crash_loop ("an unknown failure kept
+    recurring, stop for real") reached via a different signal (no usage
+    instead of a non-zero exit) -- not a new failure class, so it needs no new
+    entry in the systemd unit's ``RestartPreventExitStatus``. Same redacted-
+    log-tail reason capture as crash_loop/mem_loop."""
+    from agent_runner.events import STALLED_NO_PROGRESS
+
+    _emit_giveup(
+        log_dir,
+        STALLED_NO_PROGRESS,
+        consecutive=consecutive,
+        exit_code=exit_code,
+        log_path=log_path,
+    )
 
 
 def emit_config_migrated(
@@ -172,15 +261,96 @@ def emit_round_resumed(log_dir: Path, *, deferred_for_s: int) -> None:
 
 
 def emit_round_mem_terminated(
-    log_dir: Path, *, pid: int, severity: str, signal: str, message: str
+    log_dir: Path,
+    *,
+    pid: int,
+    severity: str,
+    signal: str,
+    message: str,
+    consecutive: int,
+    context: dict,
 ) -> None:
     """Emit when _spawn_round's mid-round hard floor terminated a ballooning
     round on critical host_health pressure -- the actual coma-preventer (a
     pre-round-only gate can't stop a single round mid-flight). Distinct from
-    round_supervisor_wedged (a wall-clock ceiling breach, unrelated cause)."""
+    round_supervisor_wedged (a wall-clock ceiling breach, unrelated cause).
+
+    0.2.16: ``consecutive`` (the critical_streak that crossed the threshold)
+    and ``context`` (Pressure.context -- the actual psi/mem numbers, e.g.
+    psi_full_avg10) make the kill legible from the event stream alone, so an
+    operator can retune host_health thresholds without SSH."""
     from agent_runner.events import ROUND_MEM_TERMINATED, emit
 
-    emit(log_dir, ROUND_MEM_TERMINATED, pid=pid, severity=severity, signal=signal, message=message)
+    emit(
+        log_dir,
+        ROUND_MEM_TERMINATED,
+        pid=pid,
+        severity=severity,
+        signal=signal,
+        message=message,
+        consecutive=consecutive,
+        context=context,
+    )
+
+
+def emit_round_mem_critical_sample(
+    log_dir: Path, *, round_num: int, consecutive: int, context: dict
+) -> None:
+    """Emit on EACH critical host_health sample inside _spawn_round's mid-round
+    hard floor -- deliberately NOT deduped (unlike round_mem_terminated): the
+    point is calibration visibility into near-misses, so an operator watching
+    the event stream sees the critical_streak build (1, 2, ...) even on ticks
+    that never reach the terminate threshold (a healthy tick resets it before
+    3-in-a-row). ``context`` carries the same Pressure.context numbers as
+    round_mem_terminated. Fires only during critical pressure -- never during
+    warning/healthy -- so it adds no normal-operation noise, including while
+    ``defer_to_cgroup`` steps back from terminating (the streak/context is
+    still useful calibration there; this event stays scoped to the
+    sample-level signal, distinct from mem_pressure_deferred_to_cgroup's
+    once-per-episode terminate-vs-defer notice)."""
+    from agent_runner.events import ROUND_MEM_CRITICAL_SAMPLE, emit
+
+    emit(
+        log_dir,
+        ROUND_MEM_CRITICAL_SAMPLE,
+        round_num=round_num,
+        consecutive=consecutive,
+        context=context,
+    )
+
+
+def emit_host_cgroup_memory_limit(
+    log_dir: Path, *, memory_max: int | None, memory_swap_max: int | None, cgroup_path: str | None
+) -> None:
+    """Emit once at serve startup: this process's cgroup v2 memory budget
+    (``metrics.cgroup_memory_limits``). ``None`` fields mean unlimited (or
+    cgroup v2 unavailable). Serve uses this once to decide whether the
+    mid-round hard floor can defer to kernel cgroup-OOM -- see
+    ``emit_mem_pressure_deferred_to_cgroup`` below."""
+    from agent_runner.events import HOST_CGROUP_MEMORY_LIMIT, emit
+
+    emit(
+        log_dir,
+        HOST_CGROUP_MEMORY_LIMIT,
+        memory_max=memory_max,
+        memory_swap_max=memory_swap_max,
+        cgroup_path=cgroup_path,
+    )
+
+
+def emit_mem_pressure_deferred_to_cgroup(
+    log_dir: Path, *, pid: int, signal: str, message: str
+) -> None:
+    """Emit when _spawn_round's mid-round hard floor hits sustained critical
+    pressure but this cgroup's (mem+swap) budget is bounded end to end
+    (both memory.max and memory.swap.max finite) -- kernel cgroup-OOM will
+    contain the agent and keep the host responsive on its own, so the
+    cruder host-wide round-kill steps back instead of firing
+    round_mem_terminated. This OVERRIDES in_round_mem_terminate=True: a
+    bounded cgroup makes the host floor strictly worse, not just redundant."""
+    from agent_runner.events import MEM_PRESSURE_DEFERRED_TO_CGROUP, emit
+
+    emit(log_dir, MEM_PRESSURE_DEFERRED_TO_CGROUP, pid=pid, signal=signal, message=message)
 
 
 def emit_round_substrate_before(

{cli_agent_runner-0.2.14 → cli_agent_runner-0.2.16}/agent_runner/_serve_policy.py

@@ -43,6 +43,57 @@ CRASH_LOOP_THRESHOLD = 5
 CRASH_LOOP_SHORT_EXIT_S = 60  # mirrors monitor.SHORT_EXIT_THRESHOLD_S
 CRASH_LOOP_MAX_DELAY_S = 1800  # cap the escalating restart delay (30 min)
 
+# Exit code for the mem-loop give-up cap (0.2.15 coma-preventer, Task 4): serve
+# gives up after MEM_LOOP_THRESHOLD consecutive mid-round memory-terminated
+# rounds rather than retrying forever. Distinct from CRASH_LOOP_EXIT (75) on
+# purpose: this is a "break-then-restart", NOT a deliberate stop — the host
+# memory condition may have cleared by the time systemd respawns a fresh serve
+# process, so unlike config_broken/crash_loop this code is deliberately absent
+# from the unit's RestartPreventExitStatus. 71 = EX_OSERR (sysexits).
+MEM_LOOP_EXIT = 71
+# Consecutive mem-terminated rounds before serve gives up (break-then-restart).
+MEM_LOOP_THRESHOLD = 5
+
+# Exit code for mem_loop's cross-restart escalation (0.2.16 Task 5 — field
+# confirmed: NRestarts climbs, never converges). MEM_LOOP_EXIT (71) alone
+# resets on every process restart, so a host stuck in sustained pressure
+# respawns into the identical break-then-restart loop forever. Once mem_loop
+# itself keeps recurring across restarts (see _MEM_LOOP_PERSIST_THRESHOLD),
+# this is a DELIBERATE stop instead — "gave up for real, needs a human" — so,
+# unlike 71, it IS listed in the unit's RestartPreventExitStatus. 70 =
+# EX_SOFTWARE (sysexits); verified free (grep -rnE "= 70\b|code=70\b").
+MEM_LOOP_PERSISTENT_EXIT = 70
+
+# Escalation window/threshold for the cross-restart convergence above: prior
+# mem_loop events are counted only within the last _MEM_LOOP_PERSIST_WINDOW_S
+# seconds (2h) of "now" — old episodes age out on their own, so a host that
+# stays healthy for a sustained window earns a fresh restartable 71 again
+# instead of ratcheting toward the persistent stop forever (no state file:
+# purely events-tail derived, restart-safe by construction). Once this run's
+# mem_loop would be the _MEM_LOOP_PERSIST_THRESHOLD-th (3rd) occurrence inside
+# the window (~2 prior + this one, ⇒ ~15 mem-terminated kills total), serve
+# escalates to MEM_LOOP_PERSISTENT_EXIT instead of the usual 71.
+_MEM_LOOP_PERSIST_WINDOW_S = 7200
+_MEM_LOOP_PERSIST_THRESHOLD = 3
+
+
+# Exit-0 no-progress breaker (0.2.16 Task 6). Some CLIs (pi -- see
+# builtin_plugins/pi.py's "pi exits 0 on provider failure") exit 0 on a
+# provider failure that never reaches the model: `_round_ok = exit_code == 0`
+# (api_types.py) reads that as a clean round, so without this breaker an
+# invalid credential (or an exhausted-retries outage) spins as a fast,
+# invisible "success" loop with no breaker, no back-off, and no alert -- on a
+# constrained host, that tight loop is itself a memory-pressure generator.
+# "No progress" = exit 0, a SHORT round (< _NO_PROGRESS_SHORT_S), and no
+# agent_usage_recorded for it (see _throttle.round_had_no_progress for the
+# events-derived scoping). Deliberately reuses CRASH_LOOP_THRESHOLD /
+# CRASH_LOOP_EXIT rather than minting new ones: this is the SAME give-up
+# verdict as the crash-loop breaker ("an unknown failure kept recurring, stop
+# for real"), reached via a different signal (no usage instead of a non-zero
+# exit) -- not a new failure class needing its own systemd
+# RestartPreventExitStatus entry.
+_NO_PROGRESS_SHORT_S = 30
+
 
 class EnvironmentalError(Exception):
     """Marks a round-child failure as ENVIRONMENTAL (recoverable) rather than a
@@ -106,6 +157,47 @@ def post_round_decision(
     return ("continue", delay, 0)
 
 
+def _mem_loop_decision(
+    *, mem_terminated: bool, consecutive: int
+) -> tuple[Literal["mem_loop", "continue"], int]:
+    """Mem-loop give-up cap (0.2.15 Task 4) — a NEW, SEPARATE, private counter
+    from ``post_round_decision``'s crash-loop breaker, not a widening of its
+    3-tuple contract. A mem-terminated round already sets ``throttle_active``
+    for ``post_round_decision`` (never counted as a crash), so without this
+    cap a sustained memory-pressure host would retry forever instead of
+    eventually giving up. Returns ``("mem_loop", consecutive)`` once
+    ``consecutive`` reaches ``MEM_LOOP_THRESHOLD``; a non-mem-terminated round
+    resets the counter to 0, same reset shape as the crash-loop breaker."""
+    if not mem_terminated:
+        return ("continue", 0)
+    consecutive += 1
+    if consecutive >= MEM_LOOP_THRESHOLD:
+        return ("mem_loop", consecutive)
+    return ("continue", consecutive)
+
+
+def _no_progress_decision(
+    *, no_progress: bool, consecutive: int
+) -> tuple[Literal["stalled_no_progress", "continue"], int]:
+    """Exit-0 no-progress give-up cap (0.2.16 Task 6) -- a NEW, SEPARATE,
+    private counter from ``post_round_decision``'s crash-loop breaker (that one
+    keys on ``returncode != 0``; here returncode IS 0, so a round that never
+    reached the model would otherwise sail through with no breaker at all).
+    Same shape as :func:`_mem_loop_decision`: returns ``("stalled_no_progress",
+    consecutive)`` once ``consecutive`` reaches ``CRASH_LOOP_THRESHOLD`` --
+    reusing that threshold (and, at the call site, ``CRASH_LOOP_EXIT``) rather
+    than minting new ones, since this is the identical give-up verdict reached
+    via a different signal. A round WITH progress (usage recorded, non-zero
+    exit, or slow) resets the counter to 0, same reset shape as
+    crash-loop/mem-loop."""
+    if not no_progress:
+        return ("continue", 0)
+    consecutive += 1
+    if consecutive >= CRASH_LOOP_THRESHOLD:
+        return ("stalled_no_progress", consecutive)
+    return ("continue", consecutive)
+
+
 def classify_round_exit(exc: BaseException) -> int:
     """The round-child exit-code CLASSIFIER (Group A) — permanence, not exception
     identity, decides serve's response. Replaces the old exception *whitelist*

{cli_agent_runner-0.2.14 → cli_agent_runner-0.2.16}/agent_runner/_throttle.py

@@ -22,6 +22,7 @@ from agent_runner.api_types import TransientErrorState
 from agent_runner.clock import SYSTEM_CLOCK, Clock
 from agent_runner.events import (
     AGENT_USAGE_RECORDED,
+    MEM_LOOP,
     ROUND_MEM_TERMINATED,
     ROUND_SUBSTRATE_BEFORE,
     TRANSIENT_ERROR_DETECTED,
@@ -37,7 +38,9 @@ __all__ = [
     "_interruptible_sleep",
     "compute_adjusted_reset_at",
     "effective_throttle_view",
+    "mem_loop_events_in_window",
     "pending_recovered",
+    "round_had_no_progress",
     "round_was_mem_terminated",
 ]
 
@@ -277,21 +280,20 @@ def _active_throttles(
 def round_was_mem_terminated(log_dir: Path) -> bool:
     """True iff the round that JUST ran was killed by ``_spawn_round``'s
     mid-round memory-pressure hard floor (``round_mem_terminated``) rather
-    than a genuine crash — so ``serve_cmd._ran_agent_throttled`` can excuse it
-    from the crash-loop breaker exactly like an active throttle (flat
-    back-off, ``consecutive`` reset to 0), the same treatment
-    ``ENV_BATTERY_EXIT`` already gets. A mem-terminated round can be killed
-    within the first ~10s (the mid-round check's own cadence) — far under the
-    crash-loop breaker's 60s "short crash" window — so, unlike a wall-clock-
-    ceiling wedge (whose long duration alone dodges the breaker), this needs
-    an explicit signal.
+    than a genuine crash — so ``serve_cmd.cmd`` can excuse it from the
+    crash-loop breaker exactly like an active throttle (flat back-off,
+    ``consecutive`` reset to 0), the same treatment ``ENV_BATTERY_EXIT``
+    already gets. A mem-terminated round can be killed within the first ~10s
+    (the mid-round check's own cadence) — far under the crash-loop breaker's
+    60s "short crash" window — so, unlike a wall-clock-ceiling wedge (whose
+    long duration alone dodges the breaker), this needs an explicit signal.
 
     Scoped to "this round" (not some earlier one) by comparing the newest
     ``round_mem_terminated`` event's timestamp against the newest
     ``round_substrate_before``'s: ``round_substrate_before(N)`` always
     precedes round N's own attempt, and no later round's
     ``round_substrate_before(N+1)`` has been emitted yet at the point
-    ``_ran_agent_throttled`` runs (right after ``_spawn_round`` returns) —
+    ``cmd()`` runs this check (right after ``_spawn_round`` returns) —
     ``round_substrate_after``, also emitted before this check runs, cannot
     serve as that boundary, since it always comes AFTER any
     ``round_mem_terminated`` within the very round it is scoping. ``>=``, not
@@ -315,6 +317,108 @@ def round_was_mem_terminated(log_dir: Path) -> bool:
     return parse_iso_ms(newest_terminated_ts) >= parse_iso_ms(newest_before_ts)
 
 
+def round_had_no_progress(
+    log_dir: Path,
+    *,
+    returncode: int,
+    duration_s: float,
+    threshold_s: float,
+    throttle_active: bool = False,
+) -> bool:
+    """True iff the round that JUST ran exited 0, finished fast, but never
+    reached the model -- pi (and CLIs like it: see builtin_plugins/pi.py's
+    "pi exits 0 on provider failure") can exit 0 on an auth failure or an
+    exhausted-retries outage that ``_round_ok = exit_code == 0`` (api_types.py)
+    reads as clean, so this needs its own events-derived signal (feeding
+    ``_serve_policy._no_progress_decision``'s "clean-but-no-progress" streak)
+    exactly parallel to :func:`round_was_mem_terminated`'s mem-terminated one.
+
+    TWO gates guard the verdict against over-firing on healthy deployments
+    (0.2.16 fix-wave CRITICAL #1):
+
+    1. ``throttle_active`` (the SAME value ``cmd()`` already computed for
+       ``post_round_decision`` -- caller threads it through, never
+       recomputed here): a provider outage that exhausts retries and exits 0
+       (429/503) is ``transient_error_detected`` and excused from the
+       crash-loop breaker; without this gate it would be double-counted here
+       as "no progress" and stop the loop instead of riding the back-off.
+    2. Usage-capability: armed ONLY when at least one ``agent_usage_recorded``
+       event exists anywhere in the scanned tail. Some CLIs (kimi -- see
+       builtin_plugins/kimi.py; aider; any custom ``[agent] command`` with no
+       usage-emitting plugin) never emit usage BY DESIGN -- for those, "no
+       usage this round" is indistinguishable from "normal for this CLI", so
+       arming would stop a perfectly healthy deployment after 5 fast clean
+       rounds. If the CLI's plugin stack ever emits usage (pi does on a good
+       round), a round with none is genuine no-progress and still trips --
+       this makes the breaker CLI-adaptive with no config descriptor.
+
+    Short-circuits on ``returncode != 0`` or ``duration_s >= threshold_s``
+    without touching the events tail: a non-zero exit already has its own
+    crash-loop signal, and a slow round (even with no usage) is not a TIGHT
+    loop -- a wedged/hung round already has its own signal
+    (``round_supervisor_wedged``), so this floor is specifically the fast spin.
+
+    "Never reached the model" = no ``agent_usage_recorded`` event stamped at
+    or after this round's ``round_substrate_before`` -- the CLI plugins only
+    emit usage when the round actually consumed tokens (e.g.
+    ``builtin_plugins/pi.py``'s ``_aggregate_usage`` returns ``None`` on
+    all-zero usage), so an auth failure or exhausted retries leaves no such
+    event. Same round-scoping shape as :func:`round_was_mem_terminated`
+    (newest-event-timestamp comparison against the newest
+    ``round_substrate_before``), with the comparison inverted: no progress
+    means the newest usage event is EITHER absent OR older than this round's
+    own start."""
+    if throttle_active or returncode != 0 or duration_s >= threshold_s:
+        return False
+    newest_before_ts: str | None = None
+    newest_usage_ts: str | None = None
+    usage_capable = False
+    for path in sorted(log_dir.glob("events-*.jsonl"))[-2:]:
+        for ev in _iter_events(path):
+            kind = ev.get("event")
+            if kind == AGENT_USAGE_RECORDED:
+                usage_capable = True
+            ts = ev.get("ts")
+            if not ts:
+                continue
+            if kind == ROUND_SUBSTRATE_BEFORE:
+                newest_before_ts = ts
+            elif kind == AGENT_USAGE_RECORDED:
+                newest_usage_ts = ts
+    if not usage_capable:
+        return False  # this CLI/plugin stack has never emitted usage -- can't arm
+    if newest_before_ts is None:
+        return False
+    if newest_usage_ts is None:
+        return True
+    return parse_iso_ms(newest_usage_ts) < parse_iso_ms(newest_before_ts)
+
+
+def mem_loop_events_in_window(log_dir: Path, clock: Clock, window_s: int) -> int:
+    """Count ``mem_loop`` events in the events tail stamped within the last
+    ``window_s`` seconds of ``clock.epoch()`` — events-derived, no state file,
+    same tail-reconstruction shape as :func:`round_was_mem_terminated` above.
+
+    Feeds ``_serve_round.round_outcome_exit_code``'s cross-restart escalation
+    (0.2.16 Task 5): ``MEM_LOOP_EXIT`` (71) alone resets on every serve
+    process restart, so a host stuck in sustained pressure respawns into the
+    identical loop forever (field-confirmed: NRestarts climbs, never
+    converges). Counting PRIOR occurrences in a bounded window — rather than
+    an all-time total, or a restart-local counter — gives "ages out on its
+    own after a sustained-healthy stretch" for free: an old mem_loop episode
+    outside the window simply stops counting, no explicit reset needed."""
+    cutoff = clock.epoch() - window_s
+    count = 0
+    for path in sorted(log_dir.glob("events-*.jsonl"))[-2:]:
+        for ev in _iter_events(path):
+            if ev.get("event") != MEM_LOOP:
+                continue
+            ts = ev.get("ts")
+            if ts and parse_iso_ms(ts).timestamp() >= cutoff:
+                count += 1
+    return count
+
+
 def effective_throttle_view(
     log_dir: Path, *, clock: Clock = SYSTEM_CLOCK
 ) -> tuple[TransientErrorState | None, dict[str, TransientErrorState]]:

{cli_agent_runner-0.2.14 → cli_agent_runner-0.2.16}/agent_runner/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.14'
-__version_tuple__ = version_tuple = (0, 2, 14)
+__version__ = version = '0.2.16'
+__version_tuple__ = version_tuple = (0, 2, 16)
 
 __commit_id__ = commit_id = None

{cli_agent_runner-0.2.14 → cli_agent_runner-0.2.16}/agent_runner/api.py

@@ -28,6 +28,8 @@ from agent_runner._serve_policy import (  # noqa: F401 — public re-export
     CRASH_LOOP_SHORT_EXIT_S,
     CRASH_LOOP_THRESHOLD,
     ENV_BATTERY_EXIT,
+    MEM_LOOP_EXIT,
+    MEM_LOOP_PERSISTENT_EXIT,
     PERMANENT_CONFIG_EXIT,
     post_round_decision,
 )
@@ -629,6 +631,8 @@ def _poll_once(
         mem_avail_min_mb=cfg.monitor.host_health.mem_avail_min_mb,
         disk_warning_pct=cfg.monitor.host_health.disk_warning_pct,
         disk_critical_pct=cfg.monitor.host_health.disk_critical_pct,
+        swap_sout_noise_floor_mb=cfg.monitor.host_health.swap_sout_noise_floor_mb,
+        mem_free_low_mb=cfg.monitor.host_health.mem_free_low_mb,
         log_dir=cfg.runtime.log_dir,
     )
     if not monitor._PLUGIN_DETECTORS:
@@ -783,12 +787,17 @@ from agent_runner._emit import (  # noqa: E402,F401 — intentional bottom re-ex
     emit_config_migrated,
     emit_crash_loop,
     emit_fresh_eyes_round_triggered,
+    emit_host_cgroup_memory_limit,
     emit_max_rounds_reached,
+    emit_mem_loop,
+    emit_mem_loop_persistent,
+    emit_mem_pressure_deferred_to_cgroup,
     emit_rate_limit_stop,
     emit_round_deferred,
     emit_round_grace_extended,
     emit_round_grace_kill,
     emit_round_logs_prune_deferred,
+    emit_round_mem_critical_sample,
     emit_round_mem_terminated,
     emit_round_progress,
     emit_round_resumed,
@@ -799,6 +808,7 @@ from agent_runner._emit import (  # noqa: E402,F401 — intentional bottom re-ex
     emit_schedule_phase_skipped,
     emit_schedule_resumed,
     emit_stale_index_lock_cleared,
+    emit_stalled_no_progress,
     emit_stop_file_detected,
     emit_transient_error_backoff_capped,
     emit_transient_error_detected,