cli-agent-runner 0.2.15__tar.gz → 0.2.17__tar.gz

{cli_agent_runner-0.2.15 → cli_agent_runner-0.2.17}/.vulture-whitelist.py

@@ -13,6 +13,13 @@ log_dir
 offsets
 buffer
 
+# agent_runner._throttle.RoundOutcome
+mem_terminated
+usage_capable
+newest_usage_ts
+newest_substrate_before_ts
+latest_transient_per_agent
+
 # agent_runner.agent_runtime.RunResult
 exit_code
 duration_s
@@ -150,6 +157,10 @@ disk_warning_pct
 disk_critical_pct
 swap_sout_noise_floor_mb
 mem_free_low_mb
+psi_full_avg10_critical
+psi_some_avg10_warning
+mem_critical_consecutive_samples
+in_round_mem_terminate
 
 # agent_runner.config.models.PhaseOverride
 round_timeout_s

{cli_agent_runner-0.2.15 → cli_agent_runner-0.2.17}/CHANGELOG.md

@@ -5,6 +5,37 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.17] - 2026-09-05
+
+### Changed
+- Internal refactor ("round-outcome unification"): per-round classification now does one events-tail scan per round (was up to three), the give-up decision is unified behind a single `post_round_verdicts` (precedence and exit codes `78`/`70`/`71`/`75` unchanged), and the `_serve_round`↔`serve_cmd` import cycle is removed. Behavior-preserving.
+- `round_mem_critical_sample` is now capped at `2 × mem_critical_consecutive_samples` emits per critical episode (any non-critical tick resets it), so a host stuck under sustained pressure no longer writes that calibration event unbounded for a whole round.
+
+### Fixed
+- Auto-stop no longer records `monitor_auto_stop_failed` for a stop that actually succeeds: under load a round can outlast the stop-confirmation window while draining, which was mis-reported as a failure. The monitor now recognizes a still-draining round and records the real outcome (triggered or failed) once it resolves on the next poll, instead of a false failure.
+
+### Notes
+- Not breaking: no `agent-runner migrate`, no systemd unit change, no new event kinds, `peek --json` schema unchanged. The only non-fix behavior change is the bounded `round_mem_critical_sample` above.
+
+## [0.2.16] - 2026-09-05
+
+### Added
+- `[monitor.host_health]` gains PSI knobs — `psi_full_avg10_critical` (default 60.0) and `psi_some_avg10_warning` (default 5.0) — plus `mem_critical_consecutive_samples` (default 3) and an `in_round_mem_terminate` off switch for the mid-round memory floor.
+- `serve` probes this process's cgroup v2 memory budget at startup (`host_cgroup_memory_limit`); when `memory.max` and `memory.swap.max` are both finite, the mid-round floor defers to kernel cgroup-OOM (`mem_pressure_deferred_to_cgroup`) instead of terminating the round itself.
+- `mem_loop` now escalates across restarts: recurring 3 times inside a rolling 2h window stops the service for real (`mem_loop_persistent`, exit `70`) instead of respawning into the same loop forever.
+- New exit-0 no-progress breaker: 5 consecutive fast, clean rounds with no recorded usage (a CLI that exits 0 without ever reaching the model) now stop `serve` (`stalled_no_progress`, exit `75`) instead of spinning invisibly.
+
+### Fixed
+- PSI-full critical was a single ≥1% momentary sample — a hiccup, not a coma — on every PSI-capable host. It now requires a sustained ≥60% avg10 held for `mem_critical_consecutive_samples` (default 3, ~30s) consecutive checks.
+- The mid-round swap-out check is now per-tick instead of cumulative since round-start, so a host that recovers mid-round is no longer permanently flagged for the rest of the round.
+- The pre-round admission gate now defers only on `critical` pressure, not on any pressure (a mere `warning`, e.g. ordinary swap churn, no longer delays the next round).
+
+### Notes
+- New events: `host_cgroup_memory_limit`, `mem_pressure_deferred_to_cgroup`, `mem_loop_persistent`, `stalled_no_progress`, `round_mem_critical_sample`; `round_mem_terminated` gained `consecutive`/`context` fields. Not breaking: no `agent-runner migrate` needed, `peek --json` schema is unchanged.
+- Re-render the systemd unit (`agent-runner install`) after upgrading: the new unit adds exit `70` to `RestartPreventExitStatus` so `mem_loop_persistent` stops the service instead of restarting into the same loop.
+
+See `docs/migrations/0.2.md`.
+
 ## [0.2.15] - 2026-09-04
 
 ### Added

{cli_agent_runner-0.2.15 → cli_agent_runner-0.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cli-agent-runner
-Version: 0.2.15
+Version: 0.2.17
 Summary: Restart-on-exit supervisor for autonomous CLI agents
 Project-URL: Homepage, https://github.com/wan9yu/cli-agent-runner
 Project-URL: Documentation, https://github.com/wan9yu/cli-agent-runner#readme
@@ -94,7 +94,7 @@ Verb reference: [`docs/commands.md`](docs/commands.md).
 
 ## Defenses (built in)
 
-14 named defenses, structured as data — see `agent-runner peek --select defenses`.
+15 named defenses, structured as data — see `agent-runner peek --select defenses`.
 Each carries the historical incident it codifies and the invariant test that
 guards it. Highlights:
 

{cli_agent_runner-0.2.15 → cli_agent_runner-0.2.17}/README.md

@@ -58,7 +58,7 @@ Verb reference: [`docs/commands.md`](docs/commands.md).
 
 ## Defenses (built in)
 
-14 named defenses, structured as data — see `agent-runner peek --select defenses`.
+15 named defenses, structured as data — see `agent-runner peek --select defenses`.
 Each carries the historical incident it codifies and the invariant test that
 guards it. Highlights:
 

{cli_agent_runner-0.2.15 → cli_agent_runner-0.2.17}/agent_runner/_emit.py

@@ -19,13 +19,17 @@ __all__ = [
     "emit_anomaly_repetitive_tool",
     "emit_config_migrated",
     "emit_fresh_eyes_round_triggered",
+    "emit_host_cgroup_memory_limit",
     "emit_max_rounds_reached",
     "emit_mem_loop",
+    "emit_mem_loop_persistent",
+    "emit_mem_pressure_deferred_to_cgroup",
     "emit_rate_limit_stop",
     "emit_round_deferred",
     "emit_round_grace_extended",
     "emit_round_grace_kill",
     "emit_round_logs_prune_deferred",
+    "emit_round_mem_critical_sample",
     "emit_round_mem_terminated",
     "emit_round_progress",
     "emit_round_resumed",
@@ -36,6 +40,7 @@ __all__ = [
     "emit_schedule_phase_skipped",
     "emit_schedule_resumed",
     "emit_stale_index_lock_cleared",
+    "emit_stalled_no_progress",
     "emit_stop_file_detected",
     "emit_transient_error_backoff_capped",
     "emit_transient_error_detected",
@@ -110,6 +115,60 @@ def emit_mem_loop(log_dir: Path, *, consecutive: int, exit_code: int, log_path:
     _emit_giveup(log_dir, MEM_LOOP, consecutive=consecutive, exit_code=exit_code, log_path=log_path)
 
 
+def emit_mem_loop_persistent(
+    log_dir: Path, *, consecutive: int, exit_code: int, log_path: Path
+) -> None:
+    """Emit mem_loop_persistent (serve STOPS for real — 0.2.16 Task 5 cross-restart
+    convergence: mem_loop itself kept recurring across restarts within the
+    escalation window, so systemd is told to stop rather than respawn into the
+    identical loop forever). Distinct from mem_loop: this IS a deliberate stop,
+    like crash_loop/config_broken, so it belongs in the unit's
+    RestartPreventExitStatus.
+
+    ``consecutive`` counts mem_loop EPISODES within the persistence window
+    (up to ``_serve_policy._MEM_LOOP_PERSIST_THRESHOLD``) — NOT mid-round
+    terminations within one episode, which is mem_loop's own ``consecutive``
+    (up to ``MEM_LOOP_THRESHOLD``). Same redacted-log-tail reason capture as
+    crash_loop/mem_loop."""
+    from agent_runner.events import MEM_LOOP_PERSISTENT
+
+    _emit_giveup(
+        log_dir,
+        MEM_LOOP_PERSISTENT,
+        consecutive=consecutive,
+        exit_code=exit_code,
+        log_path=log_path,
+    )
+
+
+def emit_stalled_no_progress(
+    log_dir: Path, *, consecutive: int, exit_code: int, log_path: Path
+) -> None:
+    """Emit stalled_no_progress (serve gave up after consecutive clean-but-
+    no-progress rounds -- 0.2.16 Task 6). A round that exits 0 fast with no
+    ``agent_usage_recorded`` never reached the model (pi, and CLIs like it,
+    exit 0 on a provider failure) -- ``_round_ok = exit_code == 0``
+    (api_types.py) reads that as clean, so without this breaker it is a fast,
+    invisible, unclassified-failure spin, no different in kind from an
+    unknown short crash except for the exit code it hides behind.
+
+    Deliberately reuses ``CRASH_LOOP_EXIT`` (75), not a new exit code: this is
+    the SAME give-up verdict as crash_loop ("an unknown failure kept
+    recurring, stop for real") reached via a different signal (no usage
+    instead of a non-zero exit) -- not a new failure class, so it needs no new
+    entry in the systemd unit's ``RestartPreventExitStatus``. Same redacted-
+    log-tail reason capture as crash_loop/mem_loop."""
+    from agent_runner.events import STALLED_NO_PROGRESS
+
+    _emit_giveup(
+        log_dir,
+        STALLED_NO_PROGRESS,
+        consecutive=consecutive,
+        exit_code=exit_code,
+        log_path=log_path,
+    )
+
+
 def emit_config_migrated(
     log_dir: Path, *, applied: list[str], manual: list[str], path: str
 ) -> None:
@@ -202,15 +261,106 @@ def emit_round_resumed(log_dir: Path, *, deferred_for_s: int) -> None:
 
 
 def emit_round_mem_terminated(
-    log_dir: Path, *, pid: int, severity: str, signal: str, message: str
+    log_dir: Path,
+    *,
+    pid: int,
+    severity: str,
+    signal: str,
+    message: str,
+    consecutive: int,
+    context: dict,
 ) -> None:
     """Emit when _spawn_round's mid-round hard floor terminated a ballooning
     round on critical host_health pressure -- the actual coma-preventer (a
     pre-round-only gate can't stop a single round mid-flight). Distinct from
-    round_supervisor_wedged (a wall-clock ceiling breach, unrelated cause)."""
+    round_supervisor_wedged (a wall-clock ceiling breach, unrelated cause).
+
+    0.2.16: ``consecutive`` (the critical_streak that crossed the threshold)
+    and ``context`` (Pressure.context -- the actual psi/mem numbers, e.g.
+    psi_full_avg10) make the kill legible from the event stream alone, so an
+    operator can retune host_health thresholds without SSH."""
     from agent_runner.events import ROUND_MEM_TERMINATED, emit
 
-    emit(log_dir, ROUND_MEM_TERMINATED, pid=pid, severity=severity, signal=signal, message=message)
+    emit(
+        log_dir,
+        ROUND_MEM_TERMINATED,
+        pid=pid,
+        severity=severity,
+        signal=signal,
+        message=message,
+        consecutive=consecutive,
+        context=context,
+    )
+
+
+def emit_round_mem_critical_sample(
+    log_dir: Path, *, round_num: int, consecutive: int, context: dict
+) -> None:
+    """Emit on each critical host_health sample inside _spawn_round's mid-round
+    hard floor, up to the per-episode cap (0.2.17, below) -- unlike
+    round_mem_terminated (deduped to once-per-episode), this fires on every
+    critical tick within that cap: the point is calibration visibility into
+    near-misses, so an operator watching the event stream sees the
+    critical_streak build (1, 2, ...) even on ticks that never reach the
+    terminate threshold (a healthy tick resets it before 3-in-a-row).
+    ``context`` carries the same Pressure.context numbers as
+    round_mem_terminated. Fires only during critical pressure -- never during
+    warning/healthy -- so it adds no normal-operation noise, including while
+    ``defer_to_cgroup`` steps back from terminating (the streak/context is
+    still useful calibration there; this event stays scoped to the
+    sample-level signal, distinct from mem_pressure_deferred_to_cgroup's
+    once-per-episode terminate-vs-defer notice).
+
+    0.2.17: the caller caps this at ``2 * mem_critical_consecutive_samples``
+    consecutive ticks (1..6 at the default 3) -- a sustained-critical
+    don't-terminate run (cgroup-defer, or the off switch) would otherwise
+    write one event per ~10s tick for up to a whole ``round_timeout_s`` on a
+    permanently-deferred/off host. The cap is per streak-episode, not a
+    lifetime limit: any non-critical tick still resets the streak to 0, and
+    sampling resumes from 1 the next time critical pressure recurs."""
+    from agent_runner.events import ROUND_MEM_CRITICAL_SAMPLE, emit
+
+    emit(
+        log_dir,
+        ROUND_MEM_CRITICAL_SAMPLE,
+        round_num=round_num,
+        consecutive=consecutive,
+        context=context,
+    )
+
+
+def emit_host_cgroup_memory_limit(
+    log_dir: Path, *, memory_max: int | None, memory_swap_max: int | None, cgroup_path: str | None
+) -> None:
+    """Emit once at serve startup: this process's cgroup v2 memory budget
+    (``metrics.cgroup_memory_limits``). ``None`` fields mean unlimited (or
+    cgroup v2 unavailable). Serve uses this once to decide whether the
+    mid-round hard floor can defer to kernel cgroup-OOM -- see
+    ``emit_mem_pressure_deferred_to_cgroup`` below."""
+    from agent_runner.events import HOST_CGROUP_MEMORY_LIMIT, emit
+
+    emit(
+        log_dir,
+        HOST_CGROUP_MEMORY_LIMIT,
+        memory_max=memory_max,
+        memory_swap_max=memory_swap_max,
+        cgroup_path=cgroup_path,
+    )
+
+
+def emit_mem_pressure_deferred_to_cgroup(
+    log_dir: Path, *, pid: int, signal: str, message: str
+) -> None:
+    """Emit when _spawn_round's mid-round hard floor hits sustained critical
+    pressure but this cgroup's (mem+swap) budget is bounded end to end
+    (both memory.max and memory.swap.max finite) -- kernel cgroup-OOM will
+    contain the agent and keep the host responsive on its own, so the
+    cruder host-wide round-kill steps back instead of firing
+    round_mem_terminated. This OVERRIDES in_round_mem_terminate=True: a
+    bounded cgroup makes the host floor strictly worse, not just redundant."""
+    from agent_runner.events import MEM_PRESSURE_DEFERRED_TO_CGROUP, emit
+
+    emit(log_dir, MEM_PRESSURE_DEFERRED_TO_CGROUP, pid=pid, signal=signal, message=message)
 
 
 def emit_round_substrate_before(

{cli_agent_runner-0.2.15 → cli_agent_runner-0.2.17}/agent_runner/_serve_policy.py

@@ -54,6 +54,46 @@ MEM_LOOP_EXIT = 71
 # Consecutive mem-terminated rounds before serve gives up (break-then-restart).
 MEM_LOOP_THRESHOLD = 5
 
+# Exit code for mem_loop's cross-restart escalation (0.2.16 Task 5 — field
+# confirmed: NRestarts climbs, never converges). MEM_LOOP_EXIT (71) alone
+# resets on every process restart, so a host stuck in sustained pressure
+# respawns into the identical break-then-restart loop forever. Once mem_loop
+# itself keeps recurring across restarts (see _MEM_LOOP_PERSIST_THRESHOLD),
+# this is a DELIBERATE stop instead — "gave up for real, needs a human" — so,
+# unlike 71, it IS listed in the unit's RestartPreventExitStatus. 70 =
+# EX_SOFTWARE (sysexits); verified free (grep -rnE "= 70\b|code=70\b").
+MEM_LOOP_PERSISTENT_EXIT = 70
+
+# Escalation window/threshold for the cross-restart convergence above: prior
+# mem_loop events are counted only within the last _MEM_LOOP_PERSIST_WINDOW_S
+# seconds (2h) of "now" — old episodes age out on their own, so a host that
+# stays healthy for a sustained window earns a fresh restartable 71 again
+# instead of ratcheting toward the persistent stop forever (no state file:
+# purely events-tail derived, restart-safe by construction). Once this run's
+# mem_loop would be the _MEM_LOOP_PERSIST_THRESHOLD-th (3rd) occurrence inside
+# the window (~2 prior + this one, ⇒ ~15 mem-terminated kills total), serve
+# escalates to MEM_LOOP_PERSISTENT_EXIT instead of the usual 71.
+_MEM_LOOP_PERSIST_WINDOW_S = 7200
+_MEM_LOOP_PERSIST_THRESHOLD = 3
+
+
+# Exit-0 no-progress breaker (0.2.16 Task 6). Some CLIs (pi -- see
+# builtin_plugins/pi.py's "pi exits 0 on provider failure") exit 0 on a
+# provider failure that never reaches the model: `_round_ok = exit_code == 0`
+# (api_types.py) reads that as a clean round, so without this breaker an
+# invalid credential (or an exhausted-retries outage) spins as a fast,
+# invisible "success" loop with no breaker, no back-off, and no alert -- on a
+# constrained host, that tight loop is itself a memory-pressure generator.
+# "No progress" = exit 0, a SHORT round (< _NO_PROGRESS_SHORT_S), and no
+# agent_usage_recorded for it (see _throttle.round_had_no_progress for the
+# events-derived scoping). Deliberately reuses CRASH_LOOP_THRESHOLD /
+# CRASH_LOOP_EXIT rather than minting new ones: this is the SAME give-up
+# verdict as the crash-loop breaker ("an unknown failure kept recurring, stop
+# for real"), reached via a different signal (no usage instead of a non-zero
+# exit) -- not a new failure class needing its own systemd
+# RestartPreventExitStatus entry.
+_NO_PROGRESS_SHORT_S = 30
+
 
 class EnvironmentalError(Exception):
     """Marks a round-child failure as ENVIRONMENTAL (recoverable) rather than a
@@ -117,6 +157,25 @@ def post_round_decision(
     return ("continue", delay, 0)
 
 
+def _consecutive_streak_decision(
+    *, triggered: bool, consecutive: int, threshold: int, verdict: str
+) -> tuple[str, int]:
+    """The generic give-up-cap shape both :func:`_mem_loop_decision` and
+    :func:`_no_progress_decision` delegate to (0.2.17 Task 2 — de-duplicated
+    out of the two, which were byte-identical apart from their reset
+    condition, threshold constant, and verdict string). A NEW, SEPARATE,
+    private counter from ``post_round_decision``'s crash-loop breaker, not a
+    widening of its 3-tuple contract. Returns ``(verdict, consecutive)`` once
+    ``consecutive`` reaches ``threshold``; a non-``triggered`` round resets the
+    counter to 0."""
+    if not triggered:
+        return ("continue", 0)
+    consecutive += 1
+    if consecutive >= threshold:
+        return (verdict, consecutive)
+    return ("continue", consecutive)
+
+
 def _mem_loop_decision(
     *, mem_terminated: bool, consecutive: int
 ) -> tuple[Literal["mem_loop", "continue"], int]:
@@ -128,12 +187,34 @@ def _mem_loop_decision(
     eventually giving up. Returns ``("mem_loop", consecutive)`` once
     ``consecutive`` reaches ``MEM_LOOP_THRESHOLD``; a non-mem-terminated round
     resets the counter to 0, same reset shape as the crash-loop breaker."""
-    if not mem_terminated:
-        return ("continue", 0)
-    consecutive += 1
-    if consecutive >= MEM_LOOP_THRESHOLD:
-        return ("mem_loop", consecutive)
-    return ("continue", consecutive)
+    return _consecutive_streak_decision(
+        triggered=mem_terminated,
+        consecutive=consecutive,
+        threshold=MEM_LOOP_THRESHOLD,
+        verdict="mem_loop",
+    )
+
+
+def _no_progress_decision(
+    *, no_progress: bool, consecutive: int
+) -> tuple[Literal["stalled_no_progress", "continue"], int]:
+    """Exit-0 no-progress give-up cap (0.2.16 Task 6) -- a NEW, SEPARATE,
+    private counter from ``post_round_decision``'s crash-loop breaker (that one
+    keys on ``returncode != 0``; here returncode IS 0, so a round that never
+    reached the model would otherwise sail through with no breaker at all).
+    Same shape as :func:`_mem_loop_decision`: returns ``("stalled_no_progress",
+    consecutive)`` once ``consecutive`` reaches ``CRASH_LOOP_THRESHOLD`` --
+    reusing that threshold (and, at the call site, ``CRASH_LOOP_EXIT``) rather
+    than minting new ones, since this is the identical give-up verdict reached
+    via a different signal. A round WITH progress (usage recorded, non-zero
+    exit, or slow) resets the counter to 0, same reset shape as
+    crash-loop/mem-loop."""
+    return _consecutive_streak_decision(
+        triggered=no_progress,
+        consecutive=consecutive,
+        threshold=CRASH_LOOP_THRESHOLD,
+        verdict="stalled_no_progress",
+    )
 
 
 def classify_round_exit(exc: BaseException) -> int:
@@ -194,7 +275,7 @@ def classify_round_exit(exc: BaseException) -> int:
 _REAP_GRACE_S = 5
 _GIT_COMMIT_TIMEOUT_S = 120
 _HOOK_ALLOWANCE_S = 60  # slack for post-round hooks between the inner wall and the ceiling
-_ROUND_TERM_GRACE_S = 15  # mirrors api._ROUND_TERM_GRACE_S / serve_cmd._ROUND_TERM_GRACE_S
+_ROUND_TERM_GRACE_S = 15  # mirrors api._ROUND_TERM_GRACE_S / cli._serve_round._ROUND_TERM_GRACE_S
 _STOP_GRACE_MARGIN_S = 10  # pad above _ROUND_TERM_GRACE_S for systemd stop-request overhead