cli-agent-runner 0.2.9__tar.gz → 0.2.11__tar.gz

{cli_agent_runner-0.2.9 → cli_agent_runner-0.2.11}/.vulture-whitelist.py

@@ -57,6 +57,8 @@ throttled_until_epoch
 limit_type
 agent
 since_round
+phase
+throttled_agents
 
 # agent_runner.api_types.RoundResult
 round_num
@@ -106,6 +108,7 @@ reset_at_epoch
 classification
 agent
 since_round
+phase
 
 # agent_runner.config.AgentConfig
 command

{cli_agent_runner-0.2.9 → cli_agent_runner-0.2.11}/CHANGELOG.md

@@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.11] - 2026-08-29
+
+### Changed
+- systemd `serve` unit now uses `Restart=on-failure` with `RestartPreventExitStatus=78 75`: a deliberate stop (sentinel / stop-file / max-rounds / SIGTERM) and a config give-up stay stopped, while a genuine crash still restarts. Re-render or hand-edit existing units.
+- Throttle-aware skip is keyed on the **agent** (binary basename of `command[0]`), not the phase: every phase sharing a rate-limited agent is stepped over together, so a multi-provider rotation no longer hammers one throttled key while a healthy provider keeps running.
+
+### Fixed
+- `round` exits non-zero on a genuine crash, so the crash-loop breaker (`crash_loop`, exit 75) sees crashes instead of a masked exit 0; a permanent config failure gives up with exit 78 (`config_broken`) instead of restart-looping.
+- Round deadlines and sleeps are measured on the monotonic clock — an NTP step no longer mis-fires or defers a hard-wall timeout.
+- Back-off and restart sleeps are chunked, so a SIGTERM lands within one chunk instead of after a multi-hour throttle window; an interrupted back-off leaves the throttle active (no false recovered).
+- A grace-killed round still dispatches its dirty work and is no longer miscounted as a timeout.
+- Only one `serve` loop runs per project (loop-lifetime lock); `serve.pid` records a start-time token so `stop`/`kill` never signals a recycled PID.
+
+### Notes
+- `peek --json` schema `1.10`: `rate_limit.throttled_agents` lists all currently-throttled agents. Additive; no config change.
+
+See `docs/migrations/0.2.md`.
+
+## [0.2.10] - 2026-08-29
+
+### Added
+- Throttle-aware skip: under `[phases] phase_policy = "skip"`, a phase whose provider is currently rate-limited/erroring is stepped over in the rotation — a healthy sibling runs instead of the whole loop sleeping on the throttle. No new config; the trigger is the existing `skip` policy.
+- `transient_error_detected` now carries the failing `phase`; `peek --json` `rate_limit` and the HTTP progress state surface which phase is throttled.
+- `transient_error_recovered` is emitted when a skip-around throttle clears (events-derived, restart-safe, no double-emit with the back-off path).
+
+### Notes
+- Only `skip` routes around a throttle; `wait`, `--ignore-schedule`, and non-`[phases]` configs keep the 0.2.9 global back-off. When every phase is throttled or window-closed, the loop waits until the earlier of a window opening or the throttle resetting.
+
+See `docs/migrations/0.2.md`.
+
 ## [0.2.9] - 2026-08-28
 
 ### Added

{cli_agent_runner-0.2.9 → cli_agent_runner-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cli-agent-runner
-Version: 0.2.9
+Version: 0.2.11
 Summary: Restart-on-exit supervisor for autonomous CLI agents
 Project-URL: Homepage, https://github.com/wan9yu/cli-agent-runner
 Project-URL: Documentation, https://github.com/wan9yu/cli-agent-runner#readme

{cli_agent_runner-0.2.9 → cli_agent_runner-0.2.11}/agent_runner/_emit.py

@@ -188,8 +188,13 @@ def emit_transient_error_detected(
     reset_at_epoch: int,
     round_num: int,
     raw: str,
+    phase: str = "",
 ) -> None:
-    """Emit detection of a transient agent error (rate limit / 5xx / timeout)."""
+    """Emit detection of a transient agent error (rate limit / 5xx / timeout).
+
+    ``phase`` is the rotation phase the failing round ran under ("" when the
+    config has no ``[phases]``). It lets a ``phase_policy = "skip"`` serve loop
+    step over *this* phase while a healthy sibling keeps running."""
     from agent_runner._redact import redact_secrets
     from agent_runner.events import TRANSIENT_ERROR_DETECTED, emit
 
@@ -202,6 +207,7 @@ def emit_transient_error_detected(
         reset_at_epoch=reset_at_epoch,
         round_num=round_num,
         raw=raw,
+        phase=phase,
     )
 
 

cli_agent_runner-0.2.11/agent_runner/_throttle.py

@@ -0,0 +1,372 @@
+"""Throttle state helpers — read events.jsonl tail for transient error state,
+and own the supervisor-side back-off sleep.
+
+Internal module. Callers: cli/serve_cmd.py (throttle scan + back-off + restart
+delay), api.py (peek). The back-off lives here, NOT in runner.py, to satisfy the
+ouroboros defense: runner.py writes events.jsonl but must never read it back
+(§3 module boundary invariant), and back-off is driven by the events-derived
+throttle state this module scans.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from datetime import UTC
+from pathlib import Path
+from typing import Any
+
+from agent_runner.api_types import TransientErrorState
+from agent_runner.clock import SYSTEM_CLOCK, Clock
+from agent_runner.events import (
+    TRANSIENT_ERROR_DETECTED,
+    TRANSIENT_ERROR_RECOVERED,
+    parse_iso_ms,
+)
+
+# _scan_events_for_transient sentinel: file held no transient event at all
+# (distinct from "latest transient was a recovered" → None).
+_NO_TRANSIENT = object()
+
+
+def _iter_events(path: Path):
+    """Yield parsed event dicts from a JSONL file; skip blank / corrupt lines."""
+    with path.open() as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                yield json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+
+def _scan_events_for_transient(path: Path):
+    """The file's LAST transient event: the detected dict, ``None`` (latest was a
+    recovered), or ``_NO_TRANSIENT`` (no transient event in the file).
+
+    Forward single-pass, O(1) memory — it keeps only the latest transient event,
+    never a 100-line tail. That matters for throttle-aware skip: the loop keeps
+    emitting rounds on healthy phases during a throttle, so the ``detected`` event
+    can be thousands of lines back; a bounded tail would scroll it out and make the
+    supervisor forget the throttle mid-window."""
+    latest: Any = _NO_TRANSIENT
+    for ev in _iter_events(path):
+        kind = ev.get("event")
+        if kind == TRANSIENT_ERROR_DETECTED:
+            latest = ev
+        elif kind == TRANSIENT_ERROR_RECOVERED:
+            latest = None
+    return latest
+
+
+def _latest_unrecovered_detected(log_dir: Path) -> dict[str, Any] | None:
+    """The most recent ``transient_error_detected`` with no
+    ``transient_error_recovered`` after it, or None. Used by
+    :func:`_check_throttle_state` (still-throttled?) — the single-throttle "latest"
+    view kept for the non-skip (wait / back_off / stop) paths and peek.
+
+    Scans the newest monthly ``events-*.jsonl`` and, only if it holds no transient
+    event yet, the previous month's — so a throttle that spans a month boundary
+    (detected in the old file, cleared in the new) is not orphaned."""
+    candidates = sorted(log_dir.glob("events-*.jsonl"))
+    for path in reversed(candidates[-2:]):
+        result = _scan_events_for_transient(path)
+        if result is not _NO_TRANSIENT:
+            return result  # a detected dict, or None (latest transient was recovered)
+    return None
+
+
+def _latest_transient_per_agent(log_dir: Path) -> dict[str, Any]:
+    """Per agent label, the last transient event across the newest TWO monthly files:
+    the ``detected`` dict, or ``None`` if the latest was a ``recovered``.
+
+    Unlike :func:`_latest_unrecovered_detected` (which early-exits at the first file
+    holding any transient), this always merges BOTH files per agent — a throttle for
+    agent A in the old file must not be masked by agent B's event in the newer one.
+    Iterates old-then-new so the newest event per agent wins."""
+    latest: dict[str, Any] = {}
+    for path in sorted(log_dir.glob("events-*.jsonl"))[-2:]:
+        for ev in _iter_events(path):
+            kind = ev.get("event")
+            if kind == TRANSIENT_ERROR_DETECTED:
+                latest[str(ev.get("agent", "unknown"))] = ev
+            elif kind == TRANSIENT_ERROR_RECOVERED:
+                latest[str(ev.get("agent", "unknown"))] = None
+    return latest
+
+
+def _active_throttles(
+    log_dir: Path, *, clock: Clock = SYSTEM_CLOCK
+) -> dict[str, TransientErrorState]:
+    """Currently-active throttles keyed by AGENT label (binary basename), one entry
+    per agent whose latest transient event is an unrecovered ``detected`` with its
+    reset still in the future.
+
+    Rate limits are per-provider (per API key / agent binary), not per-phase — so the
+    serve skip-policy routes around EVERY phase sharing a throttled agent while a
+    healthy agent's phase keeps running. Restart-safe (events-derived)."""
+    now = clock.epoch()
+    active: dict[str, TransientErrorState] = {}
+    for agent, detected in _latest_transient_per_agent(log_dir).items():
+        if detected is None:
+            continue
+        reset_at = int(detected.get("reset_at_epoch", 0))
+        if reset_at <= now:
+            continue
+        active[agent] = TransientErrorState(
+            reset_at_epoch=reset_at,
+            classification=str(detected.get("classification", "rate_limit_account")),
+            agent=agent,
+            since_round=int(detected.get("round_num", 0)),
+            phase=str(detected.get("phase", "")),
+        )
+    return active
+
+
+def _check_throttle_state(
+    log_dir: Path, *, clock: Clock = SYSTEM_CLOCK
+) -> TransientErrorState | None:
+    """Scan events.jsonl tail for latest unmatched transient error.
+
+    Reads `transient_error_detected` / `transient_error_recovered` event names.
+    Returns TransientErrorState if currently throttled (reset still in future,
+    no matching recovered after). Restart-safe.
+
+    ``clock`` supplies the wall clock the reset is compared against — inject a
+    ``FakeClock`` in tests to pin an exact instant. Only the reset comparison
+    reads it; the event scan is pure.
+    """
+    now = clock.epoch()
+    latest_detected = _latest_unrecovered_detected(log_dir)
+    if latest_detected is None:
+        return None
+    reset_at = int(latest_detected.get("reset_at_epoch", 0))
+    if reset_at <= now:
+        return None  # Reset already passed without recovery emit; treat as recovered
+
+    classification = str(latest_detected.get("classification", "rate_limit_account"))
+
+    return TransientErrorState(
+        reset_at_epoch=reset_at,
+        classification=classification,
+        agent=str(latest_detected.get("agent", "unknown")),
+        since_round=int(latest_detected.get("round_num", 0)),
+        phase=str(latest_detected.get("phase", "")),
+    )
+
+
+def _throttled_for_s(ts: Any, *, clock: Clock = SYSTEM_CLOCK) -> int:
+    """Seconds since a detected event's ``ts`` (best-effort; 0 if unparseable).
+    ``clock`` supplies now; inject a ``FakeClock`` for exact-value tests."""
+    if not ts:
+        return 0
+    try:
+        detected = parse_iso_ms(str(ts))  # events.py owns the Z→+00:00 workaround
+    except (ValueError, TypeError):
+        return 0
+    if detected.tzinfo is None:
+        detected = detected.replace(tzinfo=UTC)  # hand-injected naive ts → treat as UTC
+    return max(0, int(clock.epoch() - detected.timestamp()))
+
+
+def pending_recovered(log_dir: Path, *, clock: Clock = SYSTEM_CLOCK) -> list[tuple[str, str, int]]:
+    """One ``(agent, classification, throttled_for_s)`` per agent whose throttle
+    cleared WITHOUT a recovered breadcrumb — its latest transient event is a
+    ``detected`` with ``reset_at <= now`` and NO ``transient_error_recovered`` after.
+
+    This is the throttle-aware-skip path: the loop rotated to a healthy phase instead
+    of calling ``_apply_back_off`` (which emits its own recovered), so a multi-hour
+    throttle would otherwise close with no breadcrumb. Events-derived → restart-safe
+    AND never double-emits: once the caller emits recovered, that event sits after the
+    detected one and the agent drops out. Per-agent, so an agent that cleared is
+    reported even while a sibling is still throttled (overlapping recovery); an empty
+    list means nothing cleared (all still throttled, or the back-off path already left
+    a recovered)."""
+    now = clock.epoch()
+    cleared: list[tuple[str, str, int]] = []
+    for agent, detected in _latest_transient_per_agent(log_dir).items():
+        if detected is None or int(detected.get("reset_at_epoch", 0)) > now:
+            continue  # None = latest was a recovered; > now = still throttled
+        cleared.append(
+            (
+                agent,
+                str(detected.get("classification", "rate_limit_account")),
+                _throttled_for_s(detected.get("ts"), clock=clock),
+            )
+        )
+    return cleared
+
+
+# Module-level supervisor state — bucket → consecutive-failure count.
+# Cleared by reset_counters() or by serve restart.
+_consecutive_failures: dict[str, int] = {}
+
+
+def compute_adjusted_reset_at(
+    *,
+    classification: str,
+    original_reset_at_epoch: int,
+    agent: str,
+    log_dir: Path,
+    clock: Clock = SYSTEM_CLOCK,
+) -> tuple[int, int, bool]:
+    """Apply exp backoff for estimated-class transient errors.
+
+    Returns (applied_reset_at_epoch, consecutive_count, capped_by_absolute_max).
+
+    For server-authoritative classification (``rate_limit_account``): returns
+    the original reset epoch verbatim, never increments the counter, and
+    never emits an adjustment event. Anthropic's resetsAt is authoritative.
+
+    For any classification absent from ``_BACK_OFF_DEFAULTS`` — i.e. one a
+    third-party plugin defined itself, which ``api_types.py`` types
+    ``classification`` as ``str`` to permit — the same verbatim path applies:
+    core has no base duration for it, so the emitter's reset_at_epoch is the
+    only non-invented answer.
+
+    For estimated classifications (``rate_limit_model``, ``api_transient_5xx``,
+    ``api_timeout``): increments the counter for this bucket, computes
+    duration = base × 2^min(n, _EXP_CAP), caps at _ABSOLUTE_CAP_S, emits
+    ``transient_error_backoff_capped`` if multiplier > 1 or capped.
+    """
+    from agent_runner._emit import emit_transient_error_backoff_capped
+    from agent_runner.builtin_plugins._constants import (
+        _ABSOLUTE_CAP_S,
+        _BACK_OFF_DEFAULTS,
+        _EXP_CAP,
+    )
+
+    if classification == "rate_limit_account" or classification not in _BACK_OFF_DEFAULTS:
+        # Server-authoritative, or a plugin's own classification (api_types.py types
+        # `classification` as str precisely so plugins can add their own): the emitter
+        # supplied reset_at_epoch, so respect it verbatim and never touch the counter.
+        return (original_reset_at_epoch, 0, False)
+
+    # Estimated class: apply exp backoff.
+    base = _BACK_OFF_DEFAULTS[classification]
+    n = _consecutive_failures.get(classification, 0)
+    multiplier = 2 ** min(n, _EXP_CAP)
+    extended_duration = base * multiplier
+    capped_by_absolute_max = extended_duration > _ABSOLUTE_CAP_S
+    applied_duration = min(extended_duration, _ABSOLUTE_CAP_S)
+    applied_reset_at = int(clock.epoch()) + applied_duration
+
+    new_count = n + 1
+    _consecutive_failures[classification] = new_count
+
+    # Emit observability event when supervisor adjusted the wait.
+    if multiplier > 1 or capped_by_absolute_max:
+        emit_transient_error_backoff_capped(
+            log_dir,
+            classification=classification,
+            agent=agent,
+            requested_sleep_s=int(base),
+            applied_sleep_s=applied_duration,
+            original_reset_at_epoch=original_reset_at_epoch,
+            applied_reset_at_epoch=applied_reset_at,
+            consecutive_count=new_count,
+            capped_by_absolute_max=capped_by_absolute_max,
+        )
+
+    return (applied_reset_at, new_count, capped_by_absolute_max)
+
+
+def reset_counters() -> None:
+    """Clear all bucket counters. Called by serve loop when no active throttle."""
+    _consecutive_failures.clear()
+
+
+_BACK_OFF_CAP_S = 28800  # 8h — defensive cap; 1.6× the 5h-window
+_BACK_OFF_JITTER_MIN_S = 5
+_BACK_OFF_JITTER_MAX_S = 30
+
+
+def _interruptible_sleep(
+    total_s: float, stop: dict[str, bool], *, clock: Clock = SYSTEM_CLOCK, chunk_s: int = 30
+) -> bool:
+    """Sleep ``total_s`` in ``<= chunk_s`` slices, re-checking ``stop`` at each
+    boundary; return True iff ``stop["requested"]`` cut it short. Shared by the serve
+    restart delay and :func:`_apply_back_off` so a SIGTERM lands within one chunk.
+
+    Counts down the *intended* nap per slice rather than measuring a wall/monotonic
+    deadline: NTP-step immune (no clock read for the deadline) AND does not busy-spin
+    when ``clock.sleep`` is a no-op (a test patching ``time.sleep`` runs it instantly
+    instead of looping until real time advances)."""
+    remaining = float(total_s)
+    while remaining > 0:
+        if stop["requested"]:
+            return True
+        nap = min(float(chunk_s), remaining)
+        clock.sleep(nap)
+        remaining -= nap
+    return False
+
+
+def _apply_back_off(
+    log_dir: Path,
+    throttle: TransientErrorState,
+    *,
+    stop: dict[str, bool],
+    clock: Clock = SYSTEM_CLOCK,
+    chunk_s: int = 30,
+) -> bool:
+    """Sleep until adjusted reset_at + jitter, then emit recovered. Returns True iff
+    ``stop["requested"]`` was set before the sleep completed — an interrupted back-off,
+    where the caller must break WITHOUT treating the throttle as recovered: the reset
+    has not passed, so a recovered breadcrumb would poison restart-safe state.
+
+    The sleep is chunked and re-checks ``stop`` at each boundary (see
+    :func:`_interruptible_sleep`), so a SIGTERM lands within one chunk instead of after
+    a multi-hour window.
+
+    For estimated-class classifications (rate_limit_model / api_transient_5xx /
+    api_timeout), applies exp backoff on consecutive failures via
+    :func:`compute_adjusted_reset_at`. For server-authoritative rate_limit_account, the
+    original reset_at_epoch is used verbatim. The 8h cap is a last-line defense against a
+    malformed (far-future) reset epoch (e.g. a manual event with a far-future ts).
+    """
+    from agent_runner._emit import (
+        emit_transient_error_backoff_capped,
+        emit_transient_error_recovered,
+    )
+
+    adjusted_reset_at, _consecutive_count, _capped = compute_adjusted_reset_at(
+        classification=throttle.classification,
+        original_reset_at_epoch=throttle.reset_at_epoch,
+        agent=throttle.agent,
+        log_dir=log_dir,
+        clock=clock,
+    )
+
+    requested = (
+        adjusted_reset_at
+        - clock.epoch()
+        + random.uniform(_BACK_OFF_JITTER_MIN_S, _BACK_OFF_JITTER_MAX_S)
+    )
+    if requested > _BACK_OFF_CAP_S:
+        # Defensive: malformed reset epoch. The exp-backoff layer caps at 30min, so
+        # legitimate flow never reaches this branch.
+        emit_transient_error_backoff_capped(
+            log_dir,
+            classification=throttle.classification,
+            agent=throttle.agent,
+            requested_sleep_s=int(requested),
+            applied_sleep_s=_BACK_OFF_CAP_S,
+        )
+        sleep_s = float(_BACK_OFF_CAP_S)
+    else:
+        sleep_s = max(requested, 0.0)
+
+    sleep_start = clock.epoch()
+    if _interruptible_sleep(sleep_s, stop, clock=clock, chunk_s=chunk_s):
+        return True  # SIGTERM during back-off — leave the throttle active, no breadcrumb
+
+    emit_transient_error_recovered(
+        log_dir,
+        classification=throttle.classification,
+        agent=throttle.agent,
+        throttled_for_s=int(clock.epoch() - sleep_start),
+    )
+    return False

{cli_agent_runner-0.2.9 → cli_agent_runner-0.2.11}/agent_runner/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.9'
-__version_tuple__ = version_tuple = (0, 2, 9)
+__version__ = version = '0.2.11'
+__version_tuple__ = version_tuple = (0, 2, 11)
 
 __commit_id__ = commit_id = None

{cli_agent_runner-0.2.9 → cli_agent_runner-0.2.11}/agent_runner/agent_runtime.py

@@ -15,13 +15,14 @@ import shutil
 import signal
 import subprocess  # noqa: TID251 — sanctioned subprocess caller
 import threading
-import time
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from pathlib import Path
 
 import psutil
 
+from agent_runner.clock import SYSTEM_CLOCK, Clock
+
 REAP_GRACE_S = 5
 
 
@@ -68,15 +69,17 @@ def _build_argv(command: list[str], prompt_arg_template: list[str], prompt: str)
     return list(command) + [a.replace("{prompt}", prompt) for a in prompt_arg_template]
 
 
-def _kill_pgroup(proc: subprocess.Popen) -> None:
+def _kill_pgroup(proc: subprocess.Popen, clock: Clock = SYSTEM_CLOCK) -> None:
     pgid = proc.pid
     try:
         os.killpg(pgid, signal.SIGTERM)
     except OSError:
         pass
-    deadline = time.time() + REAP_GRACE_S
-    while time.time() < deadline and proc.poll() is None:
-        time.sleep(0.1)
+    deadline = (
+        clock.monotonic() + REAP_GRACE_S
+    )  # monotonic: an NTP step must not stretch/skip the reap
+    while clock.monotonic() < deadline and proc.poll() is None:
+        clock.sleep(0.1)
     try:
         os.killpg(pgid, signal.SIGKILL)
     except OSError:
@@ -173,6 +176,7 @@ def run(
     progress_interval_s: int = 0,
     on_grace_extended: Callable[[list[dict], list[dict]], None] | None = None,
     grace_kill_ignore_patterns: list[re.Pattern[str]] | None = None,
+    clock: Clock = SYSTEM_CLOCK,
 ) -> RunResult:
     """Spawn the agent subprocess and wait for exit or timeout.
 
@@ -213,7 +217,7 @@ def run(
     env = {**os.environ, **env_extra, "PWD": str(work_dir)}
     log_path.parent.mkdir(parents=True, exist_ok=True)
     log_file = log_path.open("w", encoding="utf-8")
-    start = time.time()
+    start = clock.monotonic()  # all round-duration/deadline math is monotonic (NTP-safe)
     last_progress_at = start
     proc = subprocess.Popen(
         argv,
@@ -246,13 +250,13 @@ def run(
     try:
         while True:
             ret = proc.poll()
-            now = time.time()
+            now = clock.monotonic()  # duration / R1128 hard-wall / grace / interval — all monotonic
             if ret is not None:
                 duration = now - start
                 return RunResult(exit_code=ret, duration_s=duration, timed_out=False, pid=proc.pid)
             if now - start > timeout_s:
-                _kill_pgroup(proc)
-                duration = time.time() - start
+                _kill_pgroup(proc, clock)
+                duration = clock.monotonic() - start
                 exit_code = proc.returncode if proc.returncode is not None else -1
                 return RunResult(
                     exit_code=exit_code, duration_s=duration, timed_out=True, pid=proc.pid
@@ -276,8 +280,8 @@ def run(
                                 on_grace_extended(live, ignored)
                             grace_extended_emitted = True
                     else:
-                        _kill_pgroup(proc)
-                        duration = time.time() - start
+                        _kill_pgroup(proc, clock)
+                        duration = clock.monotonic() - start
                         exit_code = proc.returncode if proc.returncode is not None else -1
                         return RunResult(
                             exit_code=exit_code,
@@ -293,7 +297,7 @@ def run(
                     try:
                         st = log_path.stat()
                         log_size_kb = st.st_size // 1024
-                        last_write_age_s = max(0, int(now - st.st_mtime))
+                        last_write_age_s = max(0, int(clock.epoch() - st.st_mtime))
                     except OSError:
                         log_size_kb = 0
                         last_write_age_s = 0
@@ -305,6 +309,6 @@ def run(
                         }
                     )
                     last_progress_at = now
-            time.sleep(0.2)
+            clock.sleep(0.2)
     finally:
         log_file.close()

{cli_agent_runner-0.2.9 → cli_agent_runner-0.2.11}/agent_runner/api.py

@@ -15,7 +15,6 @@ import shutil
 import signal
 import subprocess  # noqa: TID251 — api uses systemctl + ssh, both subprocess
 import sysconfig
-import time
 from collections.abc import Iterator, Sequence
 from pathlib import Path
 from typing import Any, Literal, TextIO
@@ -30,6 +29,7 @@ from agent_runner.api_types import (
     ServiceStatus,
     select_path,
 )
+from agent_runner.clock import SYSTEM_CLOCK
 from agent_runner.config import Config, RuntimeConfig, load_config
 from agent_runner.lifecycle import (
     PIDFile,
@@ -52,6 +52,13 @@ from agent_runner.service_unit import (
 # facade without coupling to runner (runner imports api, not the reverse).
 PERMANENT_CONFIG_EXIT = 78
 
+# Exit code serve returns when the crash-loop breaker trips (a give-up stop, like
+# config_broken). Distinct from 78 so `systemctl status` / SuccessExitStatus can
+# tell the two "needs intervention" stops apart. 75 = EX_TEMPFAIL (sysexits).
+# The systemd unit lists both in RestartPreventExitStatus so a deliberate stop
+# stays stopped (and visibly failed) while an unexpected supervisor crash restarts.
+CRASH_LOOP_EXIT = 75
+
 # Crash-loop circuit breaker (b12). The serve loop escalates the restart delay
 # on consecutive UNKNOWN short crashes (non-zero exit, short duration, no
 # classified transient) and STOPS after CRASH_LOOP_THRESHOLD of them — the Run 6
@@ -323,13 +330,13 @@ def kill(project: str | Path) -> ServiceStatus:
     if pid is None:
         return status(project)
     send_signal_to_pid(pid, signal.SIGTERM)
-    deadline = time.time() + 5
+    deadline = SYSTEM_CLOCK.epoch() + 5
     alive = True
-    while time.time() < deadline:
+    while SYSTEM_CLOCK.epoch() < deadline:
         alive = pid_alive(pid)
         if not alive:
             break
-        time.sleep(0.1)
+        SYSTEM_CLOCK.sleep(0.1)
     if alive:
         send_signal_to_pid(pid, signal.SIGKILL)
     return ServiceStatus(mode=ServiceMode.PID_FILE, active=alive, pid=pid)
@@ -438,9 +445,16 @@ def peek(
     )
     recent_blips = _recent_events_of_kind(parsed_events, AGENT_NETWORK_BLIP, _RECENT_BLIPS_LIMIT)
 
-    from agent_runner._throttle import _check_throttle_state
+    from agent_runner._throttle import _active_throttles, _check_throttle_state
 
     throttle = _check_throttle_state(log_dir)
+    active = _active_throttles(log_dir)
+    if throttle is None and active:
+        # The global-latest scalar view can be None while a sibling agent is still
+        # throttled (the newest transient event is another agent's recovered). Fall
+        # back to the active throttle that clears LAST so the scalar fields stay
+        # coherent and the plural throttled_agents view is not dropped to null.
+        throttle = max(active.values(), key=lambda s: s.reset_at_epoch)
     rate_limit: RateLimitState | None = None
     if throttle is not None:
         rate_limit = RateLimitState(
@@ -448,6 +462,8 @@ def peek(
             limit_type=throttle.classification,
             agent=throttle.agent,
             since_round=throttle.since_round,
+            phase=throttle.phase,
+            throttled_agents=tuple(sorted(active)),
         )
     raw_service = status(project if project is not None else work_dir)
     svc = dataclasses.replace(raw_service, rate_limit=rate_limit)
@@ -567,7 +583,7 @@ def _monitor_loop_iter(
                 log_dir=cfg.runtime.log_dir,
                 allowed_stop_names=cfg.monitor.auto_stop_on,
             )
-        time.sleep(interval_s)
+        SYSTEM_CLOCK.sleep(interval_s)
 
 
 def _tail_events_jsonl(
@@ -586,7 +602,6 @@ def _tail_events_jsonl(
     appears, it is picked up from byte 0.
     """
     import json as _json
-    import time as _time
 
     seen_positions: dict[Path, int] = {}
     if start_at_now:
@@ -620,7 +635,7 @@ def _tail_events_jsonl(
                     any_new = True
                 seen_positions[path] = f.tell()
         if not any_new:
-            _time.sleep(poll_interval_s)
+            SYSTEM_CLOCK.sleep(poll_interval_s)
 
 
 def _primary_prompt_file(cfg: Config) -> Path | None: