code-coordinator 0.5.577__tar.gz → 0.5.578__tar.gz

{code_coordinator-0.5.577 → code_coordinator-0.5.578}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.577
+Version: 0.5.578
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.577 → code_coordinator-0.5.578}/code_coordinator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-coordinator
-Version: 0.5.577
+Version: 0.5.578
 Summary: Coordinate Claude Code workers across multiple machines and repos.
 Author-email: John Donaghy <johnfdonaghy@gmail.com>
 License-Expression: LicenseRef-FSL-1.1-MIT

{code_coordinator-0.5.577 → code_coordinator-0.5.578}/code_coordinator.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "0.5.577",
+  "tag": "0.5.578",
   "distance": 0,
-  "node": "gd912c07c97846af080c719b433367120de7bfccb",
+  "node": "g27497ebf63627571d2a865a710e3c9c502a8053d",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-10-03"

{code_coordinator-0.5.577 → code_coordinator-0.5.578}/coord/ci_github.py

@@ -797,10 +797,16 @@ class GitHubCi:
         # out to `gh pr checks` directly (a non-zero exit there can still
         # carry usable JSON on stdout, which `_gh()`'s raise-on-nonzero
         # contract can't express, see `get_pr_checks`'s own docstring) — so
-        # this call is the *only* forge-availability observation this read
-        # produces. It stands alone, not "layered on top" of anything.
-        # Only fires on a real cache miss — a cached `list_checks_for_pr`
-        # hit never reaches here, so this costs nothing extra either.
+        # the `record_ci_check_fetch` call below is the only *CI-outcome*
+        # observation this read produces: the distribution of pass/fail/
+        # pending per check. (#3536 review follow-up: `get_pr_checks` itself
+        # now also feeds the generic, caller-attributed `record_gh_call` seam
+        # and the shared `github_throttle` pacing/backoff — same as every
+        # `_gh()`-backed call — so this is no longer the *only*
+        # forge-availability signal the read produces, just the only
+        # CI-specific one.) Only fires on a real cache miss — a cached
+        # `list_checks_for_pr` hit never reaches here, so this costs nothing
+        # extra either.
         _t0 = time.monotonic()
         try:
             raw = github_ops.get_pr_checks(repo, number)

{code_coordinator-0.5.577 → code_coordinator-0.5.578}/coord/drive_queue.py

@@ -438,6 +438,25 @@ def is_permanent_block_reason(text: str | None) -> bool:
 # the ceiling is reached.
 MAX_BLOCKED_RESUMES = 3
 
+# #3536: default cap on how many CONFIRMED-CLEAR `blocked`-entry resumes/
+# merge-only releases one `plan_tick` call grants per repo, from EITHER
+# `_reconcile_blocked`'s gate-clear check (#2230) or `_reconcile_blocked_
+# after`'s `after=` prereq-chain resume (#2362/#2756) -- a latch clearing
+# for a whole repo's worth of `blocked` rows at once (#3386's baseline-red
+# gate, cleared by #3532) and a single landed prereq clearing every row
+# chained `--after` it are the SAME synchronized-poll shape, so both count
+# against one cap. Either one otherwise releases every row on the SAME
+# tick -- each then driving its own live `gh` re-checks on the very next
+# poll, which is exactly the pattern that trips GitHub's secondary rate
+# limit fleet-wide (the 2026-10-02 incident). 2 is deliberately small:
+# large enough that a single or double blocked entry (the overwhelming
+# common case, and every pre-#3536 test fixture) still resumes in the same
+# tick it clears, small enough that a dozen-row mass clear spreads over
+# several ticks instead of one. `None`/`0` disables the cap entirely
+# (unstaggered, pre-#3536 behaviour) — see `plan_tick`'s
+# `max_resumes_per_tick_per_repo` parameter.
+DEFAULT_MAX_RESUMES_PER_TICK_PER_REPO = 2
+
 # ── deploy-gate states (#1757) ───────────────────────────────────────────────
 #
 # `hold_state` is the gate's LIFECYCLE, orthogonal to the entry's queue
@@ -2524,6 +2543,17 @@ class Reconcile:
       see :func:`coord.commands.drive_queue._run_merge_only_candidates`),
       which is why this Reconcile's own ``updates`` stays empty: neither
       outcome is knowable at the moment :func:`plan_tick` returns.
+    * ``resume_staggered`` — #3536: either `_reconcile_blocked`'s gate-clear
+      check or `_reconcile_blocked_after`'s `after=` prereq-chain resume
+      read CONFIRMED CLEAR (same evidence ``resumed``/``merge_only`` act
+      on), but this tick's *max_resumes_per_tick_per_repo* cap for this
+      entry's repo was already spent by an earlier entry THIS SAME tick —
+      a mass latch clear, or a single landed prereq clearing many rows
+      chained `--after` it, releasing a dozen rows at once, the exact
+      synchronized-poll shape that trips GitHub's secondary rate limiter.
+      Writes NO state: the entry stays `blocked` and gets first refusal at
+      the same check next tick, so a repo's worth of newly-clear rows
+      releases gradually over several ticks instead of one.
     """
 
     key: str
@@ -6384,6 +6414,7 @@ def plan_tick(
     roll_pending_reason: str = "",
     live_prereq_terminal: Mapping[str, bool] | None = None,
     fix_round_config_default: int | None = None,
+    max_resumes_per_tick_per_repo: int | None = DEFAULT_MAX_RESUMES_PER_TICK_PER_REPO,
 ) -> TickPlan:
     """Decide one tick.  Pure; the caller executes the returned plan.
 
@@ -6710,6 +6741,19 @@ def plan_tick(
     relaunch decision and the launch it gates always agree on the budget.
     ``None`` (the default) falls back to :data:`DEFAULT_TICK_MAX_FIX_ROUNDS`,
     same as every other caller of :func:`effective_max_fix_rounds`.
+
+    *max_resumes_per_tick_per_repo* (#3536) caps how many CONFIRMED-CLEAR
+    `blocked`-entry resumes/merge-only releases THIS tick grants per repo,
+    from EITHER `_reconcile_blocked`'s gate-clear check or
+    `_reconcile_blocked_after`'s `after=` prereq-chain resume (#2362/#2756)
+    — see :data:`DEFAULT_MAX_RESUMES_PER_TICK_PER_REPO` for why both count
+    against the same cap. The ``resumed``/``merge_only`` reconciles beyond
+    the cap are replaced with a ``resume_staggered`` one: no state write,
+    so the entry stays `blocked` and re-enters this same check next tick,
+    where it either wins a slot then or is staggered again. Every state
+    other than `blocked` is unaffected. ``None`` or ``0`` disables the cap
+    entirely (unstaggered — the only behaviour this repo's test suite
+    exercised before #3536).
     """
     ordered = sorted(entries, key=lambda e: (e.position, e.key))
     states: dict[str, str] = {e.key: e.state for e in ordered}
@@ -6766,6 +6810,10 @@ def plan_tick(
     # #2350: entries this tick decided to attempt `coord merge --only` on
     # directly rather than relaunch — see `Reconcile`'s `merge_only` outcome.
     merge_only_candidates: list[QueueEntry] = []
+    # #3536: how many `_reconcile_blocked` CONFIRMED-CLEAR resumes/merge-only
+    # releases this tick has already granted, per repo — see the stagger
+    # check inside the `STATE_BLOCKED` branch below.
+    resume_quota_used: dict[str, int] = {}
     occupied = 0
     # #1972: the same count, keyed by repo.  Populated from the SAME
     # `reconcile.occupies` verdict as `occupied` above — one source of truth, so
@@ -6971,6 +7019,59 @@ def plan_tick(
                     live_blocked_unreadable,
                     live_blocked_gate_reason,
                 )
+            # #3536: a latch that was gating many entries at once (#3386's
+            # baseline-red gate, cleared by one merge) reads CONFIRMED CLEAR
+            # for every one of them on the SAME tick — without a cap, EITHER
+            # `_reconcile_blocked`'s gate-clear check above OR
+            # `_reconcile_blocked_after`'s prereq-chain resume (#2362/#2756:
+            # one landed dep clearing the SAME unsatisfiable verdict for
+            # every row chained `--after` it) hands back `resumed`/
+            # `merge_only` for all of them here, and every one of those rows
+            # then drives its OWN live `gh` re-checks (branch head, CI
+            # status) on this same tick's very next poll. A dozen rows doing
+            # that together is exactly the synchronized-poll pattern
+            # `coord.github_throttle`'s module docstring identifies as what
+            # trips GitHub's secondary rate limiter — the 2026-10-02 incident
+            # this issue describes, and issue #3536 itself names BOTH the
+            # #2230 and #2362 paths as in scope for the stagger (a review
+            # round of this same fix caught that an earlier version capped
+            # only `_reconcile_blocked`'s gate-clear path, leaving the
+            # `after=` cascade above unstaggered even though a many-rows-
+            # chained-on-one-prereq clear is the identical synchronized-poll
+            # shape). Stagger it: only the first
+            # `max_resumes_per_tick_per_repo` CONFIRMED-CLEAR resumes per
+            # repo actually release this tick, from EITHER path; the rest
+            # stay `blocked` (no state write, so they re-enter this exact
+            # branch next tick and get first refusal then) with a reason
+            # explaining why, instead of all firing at once.
+            if (
+                blocked_reconcile is not None
+                and blocked_reconcile.outcome in ("resumed", "merge_only")
+                and max_resumes_per_tick_per_repo is not None
+                and max_resumes_per_tick_per_repo > 0
+            ):
+                used = resume_quota_used.get(entry.repo, 0)
+                if used >= max_resumes_per_tick_per_repo:
+                    reason = (
+                        f"{entry.key}'s block cleared (merge gate reads "
+                        "clear, or a chained `after=` prereq landed), but "
+                        f"this tick already released "
+                        f"{max_resumes_per_tick_per_repo}/"
+                        f"{max_resumes_per_tick_per_repo} {entry.repo} "
+                        "resume slot(s) — staggering the rest to avoid "
+                        "a mass clear tripping GitHub's secondary "
+                        "rate limit fleet-wide (#3536); resuming on a "
+                        "later tick"
+                    )
+                    blocked_reconcile = Reconcile(
+                        entry.key,
+                        "resume_staggered",
+                        reason,
+                        occupies=False,
+                        updates=_refresh_only(reason, entry.last_reason),
+                    )
+                else:
+                    resume_quota_used[entry.repo] = used + 1
             if blocked_reconcile is not None:
                 reconciles.append(blocked_reconcile)
                 if blocked_reconcile.outcome == "merge_only":

{code_coordinator-0.5.577 → code_coordinator-0.5.578}/coord/github_ops.py

@@ -486,6 +486,67 @@ def _resolve_caller(explicit: str) -> str:
         return "unknown"
 
 
+def _precall_throttle(
+    args: Sequence[str], caller: str, *, force_through_backoff: bool = False,
+) -> None:
+    """Shared pre-call seam: consult the reactive backoff, then take a token
+    from the proactive pacing bucket (#3536 review follow-up).
+
+    Extracted out of :func:`_gh` so :func:`get_pr_checks` — which cannot
+    funnel through ``_gh()`` itself, because a non-zero ``gh pr checks`` exit
+    can still carry usable JSON on stdout and ``_gh()``'s raise-on-nonzero
+    contract can't express that, see that function's own docstring — still
+    gets paced and damped the same way every ``_gh()``-backed call does.
+    ``get_pr_checks`` is the single ``gh`` sink for
+    :class:`coord.ci_github.GitHubCi`, i.e. the dominant CI-check/branch-head
+    read the #3536 incident actually stalled on; a `pace()` call added only
+    inside `_gh()` would leave exactly that path unpaced, which an earlier
+    review round of this issue caught.
+
+    Raises :class:`GhRateLimitError` when deep inside an active backoff
+    window (same as ``_gh()``); otherwise sleeps for the jittered amount
+    ``consult()``/``pace()`` compute (possibly ``0.0``) and returns.
+    """
+    backoff_sleep_s, active_backoff = github_throttle.consult()
+    if active_backoff is not None:
+        remaining = active_backoff.until - time.time()
+        if remaining > github_throttle.MAX_PRECALL_SLEEP_S and not force_through_backoff:
+            # Still well inside a known backoff window -- don't add another
+            # request to a limiter that only recovers when the rate drops.
+            # This is not a fresh observation, so it is not re-recorded.
+            record_gh_call(
+                args, outcome="transient", duration_s=0.0, caller=caller,
+                detail=(
+                    f"skipped: coordinated backoff active ({active_backoff.reason}, "
+                    f"{remaining:.0f}s remaining)"
+                ),
+            )
+            raise GhRateLimitError(
+                f"gh {' '.join(args)} skipped: GitHub {active_backoff.reason} "
+                f"backoff active for {remaining:.0f}s more "
+                f"(status={active_backoff.status}, request_id={active_backoff.request_id})",
+                status_code=active_backoff.status,
+                request_id=active_backoff.request_id,
+                retry_after_s=remaining,
+                secondary=active_backoff.reason == "secondary_rate_limit",
+                from_cache=True,
+            )
+        if backoff_sleep_s > 0:
+            time.sleep(backoff_sleep_s)
+    # #3536: proactive pacing, ahead of any 403 -- a reactive backoff above
+    # has nothing to say about a caller that hasn't been rate-limited yet,
+    # which is exactly the mass-resume shape #3536 describes (a dozen
+    # newly-unblocked rows each making their own first live `gh` call with
+    # no shared history of a hit). Both ``_gh()`` and ``get_pr_checks()``
+    # call this one seam, so pacing it once here paces every polling path
+    # (drive-queue merge-wait re-checks -- including the CI-check read
+    # itself -- the merge-queue sweep, `coord notify` ticks) without each one
+    # needing its own call to `github_throttle.pace()`.
+    pace_sleep_s = github_throttle.pace()
+    if pace_sleep_s > 0:
+        time.sleep(pace_sleep_s)
+
+
 def _gh(*args: str, caller: str = "", force_through_backoff: bool = False) -> str:
     """Run ``gh`` with *args* and return its stdout, or raise :class:`GhError`.
 
@@ -536,32 +597,7 @@ def _gh(*args: str, caller: str = "", force_through_backoff: bool = False) -> st
     low-frequency caller from being permanently outbid by them.
     """
     caller = _resolve_caller(caller)
-    backoff_sleep_s, active_backoff = github_throttle.consult()
-    if active_backoff is not None:
-        remaining = active_backoff.until - time.time()
-        if remaining > github_throttle.MAX_PRECALL_SLEEP_S and not force_through_backoff:
-            # Still well inside a known backoff window -- don't add another
-            # request to a limiter that only recovers when the rate drops.
-            # This is not a fresh observation, so it is not re-recorded.
-            record_gh_call(
-                args, outcome="transient", duration_s=0.0, caller=caller,
-                detail=(
-                    f"skipped: coordinated backoff active ({active_backoff.reason}, "
-                    f"{remaining:.0f}s remaining)"
-                ),
-            )
-            raise GhRateLimitError(
-                f"gh {' '.join(args)} skipped: GitHub {active_backoff.reason} "
-                f"backoff active for {remaining:.0f}s more "
-                f"(status={active_backoff.status}, request_id={active_backoff.request_id})",
-                status_code=active_backoff.status,
-                request_id=active_backoff.request_id,
-                retry_after_s=remaining,
-                secondary=active_backoff.reason == "secondary_rate_limit",
-                from_cache=True,
-            )
-        if backoff_sleep_s > 0:
-            time.sleep(backoff_sleep_s)
+    _precall_throttle(args, caller, force_through_backoff=force_through_backoff)
     # #1896 Phase 0: time + classify every `gh` invocation through this one
     # seam so `coord diagnose --forge-availability` has real data on how
     # often the forge is actually unreachable, not just anecdote from one
@@ -2745,28 +2781,62 @@ def get_pr_checks(repo: str, number: int) -> list[dict]:
     empty (a real lookup failure: bad PR number, auth, rate-limit, an old gh
     that doesn't support ``--json`` at all, ...). The single ``gh`` sink for
     :class:`coord.ci_github.GitHubCi`, the CI backend behind the merge gate
-    (#1483).
+    (#1483) — and, per #3536's incident narrative, the dominant call the
+    merge gate's CI-wait polling actually makes, so it is paced and damped
+    the same way ``_gh()``-backed calls are even though it cannot itself
+    route through ``_gh()`` (see the non-zero-exit-with-usable-stdout note
+    above).
 
     Raises :class:`GhTooOldForJsonChecks` — instead of the generic
     ``RuntimeError`` below — when the installed ``gh`` doesn't recognise
     ``--json`` on ``pr checks`` at all (#1564 Addendum 2), so callers can
     surface a distinct, actionable "upgrade gh" message rather than lumping
     it in with ordinary read failures.
-    """
+
+    #3536 review follow-up: this used to shell out directly with no pacing
+    or backoff of its own, which meant the module-wide claim "every `gh`
+    call funnels through `_gh()`, so pacing it once there paces every
+    polling path" was false for exactly the path #3521 stalled on for 40+
+    minutes. It now calls :func:`_precall_throttle` (the same seam ``_gh()``
+    calls) before the network call, and feeds a rate-limit hit it detects
+    itself back into :mod:`coord.github_throttle` the same way ``_gh()``
+    does, so every OTHER caller on this host learns about a 403 this read
+    hits, not just the ones that happen to go through ``_gh()``.
+    """
+    caller = _resolve_caller("github_ops.get_pr_checks")
+    args = (
+        "pr", "checks", str(number),
+        "--repo", repo,
+        "--json", ",".join(PR_CHECKS_JSON_FIELDS),
+    )
+    _precall_throttle(args, caller)
+    _t0 = time.monotonic()
     result = subprocess.run(
-        [
-            "gh", "pr", "checks", str(number),
-            "--repo", repo,
-            "--json", ",".join(PR_CHECKS_JSON_FIELDS),
-        ],
+        ["gh", *args],
         capture_output=True, text=True, timeout=30,
     )
+    duration = time.monotonic() - _t0
     stdout = (result.stdout or "").strip()
     if result.returncode != 0 and not stdout:
         stderr = result.stderr.strip()
         if _GH_UNKNOWN_JSON_FLAG_MARKER in stderr:
+            record_gh_call(args, outcome="unreachable", duration_s=duration,
+                            detail=stderr, caller=caller)
             raise GhTooOldForJsonChecks(_gh_too_old_message(stderr))
+        is_rate_limit, is_secondary = _classify_rate_limit(stderr)
+        if is_rate_limit:
+            if not is_secondary and _primary_quota_healthy():
+                is_secondary = True
+            text_meta = _extract_rate_limit_detail(stderr)
+            github_throttle.record(
+                reason="secondary_rate_limit" if is_secondary else "primary_rate_limit",
+                status=text_meta.status, request_id=text_meta.request_id,
+                retry_after_s=text_meta.retry_after_s,
+            )
+        record_gh_call(args, outcome=_classify_gh_exit(stderr), duration_s=duration,
+                        detail=stderr, caller=caller)
         raise RuntimeError(f"gh pr checks failed: {stderr}")
+    record_gh_call(args, outcome="ok", duration_s=duration, caller=caller)
     # #1525: unlike the fail-open sites elsewhere in this module, a malformed
     # (non-empty) response here must NOT be swallowed to a quiet ``[]`` —
     # ``ci_github.GitHubCi._fetch`` deliberately catches the ``ValueError``

{code_coordinator-0.5.577 → code_coordinator-0.5.578}/coord/github_throttle.py

@@ -62,6 +62,33 @@ missing/unwritable state dir, a corrupt file, an unreachable daemon) without
 ever raising into or delaying a caller beyond the bound it already asked
 for — damping must never become a new way to break `gh` access, only a way
 to reduce it.
+
+**Proactive pacing (#3536), not just reactive backoff.** Everything above
+reacts to a 403 that already happened. #3536's incident showed that is not
+enough on its own: when a latch clears and a dozen `blocked` rows resume
+together, EVERY ONE of them immediately drives its own live `gh` re-checks
+(branch head, CI status) with no shared awareness of the others — none of
+them has hit a 403 yet, so :func:`consult` above has nothing to say, and the
+fleet trips the secondary limiter from a cold start, then backs off and
+resumes in lockstep exactly as the module docstring above already warns a
+synchronized-poll pattern will. :func:`pace` is the proactive half: a small,
+best-effort, file-backed token bucket (:func:`local_pace_acquire`) that every
+`gh` call takes a token from BEFORE it knows whether it will even be
+rate-limited, smoothing the aggregate request rate instead of only reacting
+once GitHub has already said no. Same failure posture as everything else
+here — a missing/unwritable state dir degrades to "no pacing" (0.0s),
+never a raised exception or an unbounded sleep.
+
+Today this is single-host only (the same file every process on this host
+shares, same pattern :func:`local_record`/:func:`local_consult` use) — true
+fleet-wide pacing across machines would need a daemon endpoint the way
+:func:`record`/:func:`consult` get one via ``/github-backoff``
+(:mod:`coord.board_service`/:mod:`coord.client`/:mod:`coord.serve_app`), which
+is out of scope for the files this change touches. In practice this still
+covers #3536's own incident: the board daemon host is where the drive-queue
+tick, `coord notify`, and the merge-queue sweep all actually run, so pacing
+their shared funnel on that one host already smooths the aggregate rate the
+incident describes.
 """
 
 from __future__ import annotations
@@ -109,6 +136,30 @@ MAX_PRECALL_SLEEP_S = 20.0
 # secondary limit tripped in the first place.
 _JITTER_FRACTION = 0.2
 
+# #3536: filename for the proactive-pacing token bucket's state, kept
+# separate from `_STATE_FILENAME` (the reactive backoff) — the two answer
+# different questions (one "is a hit already in force", the other "is this
+# call about to push the rate too high") and have independent schemas, so a
+# corrupt/stale one must never be able to take the other down with it.
+_PACE_STATE_FILENAME = "github_pace.json"
+
+# Steady-state ceiling: one token every 2s, i.e. 30 `gh` calls/minute
+# fleet-wide (today: per-host — see the module docstring). Chosen well under
+# GitHub's own documented secondary-limit guidance (no more than ~1 request/
+# second sustained, a few minutes' cooldown once tripped) so staying inside
+# this budget should mean never tripping it in the first place, while still
+# comfortably covering ordinary poll cadences (#2809's own 15s-60s range).
+PACE_DEFAULT_RATE_PER_S = 0.5
+
+# Burst allowance: a short caller clump (e.g. a drive-queue tick's handful of
+# live re-checks firing in the same second) can spend up to this many tokens
+# at once before the steady-state rate above starts throttling it. Small on
+# purpose — big enough that an ordinary tick's few calls sail through
+# un-delayed, small enough that a dozen-row mass resume (#3536's own
+# incident) still gets smoothed out over several seconds rather than firing
+# all at once.
+PACE_DEFAULT_BURST = 8.0
+
 
 class Backoff(NamedTuple):
     """One recorded rate-limit hit and how long to back off because of it."""
@@ -383,3 +434,151 @@ def local_consult(*, now: float | None = None) -> tuple[float, Backoff | None]:
     """
     now = now if now is not None else time.time()
     return _consult_from(current(now=now), now=now)
+
+
+# ── #3536: proactive pacing (a token bucket, ahead of any 403) ──────────────
+
+
+def _pace_state_path() -> Path:
+    """Resolve the pacing-bucket state file path.
+
+    ``$COORD_GITHUB_PACE_STATE`` overrides first — same seam
+    :func:`_state_path` offers for the backoff file, so a test can redirect
+    this with a one-line ``monkeypatch.setenv`` instead of patching a private
+    function.
+    """
+    override = os.environ.get("COORD_GITHUB_PACE_STATE")
+    if override:
+        return Path(override).expanduser()
+    return default_coord_dir() / _PACE_STATE_FILENAME
+
+
+def _pace_step(
+    state: dict | None, *, now: float, rate_per_s: float, burst: float,
+) -> tuple[float, dict]:
+    """One token-bucket step: ``(sleep_s, new_state)``.
+
+    Pure — no file I/O, no clock read — so the bucket arithmetic itself is
+    cheaply unit-testable independent of :func:`local_pace_acquire`'s
+    read/write wrapper. *state* is the previous ``{"tokens", "last"}``
+    mapping (or ``None``/malformed, treated as a freshly-full bucket — a
+    missing/corrupt file must never itself become a reason to block `gh`
+    traffic, same fail-open posture :func:`_read` takes for the backoff
+    file).
+
+    Refills *tokens* by elapsed wall-clock time at *rate_per_s*, capped at
+    *burst*. A caller that finds at least one token available is granted
+    immediately (``sleep_s == 0.0``) and consumes it. Otherwise the bucket
+    is empty: the caller is charged the wait needed for ONE token to accrue
+    (``deficit / rate_per_s``) rather than being granted for free — the
+    token it is about to "spend" is accounted for up front (``tokens``
+    pinned at ``0.0``, ``last`` advanced to *now*) so a second concurrent
+    caller reading the freshly-written state a moment later sees the same
+    empty bucket, not a stale one that would hand out the same token twice.
+
+    A non-positive *rate_per_s* never grants a wait longer than the deficit
+    itself would imply at a reasonable floor — guarded by the caller
+    (:func:`local_pace_acquire`) clamping *rate_per_s* to a small positive
+    floor before this ever sees it, so a float divide-by-zero here is not a
+    live concern; this function still tolerates it defensively (returns
+    ``0.0`` rather than raising) because the caller's clamp isn't part of
+    this function's own contract.
+    """
+    tokens = burst
+    last = now
+    if isinstance(state, dict):
+        try:
+            tokens = float(state.get("tokens", burst))
+            last = float(state.get("last", now))
+        except (TypeError, ValueError):
+            tokens, last = burst, now
+    elapsed = max(0.0, now - last)
+    tokens = min(burst, tokens + elapsed * rate_per_s)
+    if tokens >= 1.0:
+        return 0.0, {"tokens": tokens - 1.0, "last": now}
+    if rate_per_s <= 0:
+        return 0.0, {"tokens": tokens, "last": now}
+    deficit = 1.0 - tokens
+    wait = deficit / rate_per_s
+    return wait, {"tokens": 0.0, "last": now}
+
+
+def local_pace_acquire(
+    *,
+    now: float | None = None,
+    rate_per_s: float = PACE_DEFAULT_RATE_PER_S,
+    burst: float = PACE_DEFAULT_BURST,
+) -> float:
+    """Take one token from THIS HOST's shared pacing bucket; return the
+    jittered number of seconds the caller should sleep first (#3536).
+
+    Shared across every process on this host via the same small JSON file
+    :func:`local_record`/:func:`local_consult` use for the reactive backoff
+    (a different file — see :data:`_PACE_STATE_FILENAME` — but the identical
+    read/compute/atomic-replace shape), so a drive-queue tick, `coord
+    notify`, and the merge-queue sweep all draw from the SAME bucket instead
+    of each pacing only its own calls. ``0.0`` means "go ahead now" — the
+    overwhelmingly common case outside a burst.
+
+    Best-effort, unconditionally (never raises): a missing/unwritable state
+    dir or a corrupt file degrades to "no pacing" (``0.0``), same posture
+    every other function in this module takes — pacing must never become a
+    new way to break `gh` access, only a way to smooth it.
+
+    Jittered the same way :func:`_jitter` smooths every other wait here, for
+    the identical reason: several callers racing the exact same computed
+    wait would otherwise wake and retry in lockstep, right back into the
+    synchronized-poll pattern this module exists to avoid.
+    """
+    now = now if now is not None else time.time()
+    rate_per_s = rate_per_s if rate_per_s > 0 else PACE_DEFAULT_RATE_PER_S
+    burst = burst if burst > 0 else PACE_DEFAULT_BURST
+    try:
+        path = _pace_state_path()
+        try:
+            raw = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError, ValueError):
+            raw = None
+        sleep_s, new_state = _pace_step(
+            raw if isinstance(raw, dict) else None,
+            now=now, rate_per_s=rate_per_s, burst=burst,
+        )
+        path.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(prefix=".github_pace.", suffix=".tmp", dir=path.parent)
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as fh:
+                json.dump(new_state, fh, sort_keys=True)
+                fh.write("\n")
+            os.replace(tmp, path)
+        except Exception:
+            try:
+                os.unlink(tmp)
+            except OSError:
+                pass
+            raise
+        return _jitter(sleep_s)
+    except Exception as exc:  # noqa: BLE001 -- pacing must never break a caller
+        _log.debug("github_throttle: pace acquire failed: %s", exc)
+        return 0.0
+
+
+def pace(
+    *,
+    now: float | None = None,
+    rate_per_s: float = PACE_DEFAULT_RATE_PER_S,
+    burst: float = PACE_DEFAULT_BURST,
+) -> float:
+    """Proactive pacing for a caller about to issue a `gh` call (#3536).
+
+    Today this is exactly :func:`local_pace_acquire` — single-host only, no
+    daemon branch, because true fleet-wide pacing across machines would need
+    a ``/github-pace``-shaped endpoint the way :func:`record`/:func:`consult`
+    get one for the reactive backoff, which is out of scope for the files
+    this change touches (see the module docstring). Kept as its own public
+    name — rather than every caller importing :func:`local_pace_acquire`
+    directly — so a future daemon-routed version slots in behind this ONE
+    name, the same ``record``/``local_record`` and ``consult``/``local_
+    consult`` split this module already uses, without every call site
+    needing to change again.
+    """
+    return local_pace_acquire(now=now, rate_per_s=rate_per_s, burst=burst)

{code_coordinator-0.5.577 → code_coordinator-0.5.578}/coord/merge_queue.py

@@ -2218,6 +2218,57 @@ def is_ci_pending_reason(reason: str | None) -> bool:
     return (reason or "").startswith(CI_PENDING_PREFIX)
 
 
+# #3536: floor/cap for `ci_pending_poll_interval_s`'s exponential growth.
+# `CI_PENDING_POLL_BASE_S` matches the fastest fixed poll cadence #2809's own
+# incident narrative already named (15s); `CI_PENDING_POLL_CAP_S` keeps a
+# long-pending entry from ever waiting so long that a genuinely-resolved
+# check goes unnoticed for minutes at a stretch.
+CI_PENDING_POLL_BASE_S = 15.0
+CI_PENDING_POLL_CAP_S = 300.0
+
+
+def ci_pending_poll_interval_s(
+    consecutive_pending_polls: int,
+    *,
+    base: float = CI_PENDING_POLL_BASE_S,
+    cap: float = CI_PENDING_POLL_CAP_S,
+) -> float:
+    """How long a caller should wait before re-checking CI again, given how
+    many CONSECUTIVE times in a row it has already observed this entry as
+    ``checks_pending`` (#3536).
+
+    Doubles *base* once per consecutive pending observation, capped at
+    *cap*: ``ci_pending_poll_interval_s(0)`` is *base* itself (the very
+    first pending read — no reason yet to slow down),
+    ``ci_pending_poll_interval_s(1)`` is ``2 * base``, and so on, never
+    exceeding *cap* once the doubling would.
+
+    #2814's fix made every poll re-issue a real, attempt-exempt ``coord
+    merge --only`` re-check for as long as an entry sits CI-pending — the
+    only thing that refreshes a standalone drive's cached ``error`` (see
+    :func:`coord.drive._decide_merge`'s module comment on the four
+    CI-outcome siblings `is_ci_pending_reason` is one of). That is correct
+    for a single entry in isolation, but #3536's incident shows it does not
+    scale: several sibling entries on the same host parked on CI-pending at
+    once all re-check on the SAME fixed interval, serializing through the
+    host-wide merge lock and adding `gh` calls that only grow GitHub's
+    secondary-limit exposure the longer CI genuinely stays pending — exactly
+    when the fleet can least afford more traffic.
+
+    This is the pure interval policy a caller tracking a per-entry pending
+    streak (a persisted counter alongside the existing
+    ``ci_infra_reruns``/``ci_flaky_reruns``/``ci_unreadable_reruns``) can use
+    to space those re-checks out the longer CI stays pending, instead of
+    polling on the same fixed cadence indefinitely. Deliberately pure (no
+    clock read, no persistence, no `gh` call) so the growth policy itself is
+    cheaply unit-testable independent of wherever a caller decides to track
+    the streak and the wall-clock moment it last actually checked.
+    """
+    consecutive_pending_polls = max(0, int(consecutive_pending_polls))
+    interval = base * (2.0 ** consecutive_pending_polls)
+    return min(interval, cap)
+
+
 # #2347: the reason string prefix for a `checks_failed` block whose failing
 # check(s) are ALL the #1525 synthetic "could not read CI status"/"gh too
 # old" stand-ins (`coord.ci_store.is_unreadable_check`) — the check-list