pr-shepherd 0.53.0 → 0.54.0

package/bin/commands/poll-quota.mjs

@@ -1,10 +1,11 @@
-import { evaluateWorktreeGraphqlQuotaWarning } from "../state/graphql-quota-warnings.mjs";
-import { summarizeApiTelemetry, withGraphqlCredentialFingerprint, } from "../github/api-telemetry.mjs";
+import { summarizeApiTelemetry } from "../github/api-telemetry.mjs";
 import { GitHubRequestError } from "../github/errors.mjs";
 import { isRateLimitMessage } from "../comments/rate-limit.mjs";
+import { exhaustedPrimaryLimitDelayMs } from "./poll-rate-limit-delay.mjs";
+import { selectQuotaWarning } from "./quota-selection.mjs";
 const GRAPHQL_RETRY_AFTER_DEFAULT_MS = 60_000;
 /** Sleep at least `--interval`, and at least the active crossed quota band. */
-export function graphqlQuotaPollIntervalMs(bands, usage, fallbackMs, maxMs) {
+function graphqlQuotaPollIntervalMs(bands, usage, fallbackMs, maxMs) {
     if (usage === undefined || usage.limit <= 0 || bands.length === 0) {
         return Math.min(fallbackMs, maxMs);
     }
@@ -15,34 +16,84 @@ export function graphqlQuotaPollIntervalMs(bands, usage, fallbackMs, maxMs) {
     const bandMs = active.pollIntervalMinutes * 60_000;
     return Math.min(Math.max(fallbackMs, bandMs), maxMs);
 }
+/** Slow the poll for whichever of GraphQL or REST core is in a tighter band. */
+export function quotaPollIntervalMs(bands, usage, fallbackMs, maxMs) {
+    const core = usage?.rest?.find((item) => item.resource === "core");
+    return Math.max(graphqlQuotaPollIntervalMs(bands, usage?.graphql, fallbackMs, maxMs), graphqlQuotaPollIntervalMs(bands, core, fallbackMs, maxMs));
+}
 /**
- * Retry delay for `--until-terminal` when GitHub returns a GraphQL 429 / secondary
- * limit. `null` means the error is not a retryable rate limit.
+ * Retry delay for `--until-terminal` when GitHub exhausts a primary quota or
+ * returns a secondary limit. `null` means the error is not a retryable rate limit.
  */
-export function pollGraphQlRetryAfterMs(err) {
+export function pollRateLimitRetryAfterMs(err) {
     if (!(err instanceof GitHubRequestError))
         return null;
-    const retryable = err.status === 429 ||
-        err.retryAfterSeconds !== undefined ||
-        isRateLimitMessage(err.message) ||
-        (err.graphqlErrors?.some((error) => isRateLimitMessage(error.message)) ?? false) ||
-        (err.rateLimit !== undefined && err.rateLimit.remaining <= 0);
+    const rateLimitMessage = isRateLimitMessage(err.message) ||
+        (err.graphqlErrors?.some((error) => isRateLimitMessage(error.message)) ?? false);
+    const exhausted = err.rateLimit !== undefined && err.rateLimit.remaining <= 0;
+    const retryable = err.status === 429 || err.retryAfterSeconds !== undefined || rateLimitMessage || exhausted;
     if (!retryable)
         return null;
-    if (err.retryAfterSeconds !== undefined)
-        return Math.max(err.retryAfterSeconds, 0) * 1000;
-    if (err.rateLimit !== undefined && err.rateLimit.remaining <= 0) {
-        return Math.max(err.rateLimit.resetAt * 1000 - Date.now(), 0);
+    const resource = retryResource(err);
+    const rateLimit = err.rateLimit;
+    const details = {
+        resource,
+        ...(rateLimit?.remaining !== undefined && { remaining: rateLimit.remaining }),
+        ...(rateLimit?.limit !== undefined && { limit: rateLimit.limit }),
+        ...(rateLimit?.resetAt !== undefined && { resetAt: rateLimit.resetAt }),
+    };
+    if (err.retryAfterSeconds !== undefined) {
+        return { ...details, ms: Math.max(err.retryAfterSeconds, 0) * 1000 };
+    }
+    if (exhausted && rateLimit !== undefined) {
+        return { ...details, ms: exhaustedPrimaryLimitDelayMs(rateLimit.resetAt, Date.now()) };
     }
-    return GRAPHQL_RETRY_AFTER_DEFAULT_MS;
+    return { ...details, ms: GRAPHQL_RETRY_AFTER_DEFAULT_MS };
+}
+function retryResource(err) {
+    if (err.rateLimit?.resource)
+        return err.rateLimit.resource;
+    const secondary = err.status === 429 ||
+        err.retryAfterSeconds !== undefined ||
+        /secondary/i.test(err.message) ||
+        (err.graphqlErrors?.some((error) => /secondary/i.test(error.message)) ?? false);
+    return secondary ? "secondary" : "graphql";
+}
+function rateLimitResourceLabel(resource) {
+    if (resource === "graphql")
+        return "GitHub GraphQL rate limit";
+    if (resource === "secondary")
+        return "GitHub secondary rate limit";
+    return `GitHub REST ${resource} rate limit`;
+}
+function rateLimitCounts(retry) {
+    return retry.remaining !== undefined && retry.limit !== undefined
+        ? ` (${retry.remaining}/${retry.limit})`
+        : "";
+}
+function rateLimitClock(retry) {
+    const resetAt = retry.resetAt ?? Math.ceil((Date.now() + retry.ms) / 1000);
+    return `${new Date(resetAt * 1000).toISOString().slice(11, 19)}Z`;
+}
+/** Stderr line naming the exhausted budget and when the sleep ends. */
+export function formatRateLimitRetryLine(tickLabel, elapsedSeconds, retry) {
+    const clock = rateLimitClock(retry);
+    return `[${tickLabel} / +${elapsedSeconds}s] ${rateLimitResourceLabel(retry.resource)}${rateLimitCounts(retry)} — retrying at ${clock} (in ${Math.round(retry.ms / 1000)}s)\n`;
+}
+/** One stderr line before `--until-terminal` gives up and exits 75. */
+export function formatRateLimitGiveUpLine(tickLabel, elapsedSeconds, retry) {
+    const clock = rateLimitClock(retry);
+    return `[${tickLabel} / +${elapsedSeconds}s] ${rateLimitResourceLabel(retry.resource)}${rateLimitCounts(retry)} still exhausted at ${clock} after 5 attempts with no progress\n`;
 }
 export async function aggregateQuotaWarning(result, bands, intervalSeconds) {
-    const usage = summarizeApiTelemetry()?.graphql;
+    const usage = summarizeApiTelemetry();
     const [owner, repo] = result.repo.split("/");
     if (!usage || !owner || !repo)
         return undefined;
-    return evaluateWorktreeGraphqlQuotaWarning({ owner, repo }, bands.map((band) => ({
+    if (!usage.graphql && !usage.rest?.some((item) => item.resource === "core"))
+        return undefined;
+    return selectQuotaWarning({ owner, repo }, bands.map((band) => ({
         ...band,
         pollIntervalMinutes: Math.max(band.pollIntervalMinutes, intervalSeconds / 60),
-    })), withGraphqlCredentialFingerprint(usage), true);
+    })), usage, true);
 }

package/bin/commands/poll-rate-limit-cancel.d.mts

@@ -0,0 +1,35 @@
+import type { IterateResult, MergeStateStatus, MergeableState, PollSummaryResult } from "../types.mts";
+export interface RateLimitPullTarget {
+    owner: string;
+    repo: string;
+    pr: number;
+}
+export interface RateLimitProbePull {
+    pr: number;
+    state: "MERGED" | "CLOSED";
+    title: string;
+    url: string;
+    draft: boolean;
+    mergeable: MergeableState;
+    mergeStateStatus: MergeStateStatus;
+    baseRefName: string;
+    headRefName: string;
+    headRefOid: string;
+}
+export declare function onePrRateLimitTargets(explicit: {
+    owner: string;
+    name: string;
+} | undefined, pr: number | undefined, repoSlug: string | undefined): RateLimitPullTarget[];
+/** Same fields as the merged/closed iterate cancel path, filled from the REST pull. */
+export declare function onePrCancelFromPulls(explicit: {
+    owner: string;
+    name: string;
+} | undefined, repoSlug: string | undefined, pulls: RateLimitProbePull[]): IterateResult | undefined;
+export declare function aggregateRateLimitTargets(last: PollSummaryResult | undefined, repo: {
+    owner: string;
+    name: string;
+} | undefined, prNumbers: number[] | undefined): RateLimitPullTarget[];
+export declare function aggregateCancelFromPulls(last: PollSummaryResult | undefined, repo: {
+    owner: string;
+    name: string;
+} | undefined, pulls: RateLimitProbePull[]): PollSummaryResult | undefined;

package/bin/commands/poll-rate-limit-cancel.mjs

@@ -0,0 +1,101 @@
+export function onePrRateLimitTargets(explicit, pr, repoSlug) {
+    if (pr === undefined)
+        return [];
+    const [slugOwner, slugName] = repoSlug?.split("/") ?? [];
+    const owner = explicit?.owner ?? slugOwner;
+    const repo = explicit?.name ?? slugName;
+    if (!owner || !repo)
+        return [];
+    return [{ owner, repo, pr }];
+}
+/** Same fields as the merged/closed iterate cancel path, filled from the REST pull. */
+export function onePrCancelFromPulls(explicit, repoSlug, pulls) {
+    const pull = pulls[0];
+    const slug = explicit ? `${explicit.owner}/${explicit.name}` : repoSlug;
+    if (!pull || !slug)
+        return undefined;
+    const state = pull.state;
+    return {
+        pr: pull.pr,
+        repo: slug,
+        status: state,
+        state,
+        mergeStateStatus: pull.mergeStateStatus,
+        mergeStatus: "UNKNOWN",
+        reviewDecision: null,
+        blockingBotReviewInProgress: false,
+        isDraft: pull.draft,
+        shouldCancel: true,
+        remainingSeconds: 0,
+        summary: { passing: 0, skipped: 0, filtered: 0, inProgress: 0, superseded: 0 },
+        baseBranch: pull.baseRefName,
+        branchProtection: null,
+        checks: [],
+        inProgressChecks: [],
+        action: "cancel",
+        reason: state === "MERGED" ? "merged" : "closed",
+        log: `CANCEL: PR #${pull.pr} is ${state.toLowerCase()} — stopping`,
+    };
+}
+export function aggregateRateLimitTargets(last, repo, prNumbers) {
+    if (last) {
+        return last.prs.flatMap((item) => {
+            if (item.state !== "OPEN")
+                return [];
+            const [owner, name] = item.repo.split("/");
+            if (!owner || !name)
+                return [];
+            return [{ owner, repo: name, pr: item.pr }];
+        });
+    }
+    if (!repo || !prNumbers?.length)
+        return [];
+    return prNumbers.map((pr) => ({ owner: repo.owner, repo: repo.name, pr }));
+}
+export function aggregateCancelFromPulls(last, repo, pulls) {
+    if (last) {
+        const byPr = new Map(pulls.map((pull) => [pull.pr, pull]));
+        const prs = last.prs.map((item) => applyProbe(item, byPr.get(item.pr)));
+        if (prs.some((item) => item.state !== "MERGED" && item.state !== "CLOSED"))
+            return undefined;
+        return { ...last, prs, reason: "all_terminal" };
+    }
+    if (!repo)
+        return undefined;
+    const slug = `${repo.owner}/${repo.name}`;
+    const prs = pulls.map((pull) => syntheticItem(slug, pull));
+    return {
+        mode: "summary",
+        repo: slug,
+        selection: { kind: "prs", requested: pulls.map((pull) => pull.pr) },
+        reason: "all_terminal",
+        prs,
+    };
+}
+function applyProbe(item, pull) {
+    if (!pull)
+        return item;
+    return {
+        ...item,
+        action: "cancel",
+        state: pull.state,
+        reasons: [pull.state === "MERGED" ? "merged" : "closed"],
+    };
+}
+function syntheticItem(repo, pull) {
+    return {
+        pr: pull.pr,
+        repo,
+        title: pull.title || `PR ${pull.pr}`,
+        url: pull.url || `https://github.com/${repo}/pull/${pull.pr}`,
+        action: "cancel",
+        reasons: [pull.state === "MERGED" ? "merged" : "closed"],
+        state: pull.state,
+        mergeable: pull.mergeable,
+        mergeStateStatus: pull.mergeStateStatus,
+        headRefName: pull.headRefName,
+        headRefOid: pull.headRefOid,
+        baseRefName: pull.baseRefName,
+        ...(pull.draft ? { isDraft: true } : {}),
+    };
+}

package/bin/commands/poll-rate-limit-delay.d.mts

@@ -0,0 +1 @@
+export declare function exhaustedPrimaryLimitDelayMs(resetAtSeconds: number, nowMs: number): number;

package/bin/commands/poll-rate-limit-delay.mjs

@@ -0,0 +1,7 @@
+/** Exhausted primary limit: time until resetAt, plus 5s, plus `resetAt % 5` seconds. */
+const EXHAUSTED_LIMIT_MARGIN_MS = 5_000;
+export function exhaustedPrimaryLimitDelayMs(resetAtSeconds, nowMs) {
+    return (Math.max(resetAtSeconds * 1000 - nowMs, 0) +
+        EXHAUSTED_LIMIT_MARGIN_MS +
+        (resetAtSeconds % 5) * 1000);
+}

package/bin/commands/poll-rate-limit-wait.d.mts

@@ -0,0 +1,14 @@
+import type { RateLimitProbePull, RateLimitPullTarget } from "./poll-rate-limit-cancel.mts";
+interface WaitOptions<T> {
+    untilTerminal: boolean;
+    intervalMs: number;
+    tickLabel: string;
+    startedAt: number;
+    targets: RateLimitPullTarget[];
+    onAllTerminal?: (pulls: RateLimitProbePull[]) => T | undefined;
+}
+export declare function createUntilTerminalRateLimitRetry(): {
+    reset(): void;
+    wait<T>(err: unknown, opts: WaitOptions<T>): Promise<T | undefined>;
+};
+export {};

package/bin/commands/poll-rate-limit-wait.mjs

@@ -0,0 +1,125 @@
+import { rest } from "../github/rest-http.mjs";
+import { sleep } from "../util/sleep.mjs";
+import { formatRateLimitGiveUpLine, formatRateLimitRetryLine, pollRateLimitRetryAfterMs, } from "./poll-quota.mjs";
+const MAX_NO_PROGRESS_ATTEMPTS = 5;
+const NO_PROGRESS_BACKOFF_MS = [15_000, 30_000, 60_000];
+const MERGE_STATES = new Set([
+    "BEHIND",
+    "BLOCKED",
+    "CLEAN",
+    "DIRTY",
+    "DRAFT",
+    "HAS_HOOKS",
+    "UNKNOWN",
+    "UNSTABLE",
+]);
+export function createUntilTerminalRateLimitRetry() {
+    const state = { armed: false, noProgress: 0 };
+    return {
+        reset() {
+            state.armed = false;
+            state.noProgress = 0;
+            state.lastResetAt = undefined;
+        },
+        wait(err, opts) {
+            return waitForRateLimit(state, err, opts);
+        },
+    };
+}
+function backoffMs(attempt) {
+    return NO_PROGRESS_BACKOFF_MS[Math.min(attempt, NO_PROGRESS_BACKOFF_MS.length) - 1] ?? 60_000;
+}
+async function waitForRateLimit(state, err, opts) {
+    const retry = opts.untilTerminal ? pollRateLimitRetryAfterMs(err) : null;
+    if (!retry)
+        throw err;
+    const elapsed = Math.round((Date.now() - opts.startedAt) / 1000);
+    const progressed = state.armed &&
+        retry.resetAt !== undefined &&
+        (state.lastResetAt === undefined || retry.resetAt > state.lastResetAt);
+    let sleepMs = retry.ms;
+    if (!state.armed || progressed) {
+        state.armed = true;
+        state.noProgress = 0;
+        if (retry.resetAt !== undefined)
+            state.lastResetAt = retry.resetAt;
+    }
+    else {
+        state.noProgress += 1;
+        if (state.noProgress >= MAX_NO_PROGRESS_ATTEMPTS) {
+            process.stderr.write(formatRateLimitGiveUpLine(opts.tickLabel, elapsed, retry));
+            throw err;
+        }
+        // Never sleep less than GitHub asked for. A short no-progress backoff must
+        // not undercut an explicit Retry-After or the secondary-limit default.
+        sleepMs = Math.max(retry.ms, backoffMs(state.noProgress));
+    }
+    process.stderr.write(formatRateLimitRetryLine(opts.tickLabel, elapsed, { ...retry, ms: sleepMs }));
+    return sleepRateLimit(sleepMs, opts, retry.resource);
+}
+async function sleepRateLimit(ms, opts, resource) {
+    const canProbe = resource !== "core" && opts.targets.length > 0;
+    if (ms <= opts.intervalMs || opts.intervalMs <= 0) {
+        await sleep(ms);
+        return undefined;
+    }
+    let remaining = ms;
+    let probesDisabled = !canProbe;
+    while (remaining > 0) {
+        const chunk = Math.min(opts.intervalMs, remaining);
+        await sleep(chunk);
+        remaining -= chunk;
+        if (remaining <= 0 || probesDisabled)
+            continue;
+        try {
+            const found = await probeOpenPulls(opts.targets, opts.onAllTerminal);
+            if (found !== undefined)
+                return found;
+        }
+        catch {
+            // The probe only notices a merge or close early. A 5xx, a 404, or a
+            // secondary limit must not abort the rate-limit sleep.
+            probesDisabled = true;
+        }
+    }
+    return undefined;
+}
+async function probeOpenPulls(targets, onAllTerminal) {
+    const pulls = [];
+    for (const target of targets) {
+        // GraphQL is exhausted for this sleep. REST core is a separate budget, so
+        // GET /repos/{owner}/{repo}/pulls/{n} can still see a merge or close.
+        const data = await rest("GET", `/repos/${target.owner}/${target.repo}/pulls/${target.pr}`);
+        const state = terminalState(data);
+        if (!state)
+            return undefined;
+        pulls.push(toProbePull(target.pr, state, data));
+    }
+    return onAllTerminal?.(pulls);
+}
+function terminalState(pull) {
+    if (!pull)
+        return undefined;
+    if (pull.merged === true || pull.merged_at != null)
+        return "MERGED";
+    if (pull.state?.toLowerCase() === "closed")
+        return "CLOSED";
+    return undefined;
+}
+function toProbePull(pr, state, pull) {
+    const mergeState = pull.mergeable_state?.toUpperCase();
+    return {
+        pr,
+        state,
+        title: pull.title ?? "",
+        url: pull.html_url ?? "",
+        draft: pull.draft === true,
+        mergeable: pull.mergeable === true ? "MERGEABLE" : pull.mergeable === false ? "CONFLICTING" : "UNKNOWN",
+        mergeStateStatus: MERGE_STATES.has(mergeState ?? "")
+            ? mergeState
+            : "UNKNOWN",
+        baseRefName: pull.base?.ref ?? "",
+        headRefName: pull.head?.ref ?? "",
+        headRefOid: pull.head?.sha ?? "",
+    };
+}

package/bin/commands/poll-summary.mjs

@@ -5,7 +5,9 @@ import { getRepoInfo } from "../github/client.mjs";
 import { withApiTelemetryScope, summarizeApiTelemetry } from "../github/api-telemetry.mjs";
 import { fetchPollSummary } from "../github/poll-summary.mjs";
 import { sleep } from "../util/sleep.mjs";
-import { aggregateQuotaWarning, graphqlQuotaPollIntervalMs, pollGraphQlRetryAfterMs, } from "./poll-quota.mjs";
+import { aggregateQuotaWarning, quotaPollIntervalMs } from "./poll-quota.mjs";
+import { aggregateCancelFromPulls, aggregateRateLimitTargets } from "./poll-rate-limit-cancel.mjs";
+import { createUntilTerminalRateLimitRetry } from "./poll-rate-limit-wait.mjs";
 import { planPollSummary, withPollSummaryInstructions } from "./poll-summary-instructions.mjs";
 import { summaryStatusSignature } from "./poll-summary-signature.mjs";
 import { applyStackStallGuard } from "./stack-stall.mjs";
@@ -42,13 +44,13 @@ async function runAggregatePollCore(opts) {
     let debounceUntil = null;
     let last;
     let lastStatusSignature = null;
-    let rateLimitRetries = 0;
+    const rateLimitRetry = createUntilTerminalRateLimitRetry();
     let pendingQuotaWarning;
     while (true) {
         tick += 1;
         try {
             last = await runPollSummaryCore(opts);
-            rateLimitRetries = 0;
+            rateLimitRetry.reset();
         }
         catch (error) {
             if (last?.selection.kind === "stack" && isMissingStack(error)) {
@@ -83,12 +85,17 @@ async function runAggregatePollCore(opts) {
                     ...(pendingQuotaWarning && { quotaWarning: pendingQuotaWarning }),
                 }, opts.merge);
             }
-            const retryMs = opts.untilTerminal ? pollGraphQlRetryAfterMs(error) : null;
-            if (retryMs === null || rateLimitRetries >= 1)
-                throw error;
-            rateLimitRetries += 1;
-            process.stderr.write(`[aggregate poll tick ${tick} / +${Math.round((Date.now() - start) / 1000)}s] GraphQL rate limit — retrying in ${Math.round(retryMs / 1000)}s\n`);
-            await sleep(retryMs);
+            const early = await rateLimitRetry.wait(error, {
+                untilTerminal: opts.untilTerminal === true,
+                intervalMs,
+                tickLabel: `aggregate poll tick ${tick}`,
+                startedAt: start,
+                targets: aggregateRateLimitTargets(last, opts.targetRepository, opts.prNumbers),
+                onAllTerminal: (pulls) => aggregateCancelFromPulls(last, opts.targetRepository, pulls),
+            });
+            if (early) {
+                return presentProbedCancel(pendingQuotaWarning ? { ...early, quotaWarning: pendingQuotaWarning } : early, opts.merge);
+            }
             continue;
         }
         const allTerminal = last.selection.kind === "stack"
@@ -142,7 +149,7 @@ async function runAggregatePollCore(opts) {
         const elapsedMs = Date.now() - start;
         const sleepMs = debounceUntil
             ? Math.min(intervalMs, Math.max(debounceUntil - Date.now(), 0))
-            : graphqlQuotaPollIntervalMs(quotaBands, summarizeApiTelemetry()?.graphql, intervalMs, MAX_TIMER_MS);
+            : quotaPollIntervalMs(quotaBands, summarizeApiTelemetry(), intervalMs, MAX_TIMER_MS);
         if (!opts.untilTerminal && debounceUntil === null) {
             const remainingMs = timeoutMs - elapsedMs;
             if (remainingMs <= 0 || remainingMs + TIMER_DRIFT_TOLERANCE_MS < sleepMs) {
@@ -159,6 +166,11 @@ async function runAggregatePollCore(opts) {
         await sleep(sleepMs);
     }
 }
+function presentProbedCancel(result, merge) {
+    // The stack planner already cancels an all-merged stack and escalates a
+    // closed-unmerged layer. Do not rewrite that into a successful stop.
+    return attachUsage({ ...result, reason: "all_terminal" }, merge);
+}
 function isMissingStack(error) {
     return (error instanceof ShepherdError && error.message.includes("not part of a native GitHub stack"));
 }

package/bin/commands/poll.mjs

@@ -2,7 +2,9 @@ import { runIterate } from "./iterate/index.mjs";
 import { sleep } from "../util/sleep.mjs";
 import { withPollApiUsage } from "./poll-run.mjs";
 import { loadConfig } from "../config/load.mjs";
-import { graphqlQuotaPollIntervalMs, pollGraphQlRetryAfterMs } from "./poll-quota.mjs";
+import { quotaPollIntervalMs } from "./poll-quota.mjs";
+import { onePrCancelFromPulls, onePrRateLimitTargets } from "./poll-rate-limit-cancel.mjs";
+import { createUntilTerminalRateLimitRetry } from "./poll-rate-limit-wait.mjs";
 import { writeDebounceProgress, writeWaitProgress } from "./poll-progress.mjs";
 const DEFAULT_POLL_DEBOUNCE_SECONDS = 60;
 const MAX_TIMER_MS = 2 ** 31 - 1;
@@ -29,7 +31,7 @@ async function runPollCore(opts) {
     // Pin the PR resolved by the first tick; branch inference only matches OPEN PRs.
     let prNumber = opts.prNumber;
     let debounceUntil = null;
-    let rateLimitRetries = 0;
+    const rateLimitRetry = createUntilTerminalRateLimitRetry();
     while (true) {
         tick += 1;
         const pastDebounce = debounceUntil !== null && Date.now() >= debounceUntil;
@@ -48,21 +50,24 @@ async function runPollCore(opts) {
             quotaWarningMinimumPollIntervalMinutes: intervalSeconds / 60,
         });
         const runTick = async (fingerprintCache) => {
-            try {
-                const result = await iterateTick(fingerprintCache);
-                rateLimitRetries = 0;
-                return result;
-            }
-            catch (err) {
-                const retryMs = untilTerminal ? pollGraphQlRetryAfterMs(err) : null;
-                if (retryMs === null || rateLimitRetries >= 1)
-                    throw err;
-                rateLimitRetries += 1;
-                process.stderr.write(`[poll tick ${tick} / +${Math.round((Date.now() - start) / 1000)}s] GraphQL rate limit — retrying in ${Math.round(retryMs / 1000)}s\n`);
-                await sleep(retryMs);
-                const result = await iterateTick(fingerprintCache);
-                rateLimitRetries = 0;
-                return result;
+            while (true) {
+                try {
+                    const result = await iterateTick(fingerprintCache);
+                    rateLimitRetry.reset();
+                    return result;
+                }
+                catch (err) {
+                    const early = await rateLimitRetry.wait(err, {
+                        untilTerminal,
+                        intervalMs,
+                        tickLabel: `poll tick ${tick}`,
+                        startedAt: start,
+                        targets: onePrRateLimitTargets(iterateOpts.targetRepository, prNumber, lastResult?.repo),
+                        onAllTerminal: (pulls) => onePrCancelFromPulls(iterateOpts.targetRepository, lastResult?.repo, pulls),
+                    });
+                    if (early)
+                        return early;
+                }
             }
         };
         lastResult = await runTick(allowCache);
@@ -101,7 +106,7 @@ async function runPollCore(opts) {
             if (pendingQuotaWarning === undefined)
                 debounceUntil = null;
             const elapsedMs = Date.now() - start;
-            const sleepMs = graphqlQuotaPollIntervalMs(quotaBands, lastResult.apiUsage?.graphql, intervalMs, MAX_TIMER_MS);
+            const sleepMs = quotaPollIntervalMs(quotaBands, lastResult.apiUsage, intervalMs, MAX_TIMER_MS);
             if (!untilTerminal) {
                 const remainingMs = timeoutMs - elapsedMs;
                 if (remainingMs <= 0 || remainingMs + TIMER_DRIFT_TOLERANCE_MS < sleepMs) {
@@ -126,7 +131,7 @@ async function runPollCore(opts) {
             !pastDebounce) {
             debounceUntil = null;
             const elapsedMs = Date.now() - start;
-            const sleepMs = graphqlQuotaPollIntervalMs(quotaBands, lastResult.apiUsage?.graphql, intervalMs, MAX_TIMER_MS);
+            const sleepMs = quotaPollIntervalMs(quotaBands, lastResult.apiUsage, intervalMs, MAX_TIMER_MS);
             if (!untilTerminal) {
                 const remainingMs = timeoutMs - elapsedMs;
                 if (remainingMs <= 0 || remainingMs + TIMER_DRIFT_TOLERANCE_MS < sleepMs) {

package/bin/commands/quota-selection.d.mts

@@ -0,0 +1,7 @@
+import type { GraphqlQuotaWarningBand } from "../config/load.mts";
+import type { ApiUsage, GraphqlQuotaWarning } from "../types.mts";
+/** Claim GraphQL and REST core warnings separately, then present one result. */
+export declare function selectQuotaWarning(key: {
+    owner: string;
+    repo: string;
+}, bands: GraphqlQuotaWarningBand[], usage: ApiUsage, persist: boolean): Promise<GraphqlQuotaWarning | undefined>;

package/bin/commands/quota-selection.mjs

@@ -0,0 +1,20 @@
+import { withGraphqlCredentialFingerprint } from "../github/api-telemetry.mjs";
+import { composeQuotaWarning } from "../quota-budgets.mjs";
+import { evaluateWorktreeGraphqlQuotaWarning } from "../state/graphql-quota-warnings.mjs";
+/** Claim GraphQL and REST core warnings separately, then present one result. */
+export async function selectQuotaWarning(key, bands, usage, persist) {
+    const core = usage.rest?.find((item) => item.resource === "core");
+    const graphqlWarning = usage.graphql
+        ? await evaluateWorktreeGraphqlQuotaWarning(key, bands, withGraphqlCredentialFingerprint(usage.graphql), persist)
+        : undefined;
+    const coreWarning = core
+        ? await evaluateWorktreeGraphqlQuotaWarning(key, bands, withGraphqlCredentialFingerprint(core), persist)
+        : undefined;
+    return composeQuotaWarning({
+        graphqlWarning,
+        coreWarning,
+        graphql: usage.graphql,
+        core,
+        bands,
+    });
+}

package/bin/config/load.d.mts

@@ -8,6 +8,8 @@ export interface GraphqlQuotaWarningBand {
 }
 interface PollConfig {
     intervalSeconds: number;
+    /** Multiplier for aggregate polls when `--interval` is omitted. Finite and at least 1. */
+    stackIntervalFactor: number;
     timeoutSeconds: number;
     debounceSeconds: number;
     quietStatus: boolean;

package/bin/config/load.mjs

@@ -153,11 +153,27 @@ function parsePollConfig(value) {
     const intervalSeconds = parsePollDuration(record["intervalSeconds"], "intervalSeconds");
     const timeoutSeconds = parsePollDuration(record["timeoutSeconds"], "timeoutSeconds");
     const debounceSeconds = parsePollDuration(record["debounceSeconds"], "debounceSeconds", true);
+    const stackIntervalFactor = parseStackIntervalFactor(record["stackIntervalFactor"]);
+    assertAggregateIntervalFits(intervalSeconds, stackIntervalFactor);
     const quietStatus = record["quietStatus"];
     if (typeof quietStatus !== "boolean") {
         throw new Error(`Invalid config: poll.quietStatus must be a boolean, got ${JSON.stringify(quietStatus)}`);
     }
-    return { intervalSeconds, timeoutSeconds, debounceSeconds, quietStatus };
+    return { intervalSeconds, stackIntervalFactor, timeoutSeconds, debounceSeconds, quietStatus };
+}
+function parseStackIntervalFactor(value) {
+    if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
+        throw new Error(`Invalid config: poll.stackIntervalFactor must be a finite number greater than or equal to 1, got ${JSON.stringify(value)}`);
+    }
+    return value;
+}
+/** Largest `setTimeout` delay. A bigger aggregate sleep would clamp to ~24.8 days. */
+const MAX_POLL_INTERVAL_MS = 2 ** 31 - 1;
+function assertAggregateIntervalFits(intervalSeconds, factor) {
+    const ms = intervalSeconds * factor * 1000;
+    if (!Number.isFinite(ms) || ms > MAX_POLL_INTERVAL_MS) {
+        throw new Error(`Invalid config: poll.intervalSeconds * poll.stackIntervalFactor must be finite and at most ${MAX_POLL_INTERVAL_MS} milliseconds, got ${JSON.stringify(ms)}`);
+    }
 }
 function parsePollDuration(value, key, allowZero = false) {
     if (typeof value !== "number" ||
@@ -252,7 +268,13 @@ const KNOWN_NESTED_KEYS = {
         "behindBaseHint",
         "resolveOtherHumanThreads",
     ]),
-    poll: new Set(["intervalSeconds", "timeoutSeconds", "debounceSeconds", "quietStatus"]),
+    poll: new Set([
+        "intervalSeconds",
+        "stackIntervalFactor",
+        "timeoutSeconds",
+        "debounceSeconds",
+        "quietStatus",
+    ]),
     watch: new Set(["readyDelayMinutes", "graphqlQuotaWarnings"]),
     resolve: new Set(["shaPoll"]),
     checks: new Set(["ciTriggerEvents", "ignoreLogLines"]),

package/bin/config.json

@@ -25,6 +25,7 @@
   },
   "poll": {
     "intervalSeconds": 60,
+    "stackIntervalFactor": 2,
     "timeoutSeconds": 270,
     "debounceSeconds": 60,
     "quietStatus": false

package/bin/github/check-annotation-cache.d.mts

@@ -0,0 +1,8 @@
+import { type StateKey } from "../state/rest-cache.mts";
+import type { CheckAnnotation } from "../types.mts";
+export interface AnnotationCacheOptions {
+    stateKey: StateKey;
+    headSha?: string;
+}
+export declare function readFreshCheckAnnotations(checkRunId: string, cacheOpts: AnnotationCacheOptions | undefined): Promise<CheckAnnotation[] | undefined>;
+export declare function storeCheckAnnotations(checkRunId: string, annotations: CheckAnnotation[], cacheOpts: AnnotationCacheOptions | undefined): Promise<void>;