codexLB-up 1.25.0b10__py3-none-any.whl

app/core/balancer/logic.py

@@ -0,0 +1,1612 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+import math
+import random
+import socket
+import time
+from dataclasses import dataclass
+from typing import Collection, Iterable, Literal
+
+from app.core.balancer.types import FailureClass, UpstreamError
+from app.core.usage import PLAN_CAPACITY_CREDITS_SECONDARY
+from app.core.utils.retry import backoff_seconds, parse_retry_after
+from app.db.models import AccountStatus
+
+PERMANENT_FAILURE_CODES = {
+    "refresh_token_expired": "Refresh token expired - re-login required",
+    "refresh_token_reused": "Refresh token was reused - re-login required",
+    "refresh_token_invalidated": "Refresh token was revoked - re-login required",
+    "invalid_grant": "Refresh token grant invalid - re-login required",
+    "token_invalidated": "Authentication token invalidated - re-login required",
+    # ``token_expired`` from the OAuth refresh endpoint means the refresh
+    # request itself failed because the refresh token (or the session it
+    # belonged to) is no longer usable -- access-token-only expiry would have
+    # returned a fresh token pair instead. Treat it as a permanent refresh
+    # failure while ordinary requests may continue with the stored access token.
+    "token_expired": "Authentication token expired - re-login required",
+    "app_session_terminated": "ChatGPT session ended - re-login required",
+    "account_session_expired": "ChatGPT session ended - re-login required",
+    "account_auth_invalidated": "Authentication failed after token refresh - re-login required",
+    # The OAuth endpoint uses this code for an invalid/revoked refresh token.
+    # Keep it in the permanent set so the account is surfaced for re-auth
+    # instead of remaining active while every request retries the dead token.
+    "invalid_refresh_token": "Refresh token invalid - re-login required",
+    "account_deactivated": "Account has been deactivated",
+    "deactivated_workspace": "Workspace has been deactivated",
+    "account_suspended": "Account has been suspended",
+    "account_deleted": "Account has been deleted",
+}
+
+REAUTH_REQUIRED_FAILURE_CODES = frozenset(
+    {
+        "refresh_token_expired",
+        "refresh_token_reused",
+        "refresh_token_invalidated",
+        "invalid_grant",
+        "token_invalidated",
+        "token_expired",
+        "app_session_terminated",
+        "account_session_expired",
+        "account_auth_invalidated",
+        "invalid_refresh_token",
+    }
+)
+
+SECONDS_PER_DAY = 60 * 60 * 24
+SECONDS_PER_HOUR = 60 * 60
+SECONDS_PER_WEEK = 7 * SECONDS_PER_DAY
+RATE_LIMIT_RESET_MAX_HORIZON_SECONDS = 366 * SECONDS_PER_DAY
+RATE_LIMIT_RESET_ROUNDING_TOLERANCE_SECONDS = 1.0
+UNKNOWN_RESET_BUCKET_DAYS = 10_000
+UNKNOWN_RESET_FALLBACK_SECONDS = 7 * SECONDS_PER_DAY
+RELATIVE_AVAILABILITY_MIN_DIVISOR_SECONDS = 5 * 60
+RELATIVE_AVAILABILITY_MIN_WEIGHT_FRACTION = 0.1
+DEFAULT_RELATIVE_AVAILABILITY_POWER = 2.0
+DEFAULT_RELATIVE_AVAILABILITY_TOP_K = 5
+RoutingStrategy = Literal[
+    "usage_weighted",
+    "round_robin",
+    "capacity_weighted",
+    "relative_availability",
+    "fill_first",
+    "sequential_drain",
+    "reset_drain",
+    "single_account",
+]
+TrafficClass = Literal["foreground", "opportunistic"]
+UsageWeightedOrder = Literal["secondary_first", "primary_first"]
+ResetPreferenceWindow = Literal["primary", "secondary"]
+UNKNOWN_PLAN_FALLBACK = "free"
+CAPACITY_PLAN_ALIASES = {
+    "education": "edu",
+    "k12": "edu",
+    "guest": "free",
+    "go": "free",
+    "free_workspace": "free",
+    "quorum": "free",
+    "unknown": "free",
+}
+
+HEALTH_TIER_HEALTHY = 0
+HEALTH_TIER_DRAINING = 1
+HEALTH_TIER_PROBING = 2
+
+DRAIN_PRIMARY_THRESHOLD_PCT = 85.0
+DRAIN_SECONDARY_THRESHOLD_PCT = 90.0
+DRAIN_ERROR_WINDOW_SECONDS = 60.0
+DRAIN_ERROR_COUNT_THRESHOLD = 2
+ERROR_BACKOFF_THRESHOLD = 3
+PROBE_QUIET_SECONDS = 60.0
+PROBE_SUCCESS_STREAK_REQUIRED = 3
+ROUTING_POLICY_NORMAL = "normal"
+ROUTING_POLICY_BURN_FIRST = "burn_first"
+ROUTING_POLICY_PRESERVE = "preserve"
+TRAFFIC_CLASS_FOREGROUND = "foreground"
+TRAFFIC_CLASS_OPPORTUNISTIC = "opportunistic"
+PRESERVE_MIN_WEEKLY_FLOOR_PCT = 5.0
+PRESERVE_MIN_SHORT_WINDOW_FLOOR_PCT = 10.0
+NORMAL_LAST_ACCOUNT_EMERGENCY_FLOOR_PCT = 5.0
+RECENT_FOREGROUND_ACTIVITY_SECONDS = 30 * 60
+
+logger = logging.getLogger(__name__)
+
+_RELATIVE_AVAILABILITY_LOG_PREFIX_CANDIDATE = "Relative availability candidate "
+_RELATIVE_AVAILABILITY_LOG_PREFIX_TOP_K = "Relative availability top-k     "
+_RELATIVE_AVAILABILITY_LOG_PREFIX_WINNER = "Relative availability winner    "
+
+
+@dataclass
+class AccountState:
+    account_id: str
+    status: AccountStatus
+    used_percent: float | None = None
+    reset_at: float | None = None
+    primary_reset_at: int | None = None
+    primary_window_minutes: int | None = None
+    blocked_at: float | None = None
+    cooldown_until: float | None = None
+    secondary_used_percent: float | None = None
+    secondary_reset_at: int | None = None
+    last_error_at: float | None = None
+    last_selected_at: float | None = None
+    error_count: int = 0
+    deactivation_reason: str | None = None
+    plan_type: str | None = None
+    capacity_credits: float | None = None
+    health_tier: int = 0
+    priority_used_percent: float | None = None
+    priority_secondary_used_percent: float | None = None
+    priority_reset_at: int | None = None
+    priority_capacity_credits: float | None = None
+    limit_scoped_usage: bool = False
+    access_token_expires_at: float | None = None
+    inflight_response_creates: int = 0
+    inflight_streams: int = 0
+    leased_tokens: float = 0.0
+    routing_policy: str = ROUTING_POLICY_NORMAL
+    ignore_standard_quota: bool = False
+    # Multiplier applied to this candidate's draw weight by the weighted
+    # strategies (``capacity_weighted``, ``relative_availability``); ``1.0`` is
+    # neutral. The balancer derives it from the account's recent upstream
+    # error rate. Deterministic strategies ignore it.
+    selection_weight_multiplier: float = 1.0
+
+
+@dataclass
+class SelectionResult:
+    account: AccountState | None
+    error_message: str | None
+    error_code: str | None = None
+    resets_at: int | None = None
+
+
+USAGE_LIMIT_REACHED = "usage_limit_reached"
+
+
+def pool_usage_exhaustion(
+    states: Iterable[AccountState],
+    *,
+    current: float,
+    ignore_standard_quota: bool = False,
+    ignore_standard_quota_account_ids: Collection[str] | None = None,
+) -> SelectionResult | None:
+    """Describe pool-wide subscription exhaustion without parsing retry text."""
+    if ignore_standard_quota:
+        return None
+    ignored_account_ids = set(ignore_standard_quota_account_ids or ())
+
+    def _primary_usage_evidence(state: AccountState) -> float | None:
+        return state.priority_used_percent if state.priority_used_percent is not None else state.used_percent
+
+    def _secondary_usage_evidence(state: AccountState) -> float | None:
+        if state.limit_scoped_usage and state.priority_secondary_used_percent is None:
+            return _primary_usage_evidence(state)
+        return (
+            state.priority_secondary_used_percent
+            if state.priority_secondary_used_percent is not None
+            else state.secondary_used_percent
+        )
+
+    def _usage_exhausted(state: AccountState) -> bool:
+        if state.status not in (AccountStatus.QUOTA_EXCEEDED, AccountStatus.RATE_LIMITED):
+            return False
+        usage_values = (_primary_usage_evidence(state), _secondary_usage_evidence(state))
+        return any(value is not None and float(value) >= 100.0 for value in usage_values)
+
+    def _usage_exhausted_reset_at(state: AccountState) -> float | None:
+        if state.status not in (AccountStatus.QUOTA_EXCEEDED, AccountStatus.RATE_LIMITED):
+            return None
+        candidates: list[float] = []
+        primary_evidence = _primary_usage_evidence(state)
+        secondary_evidence = _secondary_usage_evidence(state)
+        if primary_evidence is not None and float(primary_evidence) >= 100.0 and state.primary_reset_at is not None:
+            candidates.append(float(state.primary_reset_at))
+        if (
+            secondary_evidence is not None
+            and float(secondary_evidence) >= 100.0
+            and state.secondary_reset_at is not None
+        ):
+            candidates.append(float(state.secondary_reset_at))
+        if not candidates:
+            return None
+        return max(candidates)
+
+    eligible = [
+        state
+        for state in states
+        if not state.ignore_standard_quota
+        and state.account_id not in ignored_account_ids
+        and state.status
+        not in (
+            AccountStatus.PAUSED,
+            AccountStatus.DEACTIVATED,
+        )
+    ]
+    if not eligible or any(not _usage_exhausted(state) for state in eligible):
+        return None
+
+    # Only usage-proven exhausted accounts reach this branch; surface the
+    # earliest reset for an actually exhausted window so the structured 429
+    # does not retry a secondary-window exhaustion at the primary reset time.
+    reset_candidates = [reset_at for state in eligible if (reset_at := _usage_exhausted_reset_at(state)) is not None]
+    resets_at = int(min(reset_candidates)) if reset_candidates else None
+    message = "Usage limit reached"
+    if resets_at is not None:
+        message = _format_retry_hint(max(0.0, resets_at - current))
+    return SelectionResult(
+        account=None,
+        error_message=message,
+        error_code=USAGE_LIMIT_REACHED,
+        resets_at=resets_at,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class RoutingCost:
+    """Request-scoped planner cost applied after hard eligibility filters."""
+
+    total: float = 0.0
+    reason: str | None = None
+
+
+RoutingCostsByAccount = dict[str, RoutingCost]
+
+
+def _planner_cost(state: AccountState, routing_costs: RoutingCostsByAccount | None) -> float:
+    if not routing_costs:
+        return 0.0
+    cost = routing_costs.get(state.account_id)
+    if cost is None:
+        return 0.0
+    return float(cost.total)
+
+
+def _usage_sort_key(state: AccountState) -> tuple[float, float, float, str]:
+    primary_used = _priority_primary_used(state)
+    secondary_used = _priority_secondary_used(state, primary_used)
+    last_selected = state.last_selected_at or 0.0
+    return secondary_used, primary_used, last_selected, state.account_id
+
+
+def _primary_usage_sort_key(state: AccountState) -> tuple[float, float, float, str]:
+    primary_used = _priority_primary_used(state)
+    secondary_used = _priority_secondary_used(state, primary_used)
+    last_selected = state.last_selected_at or 0.0
+    return primary_used, secondary_used, last_selected, state.account_id
+
+
+def _routing_policy(state: AccountState) -> str:
+    if state.routing_policy in {
+        ROUTING_POLICY_BURN_FIRST,
+        ROUTING_POLICY_NORMAL,
+        ROUTING_POLICY_PRESERVE,
+    }:
+        return state.routing_policy
+    return ROUTING_POLICY_NORMAL
+
+
+def _used_pct(state: AccountState, *, secondary: bool) -> float | None:
+    if secondary:
+        return state.secondary_used_percent if state.secondary_used_percent is not None else state.used_percent
+    return state.used_percent
+
+
+def _remaining_pct(state: AccountState, *, secondary: bool) -> float | None:
+    used_pct = _used_pct(state, secondary=secondary)
+    if used_pct is None:
+        return None
+    return max(0.0, 100.0 - min(100.0, used_pct))
+
+
+def _seconds_until(reset_at: int | float | None, current: float) -> float | None:
+    if reset_at is None:
+        return None
+    return max(0.0, float(reset_at) - current)
+
+
+def _recent_foreground_activity(state: AccountState, current: float) -> bool:
+    return state.last_selected_at is not None and current - state.last_selected_at <= RECENT_FOREGROUND_ACTIVITY_SECONDS
+
+
+def _weekly_pace_floor_pct(state: AccountState, current: float) -> float:
+    remaining_seconds = _seconds_until(state.secondary_reset_at, current)
+    used_pct = _used_pct(state, secondary=True)
+    if remaining_seconds is None or used_pct is None:
+        return 100.0
+
+    elapsed_seconds = max(0.0, SECONDS_PER_WEEK - remaining_seconds)
+    expected_used_pct = min(100.0, (elapsed_seconds / SECONDS_PER_WEEK) * 100.0)
+    behind_pace = used_pct + 5.0 < expected_used_pct
+
+    if remaining_seconds <= 6 * SECONDS_PER_HOUR and behind_pace:
+        pace_floor = 0.0
+    elif remaining_seconds <= SECONDS_PER_DAY and behind_pace:
+        pace_floor = 2.0
+    elif behind_pace:
+        pace_floor = 5.0
+    else:
+        pace_floor = 15.0
+
+    if _recent_foreground_activity(state, current):
+        pace_floor = max(pace_floor, 25.0)
+
+    return max(PRESERVE_MIN_WEEKLY_FLOOR_PCT, pace_floor)
+
+
+def _short_window_floor_pct(state: AccountState, current: float, *, preserve_count: int) -> float:
+    remaining_seconds = _seconds_until(state.reset_at, current)
+    floor = PRESERVE_MIN_SHORT_WINDOW_FLOOR_PCT
+    if remaining_seconds is None:
+        return 100.0
+    if remaining_seconds > SECONDS_PER_HOUR:
+        floor = max(floor, 20.0)
+    if _recent_foreground_activity(state, current):
+        floor = max(floor, 30.0)
+    if preserve_count <= 1:
+        floor = max(floor, 25.0)
+    return floor
+
+
+def _preserve_allows_opportunistic_burn(state: AccountState, current: float, *, preserve_count: int) -> bool:
+    if _remaining_pct(state, secondary=True) is None or _remaining_pct(state, secondary=False) is None:
+        return False
+    if state.secondary_reset_at is None or state.reset_at is None:
+        return False
+    weekly_floor = _weekly_pace_floor_pct(state, current)
+    short_floor = _short_window_floor_pct(state, current, preserve_count=preserve_count)
+    return (_remaining_pct(state, secondary=True) or 0.0) > weekly_floor and (
+        _remaining_pct(state, secondary=False) or 0.0
+    ) > short_floor
+
+
+def _has_other_usable_foreground_capacity(
+    candidate: AccountState,
+    available: list[AccountState],
+    current: float,
+) -> bool:
+    preserve_count = sum(1 for state in available if _routing_policy(state) == ROUTING_POLICY_PRESERVE)
+    for other in available:
+        if other.account_id == candidate.account_id:
+            continue
+        if other.status not in (AccountStatus.ACTIVE, AccountStatus.REAUTH_REQUIRED):
+            continue
+        if _routing_policy(other) == ROUTING_POLICY_PRESERVE:
+            if _preserve_allows_opportunistic_burn(other, current, preserve_count=preserve_count):
+                return True
+            continue
+        return True
+    return False
+
+
+def _above_emergency_floor(state: AccountState) -> bool:
+    primary_remaining = _remaining_pct(state, secondary=False)
+    secondary_remaining = _remaining_pct(state, secondary=True)
+    if primary_remaining is None or secondary_remaining is None:
+        return False
+    return (
+        primary_remaining > NORMAL_LAST_ACCOUNT_EMERGENCY_FLOOR_PCT
+        and secondary_remaining > NORMAL_LAST_ACCOUNT_EMERGENCY_FLOOR_PCT
+    )
+
+
+def _filter_opportunistic_candidates(
+    available: list[AccountState],
+    current: float,
+) -> tuple[list[AccountState], str | None]:
+    burn_first: list[AccountState] = []
+    normal: list[AccountState] = []
+    preserve: list[AccountState] = []
+    preserve_count = sum(1 for state in available if _routing_policy(state) == ROUTING_POLICY_PRESERVE)
+
+    for state in available:
+        policy = _routing_policy(state)
+        if policy == ROUTING_POLICY_BURN_FIRST:
+            if _has_other_usable_foreground_capacity(state, available, current) or _above_emergency_floor(state):
+                burn_first.append(state)
+        elif policy == ROUTING_POLICY_PRESERVE:
+            if _preserve_allows_opportunistic_burn(state, current, preserve_count=preserve_count):
+                preserve.append(state)
+        else:
+            if _has_other_usable_foreground_capacity(state, available, current) or _above_emergency_floor(state):
+                normal.append(state)
+
+    if burn_first or normal or preserve:
+        return [*burn_first, *normal, *preserve], None
+
+    if any(_routing_policy(state) == ROUTING_POLICY_PRESERVE for state in available):
+        return [], "preserve floor or stale usage data blocks opportunistic burn"
+    return [], "no expendable account has emergency foreground reserve"
+
+
+def _reset_preference_bucket(state: AccountState, current: float, window: ResetPreferenceWindow) -> int:
+    if window == "primary":
+        reset_at = state.primary_reset_at
+        if reset_at is None:
+            reset_at = state.priority_reset_at if state.priority_reset_at is not None else state.secondary_reset_at
+    else:
+        reset_at = state.priority_reset_at if state.priority_reset_at is not None else state.secondary_reset_at
+        if reset_at is None:
+            reset_at = state.primary_reset_at
+    if reset_at is None:
+        return UNKNOWN_RESET_BUCKET_DAYS * SECONDS_PER_DAY
+    remaining_seconds = max(0, int(reset_at - current))
+    if window == "secondary":
+        return remaining_seconds // SECONDS_PER_DAY
+    return remaining_seconds
+
+
+def _prefer_earlier_reset_candidates(
+    available: list[AccountState],
+    current: float,
+    window: ResetPreferenceWindow,
+) -> list[AccountState]:
+    earliest_bucket = min(_reset_preference_bucket(state, current, window) for state in available)
+    return [state for state in available if _reset_preference_bucket(state, current, window) == earliest_bucket]
+
+
+def _fallback_secondary_capacity_credits(plan_type: str | None) -> float:
+    normalized = (plan_type or "").strip().lower()
+    resolved_plan = CAPACITY_PLAN_ALIASES.get(normalized, normalized or UNKNOWN_PLAN_FALLBACK)
+    return PLAN_CAPACITY_CREDITS_SECONDARY.get(
+        resolved_plan,
+        PLAN_CAPACITY_CREDITS_SECONDARY[UNKNOWN_PLAN_FALLBACK],
+    )
+
+
+def _known_expired_reauth(state: AccountState, current: float) -> bool:
+    """Return whether a warning-state account has crossed known token expiry."""
+    return (
+        state.status == AccountStatus.REAUTH_REQUIRED
+        and state.access_token_expires_at is not None
+        and state.access_token_expires_at <= current
+    )
+
+
+def select_account(
+    states: Iterable[AccountState],
+    now: float | None = None,
+    *,
+    prefer_earlier_reset: bool = False,
+    prefer_earlier_reset_window: ResetPreferenceWindow = "secondary",
+    routing_strategy: RoutingStrategy = "capacity_weighted",
+    allow_backoff_fallback: bool = True,
+    deterministic_probe: bool = False,
+    recovery_probe_only: bool = False,
+    relative_availability_power: float = DEFAULT_RELATIVE_AVAILABILITY_POWER,
+    relative_availability_top_k: int = DEFAULT_RELATIVE_AVAILABILITY_TOP_K,
+    usage_weighted_order: UsageWeightedOrder = "secondary_first",
+    traffic_class: TrafficClass = TRAFFIC_CLASS_FOREGROUND,
+    ignore_standard_quota: bool = False,
+    bypass_quota_exceeded: bool = False,
+    bypass_quota_exceeded_account_ids: Collection[str] | None = None,
+    primary_first_usage_weighted: bool = False,
+    routing_costs: RoutingCostsByAccount | None = None,
+    replica_salt: str | None = None,
+    selection_seed: str | None = None,
+    allow_usage_exhaustion_error: bool = True,
+    usage_exhaustion_states: Iterable[AccountState] | None = None,
+) -> SelectionResult:
+    """Select an eligible account by applying availability checks and routing strategy.
+
+    This function filters out accounts that cannot currently serve traffic
+    (for example paused, deactivated, still rate-limited, or in active
+    cooldown), attempts controlled recovery from transient error backoff,
+    and then chooses a candidate using the configured balancing strategy.
+
+    Args:
+        states: Candidate account states to evaluate for the current request.
+        now: Unix timestamp in seconds used as the evaluation clock. If
+            ``None``, the current system time is used.
+        prefer_earlier_reset: Whether to bias selection toward accounts whose
+            configured quota window resets sooner.
+        prefer_earlier_reset_window: Quota window used for reset preference
+            (``"primary"`` for short window or ``"secondary"`` for weekly),
+            with the other window used as fallback when the configured reset is
+            unavailable.
+        routing_strategy: Balancing strategy used to pick from the effective
+            pool (``"capacity_weighted"``, ``"sequential_drain"``,
+            ``"reset_drain"``, ``"single_account"``, ``"relative_availability"``,
+            ``"fill_first"``, ``"round_robin"``, or ``"usage_weighted"``).
+        allow_backoff_fallback: Whether to allow a fallback attempt with the
+            backoff account nearest to recovery when no fully available
+            account exists.
+        deterministic_probe: Whether weighted strategies should use a
+            deterministic probe order instead of random weighted choice.
+        recovery_probe_only: Whether to return only a due probing-account
+            recovery admission and otherwise return no selection. Health-tier
+            bypass strategies ignore this internal mode.
+        relative_availability_power: Exponent applied to normalized relative
+            availability weights.
+        relative_availability_top_k: Maximum number of highest-weight
+            relative-availability candidates retained before weighted draw.
+        usage_weighted_order: Whether usage-weighted routing ranks secondary
+            window pressure first, or primary-window pressure first for
+            budget-safe fallback selection.
+        traffic_class: Whether the request is normal foreground traffic or
+            opportunistic traffic that may only use explicitly expendable
+            account capacity.
+        ignore_standard_quota: Whether to ignore the account's standard
+            primary/secondary quota status. This is only for models that are
+            gated by a separate additional quota pool.
+        bypass_quota_exceeded: Backward-compatible alias for ignoring standard
+            quota status for gated quota selection.
+        bypass_quota_exceeded_account_ids: Optional narrower account-id scope
+            for the same bypass.
+        primary_first_usage_weighted: Whether usage-weighted routing should
+            rank by primary-window pressure before secondary-window pressure.
+        routing_costs: Optional request-scoped planner costs. Lower cost wins
+            after hard eligibility, health tier, and reset-bucket filtering.
+        selection_seed: Optional seed making the pick stable for a given
+            seed value instead of load-proportional. Applied to the pool the
+            configured strategy has already narrowed to, so it decides only
+            which of the accounts the strategy was about to draw from is
+            taken. Drain strategies (``sequential_drain``, ``reset_drain``,
+            ``single_account``) have one correct answer and ignore it.
+        replica_salt: Optional per-replica salt mixed into the final
+            ``round_robin`` tie-break so peer replicas break exact ties toward
+            different accounts. When ``None``, the process-wide salt configured
+            via ``configure_replica_salt`` (else the host identity) is used. It
+            affects only the final tie-break; the primary usage/health/cost
+            ordering is unchanged.
+
+    Returns:
+        A ``SelectionResult`` containing the selected ``AccountState`` and no
+        error message when routing can proceed, or ``None`` plus a
+        human-readable error message when no account is eligible.
+    """
+    current = now or time.time()
+    available: list[AccountState] = []
+    in_error_backoff: list[AccountState] = []
+    all_states = list(states)
+    usage_exhaustion_state_list = list(usage_exhaustion_states) if usage_exhaustion_states is not None else all_states
+    bypass_account_ids = None if bypass_quota_exceeded_account_ids is None else set(bypass_quota_exceeded_account_ids)
+
+    for state in all_states:
+        bypass_standard_quota = (
+            ignore_standard_quota
+            or state.ignore_standard_quota
+            or bypass_quota_exceeded
+            or (bypass_account_ids is not None and state.account_id in bypass_account_ids)
+        )
+        if state.status == AccountStatus.DEACTIVATED:
+            continue
+        if state.status == AccountStatus.PAUSED:
+            continue
+        if _known_expired_reauth(state, current):
+            continue
+        if state.status == AccountStatus.RATE_LIMITED:
+            if state.reset_at and current >= state.reset_at:
+                state.status = AccountStatus.ACTIVE
+                state.used_percent = 0.0
+                state.error_count = 0
+                state.reset_at = None
+            elif not bypass_standard_quota:
+                continue
+        if state.status == AccountStatus.QUOTA_EXCEEDED:
+            if state.reset_at and current >= state.reset_at:
+                state.status = AccountStatus.ACTIVE
+                state.used_percent = 0.0
+                state.secondary_used_percent = 0.0
+                state.reset_at = None
+            elif not bypass_standard_quota:
+                continue
+        if state.cooldown_until and current >= state.cooldown_until:
+            state.cooldown_until = None
+            state.last_error_at = None
+            state.error_count = 0
+        if state.cooldown_until and current < state.cooldown_until:
+            continue
+        if state.error_count >= ERROR_BACKOFF_THRESHOLD:
+            backoff = min(300, 30 * (2 ** (state.error_count - ERROR_BACKOFF_THRESHOLD)))
+            if state.last_error_at and current - state.last_error_at < backoff:
+                in_error_backoff.append(state)
+                continue
+            # Error backoff expired — reset error state so recovery is
+            # not penalised by stale counts. The account has already
+            # been held back for the full backoff period; letting it
+            # re-enter the pool with a clean slate avoids the problem
+            # where a previously-high error_count causes an immediate
+            # return to maximum backoff on the very next transient error.
+            state.error_count = 0
+            state.last_error_at = None
+        available.append(state)
+
+    if traffic_class == TRAFFIC_CLASS_OPPORTUNISTIC and available:
+        opportunistic_available, reason = _filter_opportunistic_candidates(available, current)
+        if not opportunistic_available:
+            return SelectionResult(None, f"opportunistic burn window closed: {reason}")
+        available = opportunistic_available
+
+    if not available:
+        in_error_backoff_ids = {state.account_id for state in in_error_backoff}
+        hard_blocked_exists = any(
+            (
+                state.status
+                in (
+                    AccountStatus.PAUSED,
+                    AccountStatus.DEACTIVATED,
+                    AccountStatus.RATE_LIMITED,
+                    AccountStatus.QUOTA_EXCEEDED,
+                )
+                or _known_expired_reauth(state, current)
+            )
+            and state.account_id not in in_error_backoff_ids
+            for state in all_states
+        )
+        if allow_backoff_fallback and (len(in_error_backoff) > 1 or (in_error_backoff and hard_blocked_exists)):
+
+            def _backoff_expires_at(s: AccountState) -> float:
+                backoff = min(300, 30 * (2 ** (s.error_count - ERROR_BACKOFF_THRESHOLD)))
+                return (s.last_error_at or 0.0) + backoff
+
+            available.append(min(in_error_backoff, key=_backoff_expires_at))
+            if traffic_class == TRAFFIC_CLASS_OPPORTUNISTIC:
+                opportunistic_available, reason = _filter_opportunistic_candidates(available, current)
+                if not opportunistic_available:
+                    return SelectionResult(None, f"opportunistic burn window closed: {reason}")
+                available = opportunistic_available
+        else:
+            if allow_usage_exhaustion_error:
+                usage_exhaustion = pool_usage_exhaustion(
+                    usage_exhaustion_state_list,
+                    current=current,
+                    ignore_standard_quota=ignore_standard_quota or bypass_quota_exceeded,
+                    ignore_standard_quota_account_ids=bypass_account_ids,
+                )
+                if usage_exhaustion is not None:
+                    return usage_exhaustion
+            expired_reauth = [state for state in all_states if _known_expired_reauth(state, current)]
+            deactivated = [s for s in all_states if s.status == AccountStatus.DEACTIVATED]
+            paused = [s for s in all_states if s.status == AccountStatus.PAUSED]
+            rate_limited = [s for s in all_states if s.status == AccountStatus.RATE_LIMITED]
+            quota_exceeded = [s for s in all_states if s.status == AccountStatus.QUOTA_EXCEEDED]
+
+            if not rate_limited and not quota_exceeded:
+                if paused and expired_reauth and deactivated:
+                    return SelectionResult(None, "All accounts are paused, deactivated, or require re-authentication")
+                if paused and expired_reauth:
+                    return SelectionResult(None, "All accounts are paused or require re-authentication")
+                if paused and deactivated:
+                    return SelectionResult(None, "All accounts are paused or deactivated")
+                if expired_reauth and deactivated:
+                    return SelectionResult(None, "All accounts are deactivated or require re-authentication")
+                if paused:
+                    return SelectionResult(None, "All accounts are paused")
+                if expired_reauth:
+                    return SelectionResult(None, "All accounts require re-authentication")
+                if deactivated:
+                    return SelectionResult(None, "All accounts are deactivated")
+            if quota_exceeded:
+                reset_candidates = [s.reset_at for s in quota_exceeded if s.reset_at]
+                if reset_candidates:
+                    wait_seconds = max(0, min(reset_candidates) - int(current))
+                    return SelectionResult(None, _format_retry_hint(wait_seconds))
+            cooldowns = [s.cooldown_until for s in all_states if s.cooldown_until and s.cooldown_until > current]
+            if cooldowns:
+                wait_seconds = max(0.0, min(cooldowns) - current)
+                return SelectionResult(None, _format_retry_hint(wait_seconds))
+            return SelectionResult(None, "No available accounts")
+
+    def _reset_first_sort_key(state: AccountState) -> tuple[int, float, float, float, float, str]:
+        reset_bucket_days = _reset_preference_bucket(state, current, prefer_earlier_reset_window)
+        secondary_used, primary_used, last_selected, account_id = _usage_sort_key(state)
+        return (
+            reset_bucket_days,
+            _planner_cost(state, routing_costs),
+            secondary_used,
+            primary_used,
+            last_selected,
+            account_id,
+        )
+
+    def _primary_reset_first_sort_key(state: AccountState) -> tuple[int, float, float, float, float, str]:
+        reset_bucket_days = _reset_preference_bucket(state, current, prefer_earlier_reset_window)
+        primary_used, secondary_used, last_selected, account_id = _primary_usage_sort_key(state)
+        return (
+            reset_bucket_days,
+            _planner_cost(state, routing_costs),
+            primary_used,
+            secondary_used,
+            last_selected,
+            account_id,
+        )
+
+    def _primary_usage_sort_key_with_cost(state: AccountState) -> tuple[float, float, float, float, str]:
+        primary_used, secondary_used, last_selected, account_id = _primary_usage_sort_key(state)
+        return _planner_cost(state, routing_costs), primary_used, secondary_used, last_selected, account_id
+
+    def _usage_sort_key_with_cost(state: AccountState) -> tuple[float, float, float, float, str]:
+        secondary_used, primary_used, last_selected, account_id = _usage_sort_key(state)
+        return _planner_cost(state, routing_costs), secondary_used, primary_used, last_selected, account_id
+
+    def _seeded_pick(pool: list[AccountState], seed: str) -> AccountState:
+        """The strategy's draw, replaced by a stable per-seed choice.
+
+        Applied to the pool the strategy has already narrowed to -- its planner
+        cost and reset-preference filters run first -- so the seed decides only
+        *which* of the accounts the strategy was about to draw from is taken,
+        never whether an account it excluded becomes eligible.
+        """
+        return _seeded_account(pool, seed)
+
+    round_robin_salt = _effective_replica_salt(replica_salt)
+
+    def _round_robin_sort_key(state: AccountState) -> tuple[float, float, str]:
+        # Primary ordering: lowest planner cost, then least-recently-selected.
+        # The final tie-break is decorrelated per replica (keyed hash of the
+        # account id) so peers spread exact ties across equally-good accounts
+        # instead of all herding onto the lexicographically-first account.
+        return (
+            _planner_cost(state, routing_costs),
+            state.last_selected_at or 0.0,
+            _decorrelated_tie_breaker(state.account_id, round_robin_salt),
+        )
+
+    if routing_strategy == "single_account":
+        selected = min(available, key=lambda state: state.account_id)
+        return SelectionResult(selected, None)
+
+    if routing_strategy == "sequential_drain":
+        selected = min(available, key=_sequential_drain_sort_key)
+        return SelectionResult(selected, None)
+
+    if routing_strategy == "reset_drain":
+        selected = min(available, key=lambda state: _reset_drain_sort_key(state, current))
+        return SelectionResult(selected, None)
+
+    healthy = [s for s in available if s.health_tier == HEALTH_TIER_HEALTHY]
+    probing = [s for s in available if s.health_tier == HEALTH_TIER_PROBING]
+    draining = [s for s in available if s.health_tier == HEALTH_TIER_DRAINING]
+    # Recovery is a liveness admission inside the already-eligible pool. Pick
+    # it before routing-policy preferences so burn/preserve policy cannot make
+    # PROBING permanent, but never before quota/cooldown/security filtering.
+    # A seeded caller never carries the probe. The due probe is whichever
+    # probing account went quiet longest, so admitting one advances its clock
+    # and hands the next turn to a different sibling -- the rotation the seed
+    # exists to prevent, and the conversation fan-out the seed exists to stop.
+    # Probes ride on unbound traffic and on every other thread, so this exempts
+    # the handful of conversations being held warm through an isolation window
+    # rather than starving recovery. ``recovery_probe_only`` is the probe pass
+    # itself and is never seeded.
+    due_probe = (
+        _oldest_due_probing_account(probing, current=current)
+        if (healthy or recovery_probe_only) and selection_seed is None
+        else None
+    )
+    if recovery_probe_only and due_probe is None:
+        return SelectionResult(None, None)
+    health_pool = [due_probe] if due_probe is not None else healthy or probing or draining or available
+    burn_first = [s for s in health_pool if _routing_policy(s) == ROUTING_POLICY_BURN_FIRST]
+    normal = [s for s in health_pool if _routing_policy(s) == ROUTING_POLICY_NORMAL]
+    preserve = [s for s in health_pool if _routing_policy(s) == ROUTING_POLICY_PRESERVE]
+    effective_pool = burn_first or normal or preserve or health_pool
+    effective_prefer_earlier_reset = prefer_earlier_reset and routing_strategy != "relative_availability"
+
+    if routing_strategy == "round_robin":
+        selected = (
+            _seeded_pick(_lowest_planner_cost_candidates(effective_pool, routing_costs), selection_seed)
+            if selection_seed is not None
+            else min(effective_pool, key=_round_robin_sort_key)
+        )
+    elif routing_strategy == "capacity_weighted":
+        candidate_pool = (
+            _prefer_earlier_reset_candidates(effective_pool, current, prefer_earlier_reset_window)
+            if effective_prefer_earlier_reset
+            else effective_pool
+        )
+        if deterministic_probe:
+            selected = min(candidate_pool, key=lambda state: _capacity_probe_sort_key_with_cost(state, routing_costs))
+        else:
+            candidate_pool = _lowest_planner_cost_candidates(candidate_pool, routing_costs)
+            selected = _select_capacity_weighted(candidate_pool, selection_seed=selection_seed)
+    elif routing_strategy == "relative_availability":
+        candidate_pool = _lowest_planner_cost_candidates(effective_pool, routing_costs)
+        selected = _select_relative_availability(
+            candidate_pool,
+            current=current,
+            power=relative_availability_power,
+            top_k=relative_availability_top_k,
+            deterministic_probe=deterministic_probe,
+            selection_seed=selection_seed,
+        )
+    elif routing_strategy == "fill_first":
+        candidate_pool = (
+            _prefer_earlier_reset_candidates(effective_pool, current, prefer_earlier_reset_window)
+            if prefer_earlier_reset
+            else effective_pool
+        )
+        # ``fill_first`` deliberately ranks by *highest* usage to drain an
+        # account before opening the next, and that ranking is already stable
+        # across admissions, so the seed has nothing to add and would only
+        # scatter threads onto fresh accounts.
+        selected = _select_fill_first(candidate_pool)
+    else:
+        effective_usage_weighted_order: UsageWeightedOrder = (
+            "primary_first" if primary_first_usage_weighted else usage_weighted_order
+        )
+        if selection_seed is not None:
+            # The sort keys this branch would otherwise use put reset bucket
+            # first and planner cost second, so the seed has to be spent below
+            # both: a thread must not be held on a 30-day-reset or high-cost
+            # account the ordering had already ruled out.
+            seeded_pool = (
+                _prefer_earlier_reset_candidates(effective_pool, current, prefer_earlier_reset_window)
+                if effective_prefer_earlier_reset
+                else effective_pool
+            )
+            selected = _seeded_pick(_lowest_planner_cost_candidates(seeded_pool, routing_costs), selection_seed)
+        elif effective_usage_weighted_order == "primary_first":
+            selected = min(
+                effective_pool,
+                key=(
+                    _primary_reset_first_sort_key
+                    if effective_prefer_earlier_reset
+                    else _primary_usage_sort_key_with_cost
+                ),
+            )
+        else:
+            selected = min(
+                effective_pool,
+                key=_reset_first_sort_key if effective_prefer_earlier_reset else _usage_sort_key_with_cost,
+            )
+    return SelectionResult(selected, None)
+
+
+def _oldest_due_probing_account(
+    probing: Iterable[AccountState],
+    *,
+    current: float,
+) -> AccountState | None:
+    due = [
+        state
+        for state in probing
+        if state.last_selected_at is None or current - state.last_selected_at >= PROBE_QUIET_SECONDS
+    ]
+    if not due:
+        return None
+    return min(
+        due,
+        key=lambda state: (
+            state.last_selected_at if state.last_selected_at is not None else float("-inf"),
+            state.account_id,
+        ),
+    )
+
+
+def _remaining_secondary_credits(state: AccountState) -> float:
+    """Return remaining absolute credits for the secondary (7-day) window."""
+    capacity = (
+        state.priority_capacity_credits if state.priority_capacity_credits is not None else state.capacity_credits
+    )
+    if capacity is None:
+        capacity = _fallback_secondary_capacity_credits(state.plan_type)
+    elif capacity <= 0:
+        return 0.0
+    primary_used = _priority_primary_used(state)
+    used_pct = _priority_secondary_used(state, primary_used)
+    return max(0.0, capacity * (1.0 - min(used_pct, 100.0) / 100.0))
+
+
+def _priority_primary_used(state: AccountState) -> float:
+    value = state.priority_used_percent if state.priority_used_percent is not None else state.used_percent
+    return value if value is not None else 0.0
+
+
+def _priority_secondary_used(state: AccountState, primary_used: float | None = None) -> float:
+    if state.limit_scoped_usage and state.priority_secondary_used_percent is None:
+        return primary_used if primary_used is not None else _priority_primary_used(state)
+    value = (
+        state.priority_secondary_used_percent
+        if state.priority_secondary_used_percent is not None
+        else state.secondary_used_percent
+    )
+    if value is not None:
+        return value
+    return primary_used if primary_used is not None else _priority_primary_used(state)
+
+
+def _relative_availability_divisor_seconds(state: AccountState, current: float) -> float:
+    reset_at = state.priority_reset_at if state.priority_reset_at is not None else state.secondary_reset_at
+    if reset_at is None:
+        remaining_seconds = float(UNKNOWN_RESET_FALLBACK_SECONDS)
+    else:
+        remaining_seconds = max(0.0, float(reset_at) - current)
+    return max(remaining_seconds, float(RELATIVE_AVAILABILITY_MIN_DIVISOR_SECONDS))
+
+
+def _relative_availability_remaining_seconds(state: AccountState, current: float) -> float:
+    reset_at = state.priority_reset_at if state.priority_reset_at is not None else state.secondary_reset_at
+    if reset_at is None:
+        return float(UNKNOWN_RESET_FALLBACK_SECONDS)
+    return max(0.0, float(reset_at) - current)
+
+
+def _relative_availability_raw_score(state: AccountState, current: float) -> float:
+    remaining_credits = _remaining_secondary_credits(state)
+    if remaining_credits <= 0.0:
+        return 0.0
+    return remaining_credits / _relative_availability_divisor_seconds(state, current)
+
+
+def _relative_availability_label(state: AccountState) -> str:
+    return state.account_id
+
+
+def _relative_availability_score_per_minute(raw_score: float) -> float:
+    return raw_score * 60.0
+
+
+def _log_relative_availability_candidate_scores(
+    raw_scores: list[tuple[AccountState, float]],
+    *,
+    current: float,
+) -> None:
+    for state, raw_score in raw_scores:
+        remaining_seconds = _relative_availability_remaining_seconds(state, current)
+        logger.debug(
+            (
+                f"{_RELATIVE_AVAILABILITY_LOG_PREFIX_CANDIDATE}account=%s "
+                "remaining_credits=%.2f remaining_minutes=%.2f score_per_minute=%.6f"
+            ),
+            _relative_availability_label(state),
+            _remaining_secondary_credits(state),
+            remaining_seconds / 60.0,
+            _relative_availability_score_per_minute(raw_score),
+        )
+
+
+def _log_relative_availability_top_k(
+    weighted_candidates: list[tuple[AccountState, float, float]],
+    *,
+    current: float,
+) -> None:
+    formatted_candidates = ", ".join(
+        (
+            f"account={_relative_availability_label(state)} "
+            f"remaining_credits={_remaining_secondary_credits(state):.2f} "
+            f"remaining_minutes={_relative_availability_remaining_seconds(state, current) / 60.0:.2f} "
+            f"score_per_minute={_relative_availability_score_per_minute(raw_score):.6f} "
+            f"weight={weight:.8f}"
+        )
+        for state, weight, raw_score in weighted_candidates
+    )
+    logger.info("%s%s", _RELATIVE_AVAILABILITY_LOG_PREFIX_TOP_K, formatted_candidates)
+
+
+def _relative_availability_weighted_candidates(
+    available: list[AccountState],
+    *,
+    current: float,
+    power: float,
+    top_k: int,
+    membership_seed: str | None = None,
+) -> list[tuple[AccountState, float, float]]:
+    raw_scores = [(state, _relative_availability_raw_score(state, current)) for state in available]
+    _log_relative_availability_candidate_scores(raw_scores, current=current)
+    best_raw_score = max((score for _, score in raw_scores), default=0.0)
+    if best_raw_score <= 0.0:
+        return []
+
+    weighted: list[tuple[AccountState, float, float]] = []
+    safe_power = power if power > 0.0 else DEFAULT_RELATIVE_AVAILABILITY_POWER
+    for state, raw_score in raw_scores:
+        normalized_score = raw_score / best_raw_score
+        weight = normalized_score**safe_power
+        # The floor is an eligibility bound, not a ranking, so it applies to a
+        # seeded caller too: a retained thread must not be parked on an account
+        # that has fallen far behind the best available one.
+        if weight < RELATIVE_AVAILABILITY_MIN_WEIGHT_FRACTION:
+            continue
+        weighted.append((state, weight, raw_score))
+
+    if not weighted:
+        return []
+
+    if membership_seed is not None:
+        # ``_usage_sort_key`` ends in ``last_selected_at``, so exact ties are
+        # broken by *recency*: each admitted turn advances the winner's
+        # timestamp and ejects it from the top-k on the next turn. Ordering by
+        # the caller's own seed instead is stable turn to turn, and keyed per
+        # thread rather than globally -- otherwise every retained conversation
+        # is handed the same k accounts and the rest of an equally scored pool
+        # never serves a turn.
+        weighted.sort(key=lambda item: _decorrelated_tie_breaker(item[0].account_id, membership_seed))
+    else:
+        weighted.sort(key=lambda item: (-item[1], -item[2], *_usage_sort_key(item[0])))
+    # The ``k`` cut is a *rank* cut over live availability, so a sibling can
+    # stay well within reach and still drop out of it when another account's
+    # usage refreshes. A seeded caller keeps every candidate above the minimum
+    # weight fraction instead: that floor moves only when an account genuinely
+    # falls away from the best, which is a change in the pool rather than a
+    # reshuffle of it.
+    safe_top_k = len(weighted) if membership_seed is not None else max(1, top_k)
+    top_candidates = weighted[:safe_top_k]
+    _log_relative_availability_top_k(top_candidates, current=current)
+    return top_candidates
+
+
+def _log_relative_availability_winner(
+    winner: AccountState,
+    *,
+    current: float,
+    weight: float | None,
+    raw_score: float,
+) -> None:
+    remaining_seconds = _relative_availability_remaining_seconds(winner, current)
+    logger.info(
+        (
+            f"{_RELATIVE_AVAILABILITY_LOG_PREFIX_WINNER}account=%s "
+            "remaining_credits=%.2f remaining_minutes=%.2f score_per_minute=%.6f weight=%s"
+        ),
+        _relative_availability_label(winner),
+        _remaining_secondary_credits(winner),
+        remaining_seconds / 60.0,
+        _relative_availability_score_per_minute(raw_score),
+        f"{weight:.8f}" if weight is not None else "fallback",
+    )
+
+
+def _select_relative_availability(
+    available: list[AccountState],
+    *,
+    current: float,
+    power: float,
+    top_k: int,
+    deterministic_probe: bool,
+    selection_seed: str | None = None,
+) -> AccountState:
+    weighted_candidates = _relative_availability_weighted_candidates(
+        available,
+        current=current,
+        power=power,
+        top_k=top_k,
+        membership_seed=selection_seed,
+    )
+    if not weighted_candidates:
+        winner = (
+            _seeded_least_used(available, selection_seed)
+            if selection_seed is not None
+            else min(available, key=_usage_sort_key)
+        )
+        _log_relative_availability_winner(
+            winner,
+            current=current,
+            weight=None,
+            raw_score=_relative_availability_raw_score(winner, current),
+        )
+        return winner
+    if deterministic_probe:
+        winner, weight, raw_score = weighted_candidates[0]
+        _log_relative_availability_winner(winner, current=current, weight=weight, raw_score=raw_score)
+        return winner
+    states = [state for state, _, _ in weighted_candidates]
+    # Top-k membership is decided by availability alone (above); the recent
+    # error rate only discounts the draw so a penalized account still competes.
+    weights = [weight * _selection_weight_multiplier(state) for state, weight, _ in weighted_candidates]
+    total = sum(weights)
+    if total <= 0.0:
+        winner = (
+            _seeded_least_used(available, selection_seed)
+            if selection_seed is not None
+            else min(available, key=_usage_sort_key)
+        )
+        _log_relative_availability_winner(
+            winner,
+            current=current,
+            weight=None,
+            raw_score=_relative_availability_raw_score(winner, current),
+        )
+        return winner
+    if selection_seed is not None:
+        # Inside the top-k the draw has already been narrowed by availability,
+        # and membership is stable for a seeded caller (see
+        # ``stable_membership``), so the seed picks uniformly among the
+        # survivors. Unlike the capacity draw, these weights are *normalized*
+        # against the current best score: they move whenever any account's
+        # availability moves, including the retained thread's own substitute as
+        # it serves. Weighting the pick by them would put every thread one
+        # admission away from flipping, which is the rotation the seed exists
+        # to prevent -- so here the narrowing carries the availability
+        # preference and the seed only chooses among what survived it.
+        # Zero-weight candidates are dropped as in the capacity draw.
+        winner = _seeded_account(
+            [state for state, weight in zip(states, weights, strict=True) if weight > 0.0],
+            selection_seed,
+        )
+    else:
+        winner = random.choices(states, weights=weights, k=1)[0]
+    for state, weight, raw_score in weighted_candidates:
+        if state.account_id == winner.account_id:
+            _log_relative_availability_winner(winner, current=current, weight=weight, raw_score=raw_score)
+            break
+    return winner
+
+
+def _seeded_account(pool: list[AccountState], seed: str) -> AccountState:
+    """The seed's choice among candidates the strategy was about to draw from."""
+    return min(pool, key=lambda state: _decorrelated_tie_breaker(state.account_id, seed))
+
+
+def _seeded_least_used(available: list[AccountState], seed: str) -> AccountState:
+    """The exhausted-pool fallback, made stable for a seeded caller.
+
+    ``min(available, key=_usage_sort_key)`` ends in ``last_selected_at``, so
+    when every candidate is equally spent -- which is exactly when this
+    fallback runs -- being chosen is what loses you the next turn, and the
+    caller's thread rotates through the tied accounts. Seed the tie instead,
+    keeping the usage ordering that decides which accounts are tied at all.
+    """
+    least_used = min(_usage_sort_key(state)[:2] for state in available)
+    return _seeded_account([state for state in available if _usage_sort_key(state)[:2] == least_used], seed)
+
+
+def _stable_tie_breaker(account_id: str) -> str:
+    return hashlib.sha256(account_id.encode("utf-8")).hexdigest()
+
+
+# ---------------------------------------------------------------------------
+# Cross-replica round-robin decorrelation
+#
+# ``round_robin`` orders candidates by planner cost, then least-recently-
+# selected time, then a stable final tie-break. In a multi-replica deployment
+# every replica shares one account set and computes the identical key, so an
+# *exact* tie on the primary keys (for example a cold start where every account
+# is never-selected, or several accounts whose ``last_selected_at`` reset to
+# ``0.0``) makes every replica break the tie toward the same account -- herding
+# load onto it. Mixing a stable per-replica salt into the final tie-break
+# decorrelates that choice so peers spread across equally-good accounts, without
+# touching the primary ordering.
+# ---------------------------------------------------------------------------
+
+_configured_replica_salt: str | None = None
+_process_replica_salt: str | None = None
+
+
+def configure_replica_salt(salt: str | None) -> None:
+    """Set the process-wide replica salt used to decorrelate round-robin ties.
+
+    Called once during proxy start-up with the replica's stable identity (the
+    HTTP responses-session bridge instance id). An empty or ``None`` value
+    clears the override and restores the lazily-resolved host default. The salt
+    is process-stable by design: it must not change between selections so a
+    replica breaks a given tie the same way every time.
+    """
+    global _configured_replica_salt
+    normalized = salt.strip() if salt else ""
+    _configured_replica_salt = normalized or None
+
+
+def _default_replica_salt() -> str:
+    """Return the cached host-identity fallback salt.
+
+    Resolved once per process so the salt never varies between calls within a
+    replica, keeping single-replica selection deterministic. Matches the HTTP
+    bridge instance-id default (the host name) so an unconfigured process still
+    decorrelates by pod/host.
+    """
+    global _process_replica_salt
+    if _process_replica_salt is None:
+        hostname = socket.gethostname().strip()
+        _process_replica_salt = hostname or "codex-lb"
+    return _process_replica_salt
+
+
+def _effective_replica_salt(explicit: str | None) -> str:
+    if explicit is not None and explicit.strip():
+        return explicit.strip()
+    if _configured_replica_salt is not None:
+        return _configured_replica_salt
+    return _default_replica_salt()
+
+
+def _decorrelated_tie_breaker(account_id: str, salt: str) -> str:
+    """Keyed hash of ``account_id`` under a per-replica ``salt``.
+
+    Deterministic for a given ``(salt, account_id)`` pair -- a replica always
+    breaks the same tie the same way -- while distinct salts yield independent
+    orderings across replicas.
+    """
+    return hashlib.sha256(f"{salt}\x00{account_id}".encode("utf-8")).hexdigest()
+
+
+def _configured_capacity_credits(state: AccountState) -> float:
+    if state.capacity_credits is not None and state.capacity_credits > 0:
+        return state.capacity_credits
+    return _fallback_secondary_capacity_credits(state.plan_type)
+
+
+def _sequential_drain_sort_key(state: AccountState) -> tuple[float, str, str]:
+    return (
+        _configured_capacity_credits(state),
+        _stable_tie_breaker(state.account_id),
+        state.account_id,
+    )
+
+
+def _weekly_reset_timestamp(state: AccountState, current: float) -> float:
+    if state.secondary_reset_at is not None and float(state.secondary_reset_at) > current:
+        return float(state.secondary_reset_at)
+    if state.reset_at is not None and float(state.reset_at) > current:
+        return float(state.reset_at)
+    return float("inf")
+
+
+def _reset_drain_sort_key(state: AccountState, current: float) -> tuple[int, float, float, float, str, str]:
+    primary_remaining = 100.0 - (state.used_percent if state.used_percent is not None else 0.0)
+    secondary_remaining = 100.0 - (
+        state.secondary_used_percent
+        if state.secondary_used_percent is not None
+        else state.used_percent
+        if state.used_percent is not None
+        else 0.0
+    )
+    reset_at = _weekly_reset_timestamp(state, current)
+    reset_bucket_days = (
+        UNKNOWN_RESET_BUCKET_DAYS if reset_at == float("inf") else max(0, int((reset_at - current) // SECONDS_PER_DAY))
+    )
+    return (
+        reset_bucket_days,
+        -max(0.0, primary_remaining),
+        -max(0.0, secondary_remaining),
+        reset_at,
+        _stable_tie_breaker(state.account_id),
+        state.account_id,
+    )
+
+
+def _capacity_probe_sort_key_with_cost(
+    state: AccountState,
+    routing_costs: RoutingCostsByAccount | None,
+) -> tuple[float, float, float, float, float, str]:
+    secondary_used, primary_used, last_selected, account_id = _usage_sort_key(state)
+    return (
+        _planner_cost(state, routing_costs),
+        -_remaining_secondary_credits(state),
+        secondary_used,
+        primary_used,
+        last_selected,
+        account_id,
+    )
+
+
+def _lowest_planner_cost_candidates(
+    available: list[AccountState],
+    routing_costs: RoutingCostsByAccount | None,
+) -> list[AccountState]:
+    if not routing_costs:
+        return available
+    lowest_cost = min(_planner_cost(state, routing_costs) for state in available)
+    return [state for state in available if _planner_cost(state, routing_costs) == lowest_cost]
+
+
+def _select_capacity_weighted(
+    available: list[AccountState],
+    *,
+    selection_seed: str | None = None,
+) -> AccountState:
+    """Select an account with probability proportional to remaining secondary credits."""
+    weights = [_remaining_secondary_credits(s) * _selection_weight_multiplier(s) for s in available]
+    total = sum(weights)
+    if total <= 0.0:
+        # All accounts exhausted — fall back to deterministic usage-weighted
+        if selection_seed is not None:
+            return _seeded_least_used(available, selection_seed)
+        return min(available, key=_usage_sort_key)
+    if selection_seed is not None:
+        # A zero-weight account is one the draw could never have returned while
+        # a positive-weight sibling existed -- an account whose effective
+        # secondary credits are spent. The seed must not readmit it.
+        #
+        # The *magnitudes* are deliberately dropped here, and only here. They
+        # are live: a usage refresh, an elapsed reset, or an error-rate update
+        # moves them without anything about eligibility changing, so a pick
+        # weighted by them puts every seeded thread one refresh away from
+        # flipping -- which is the fan-out this seed exists to prevent. Ordinary
+        # (unseeded) traffic keeps the full weighted draw, so the pool-level
+        # capacity balance is unchanged; a retained thread simply lands
+        # uniformly among the accounts that draw had already accepted.
+        return _seeded_account(
+            [state for state, weight in zip(available, weights, strict=True) if weight > 0.0],
+            selection_seed,
+        )
+    return random.choices(available, weights=weights, k=1)[0]
+
+
+def _selection_weight_multiplier(state: AccountState) -> float:
+    """Clamp the balancer-supplied draw multiplier so a bad value can never
+    negate or unbound a weight."""
+    multiplier = state.selection_weight_multiplier
+    if multiplier != multiplier or multiplier < 0.0:  # NaN or negative
+        return 1.0
+    return min(multiplier, 1.0)
+
+
+def _fill_first_sort_key(state: AccountState) -> tuple[float, float, str]:
+    primary_used = state.used_percent if state.used_percent is not None else 0.0
+    secondary_used = state.secondary_used_percent if state.secondary_used_percent is not None else 0.0
+    return -primary_used, -secondary_used, state.account_id
+
+
+def _select_fill_first(available: list[AccountState]) -> AccountState:
+    """Pick the eligible account with the highest primary 5h ``used_percent``.
+
+    Deterministic. ``None`` ``used_percent`` is treated as ``0.0`` so a
+    freshly refreshed account ties with an unknown-usage account.
+
+    When two or more candidates share the same primary ``used_percent``,
+    the account with the **higher** secondary (weekly) ``used_percent`` is
+    preferred — i.e. the one with the least remaining weekly capacity.
+    This drains the most-saturated account first and preserves the freshest
+    one for later cycles, matching operator intent for "fill first" behavior.
+    ``account_id`` ascending is the final stable tiebreaker.
+
+    Drained accounts are only reachable here when no healthy or probing
+    account exists, via the existing ``effective_pool`` ladder; this helper
+    introduces no new bypass.
+    """
+    return min(available, key=_fill_first_sort_key)
+
+
+# Minimum persisted cooldown for a metadata-free 429. The error-count backoff
+# starts near 0.2s, which is meaningless as a cross-replica cooldown: a peer
+# replica reading the row would flip it back to ACTIVE almost immediately. The
+# floor applies only to the persisted ``reset_at`` deadline written on the
+# backoff fallback path — Retry-After hints are persisted rounded up to the
+# next whole second (persistence truncates ``reset_at`` via ``int()``, so a
+# short or fractional hint must not round down to an already-elapsed
+# deadline), upstream reset metadata is persisted verbatim, and the local
+# ``cooldown_until`` keeps the raw backoff.
+RATE_LIMITED_MIN_COOLDOWN_SECONDS = 30.0
+
+
+def handle_rate_limit(state: AccountState, error: UpstreamError) -> None:
+    now = time.time()
+    state.status = AccountStatus.RATE_LIMITED
+    state.error_count += 1
+    state.last_error_at = now
+    state.blocked_at = now
+
+    reset_at = _extract_reset_at(error, now=now)
+
+    message = error.get("message")
+    delay = parse_retry_after(message) if message else None
+    if delay is None:
+        delay = backoff_seconds(state.error_count)
+        persisted_delay = max(delay, RATE_LIMITED_MIN_COOLDOWN_SECONDS)
+    else:
+        persisted_delay = delay
+    state.cooldown_until = now + delay
+
+    if reset_at is not None:
+        state.reset_at = reset_at
+    else:
+        # Persist the resolved cooldown deadline so replicas sharing the
+        # database honor it instead of flipping the account back to ACTIVE
+        # from their own (empty) runtime state. The write rides the existing
+        # ``mark_rate_limit -> _persist_state`` status update. Round the
+        # deadline UP to a whole second: ``_persist_state`` truncates via
+        # ``int()``, so a short or fractional Retry-After hint (for example
+        # "500ms", or "1s" near a second boundary) would otherwise persist a
+        # deadline that is already elapsed for peer replicas, dropping the
+        # cooldown entirely.
+        state.reset_at = float(math.ceil(now + persisted_delay))
+
+
+QUOTA_EXCEEDED_COOLDOWN_SECONDS = 120.0
+
+# Upper bound for the user-visible "Try again in {N}s" hint that
+# ``select_account`` surfaces when zero candidates are selectable. The clamp
+# protects clients from waiting the worst-case persisted ``reset_at`` after
+# OpenAI-side reset events that propagate lazily through ``/wham/usage`` (see
+# https://github.com/Soju06/codex-lb/issues/676). codex-lb's background usage
+# refresh runs every ``USAGE_REFRESH_INTERVAL_SECONDS`` (60 s) and the
+# per-status cooldowns are 120s, so a 300s ceiling lets clients reattempt
+# inside the auto-recovery window. The underlying ``AccountState.reset_at``
+# and ``AccountState.cooldown_until`` fields are not clamped.
+SELECTOR_RETRY_HINT_MAX_SECONDS = 300
+
+
+def _format_retry_hint(wait_seconds: float) -> str:
+    capped = min(max(0.0, wait_seconds), float(SELECTOR_RETRY_HINT_MAX_SECONDS))
+    return f"Rate limit exceeded. Try again in {capped:.0f}s"
+
+
+def handle_quota_exceeded(state: AccountState, error: UpstreamError) -> None:
+    now = time.time()
+    state.status = AccountStatus.QUOTA_EXCEEDED
+    state.used_percent = 100.0
+    state.blocked_at = now
+    state.cooldown_until = now + QUOTA_EXCEEDED_COOLDOWN_SECONDS
+
+    reset_at = _extract_reset_at(error, now=now)
+    if reset_at is not None:
+        state.reset_at = reset_at
+    else:
+        state.reset_at = int(now + 3600)
+
+
+def handle_permanent_failure(state: AccountState, error_code: str) -> None:
+    state.status = account_status_for_permanent_failure(error_code)
+    state.deactivation_reason = PERMANENT_FAILURE_CODES.get(
+        error_code,
+        f"Authentication failed: {error_code}",
+    )
+    state.blocked_at = None
+
+
+def account_status_for_permanent_failure(error_code: str) -> AccountStatus:
+    if error_code in REAUTH_REQUIRED_FAILURE_CODES:
+        return AccountStatus.REAUTH_REQUIRED
+    return AccountStatus.DEACTIVATED
+
+
+FailoverAction = Literal["failover_next", "retry_same_account", "surface"]
+
+# Owner-bound burst 429 (a code-less upstream HTTP 429 burst/concurrency
+# rejection on a request that cannot move to another account): bounded
+# same-account backoff before the original rejection is surfaced. Module
+# constants on purpose -- the Settings ratchet is full and this is a transport
+# invariant, not an operator knob. Waits are 1 s, 2 s, 4 s (upstream
+# ``Retry-After`` is a floor), never above ``BURST_SAME_ACCOUNT_MAX_WAIT_SECONDS``.
+BURST_SAME_ACCOUNT_MAX_RETRIES = 3
+BURST_SAME_ACCOUNT_BASE_SECONDS = 1.0
+BURST_SAME_ACCOUNT_MAX_WAIT_SECONDS = 10.0
+# ``Retry-After`` stamped on a surfaced burst 429 that carried none upstream;
+# matches ``app.core.resilience.overload.LOCAL_OVERLOAD_RETRY_AFTER_SECONDS``.
+BURST_SURFACE_RETRY_AFTER_SECONDS = 5
+
+
+def burst_same_account_backoff_seconds(retry_index: int, *, retry_after_seconds: float | None) -> float:
+    """Deterministic wait before same-account burst retry ``retry_index`` (1-based).
+
+    ``min(MAX_WAIT, max(retry_after_seconds or 0, BASE * 2 ** (retry_index - 1)))``:
+    exponential from ``BURST_SAME_ACCOUNT_BASE_SECONDS`` with the upstream
+    ``Retry-After`` (when present and positive) acting as a floor. No jitter, so
+    the caller's timing seam (``scheduler.sleep``) fully owns the wait.
+    """
+    exponential = BURST_SAME_ACCOUNT_BASE_SECONDS * (2 ** (max(retry_index, 1) - 1))
+    floor = float(retry_after_seconds) if retry_after_seconds is not None and retry_after_seconds > 0 else 0.0
+    return min(BURST_SAME_ACCOUNT_MAX_WAIT_SECONDS, max(floor, exponential))
+
+
+def failover_decision(
+    *,
+    failure_class: FailureClass,
+    downstream_visible: bool,
+    candidates_remaining: int,
+    owner_bound: bool = False,
+    same_account_retry_available: bool = False,
+) -> FailoverAction:
+    """Decide how a pre-visible upstream failure is handled.
+
+    ``owner_bound`` means the request cannot move to another account (dispatched
+    account-bound payload, required previous-response / turn-state / file
+    owner). Such a request never fails over -- ``failover_next`` would be a
+    lie -- so it either retries the same account (when the caller reports a
+    bounded same-account retry is still available) or surfaces the failure.
+    """
+    if downstream_visible:
+        return "surface"
+    if owner_bound:
+        return (
+            "retry_same_account"
+            if same_account_retry_available and failure_class != "account_unavailable"
+            else "surface"
+        )
+    if candidates_remaining <= 0:
+        return "surface"
+    if failure_class in ("rate_limit", "quota", "retryable_transient", "account_unavailable"):
+        return "failover_next"
+    return "surface"
+
+
+def plausible_rate_limit_reset_at(
+    reset_at: int | float | None,
+    *,
+    now: float | None = None,
+    allow_rounding_slack: bool = True,
+) -> float | None:
+    """Return a finite near-future reset deadline, or ``None`` when invalid."""
+
+    if reset_at is None or isinstance(reset_at, bool):
+        return None
+    current = time.time() if now is None else now
+    if isinstance(reset_at, float) and not math.isfinite(reset_at):
+        return None
+    horizon = RATE_LIMIT_RESET_MAX_HORIZON_SECONDS
+    if allow_rounding_slack:
+        # Persisted deadlines are whole seconds; ceil may move an otherwise
+        # valid raw deadline just beyond the metadata horizon.
+        horizon += RATE_LIMIT_RESET_ROUNDING_TOLERANCE_SECONDS
+    # Keep this comparison ahead of float conversion: Python integers can be
+    # arbitrarily large, while converting malformed metadata can overflow.
+    if reset_at <= current or reset_at > current + horizon:
+        return None
+    return float(reset_at)
+
+
+def _extract_reset_at(error: UpstreamError, *, now: float) -> int | None:
+    reset_at = plausible_rate_limit_reset_at(
+        error.get("resets_at"),
+        now=now,
+        allow_rounding_slack=False,
+    )
+    if reset_at is not None:
+        rounded_reset_at = int(math.ceil(reset_at))
+        if plausible_rate_limit_reset_at(rounded_reset_at, now=now) is not None:
+            return rounded_reset_at
+        return None
+
+    reset_in = error.get("resets_in_seconds")
+    if reset_in is None or isinstance(reset_in, bool):
+        return None
+    if isinstance(reset_in, float) and not math.isfinite(reset_in):
+        return None
+    if reset_in <= 0 or reset_in > RATE_LIMIT_RESET_MAX_HORIZON_SECONDS:
+        return None
+    rounded_reset_at = int(math.ceil(now + float(reset_in)))
+    if plausible_rate_limit_reset_at(rounded_reset_at, now=now) is not None:
+        return rounded_reset_at
+    return None
+
+
+def evaluate_health_tier(
+    state: AccountState,
+    *,
+    now: float | None = None,
+    drain_entered_at: float | None = None,
+    probe_success_streak: int = 0,
+    drain_primary_threshold_pct: float = DRAIN_PRIMARY_THRESHOLD_PCT,
+    drain_secondary_threshold_pct: float = DRAIN_SECONDARY_THRESHOLD_PCT,
+    drain_error_window_seconds: float = DRAIN_ERROR_WINDOW_SECONDS,
+    drain_error_count_threshold: int = DRAIN_ERROR_COUNT_THRESHOLD,
+    probe_quiet_seconds: float = PROBE_QUIET_SECONDS,
+    probe_success_streak_required: int = PROBE_SUCCESS_STREAK_REQUIRED,
+) -> int:
+    current = now or time.time()
+
+    if state.status in (
+        AccountStatus.RATE_LIMITED,
+        AccountStatus.QUOTA_EXCEEDED,
+        AccountStatus.PAUSED,
+        AccountStatus.DEACTIVATED,
+    ):
+        return state.health_tier
+
+    should_drain = False
+
+    if state.used_percent is not None and state.used_percent >= drain_primary_threshold_pct:
+        should_drain = True
+
+    if state.secondary_used_percent is not None and state.secondary_used_percent >= drain_secondary_threshold_pct:
+        should_drain = True
+
+    if (
+        state.error_count >= drain_error_count_threshold
+        and state.last_error_at is not None
+        and current - state.last_error_at < drain_error_window_seconds
+    ):
+        should_drain = True
+
+    current_tier = state.health_tier
+
+    if current_tier == HEALTH_TIER_HEALTHY:
+        return HEALTH_TIER_DRAINING if should_drain else HEALTH_TIER_HEALTHY
+
+    if current_tier == HEALTH_TIER_DRAINING:
+        if should_drain:
+            return HEALTH_TIER_DRAINING
+        if drain_entered_at is not None and current - drain_entered_at >= probe_quiet_seconds:
+            return HEALTH_TIER_PROBING
+        return HEALTH_TIER_DRAINING
+
+    if current_tier == HEALTH_TIER_PROBING:
+        if should_drain:
+            return HEALTH_TIER_DRAINING
+        if probe_success_streak >= probe_success_streak_required:
+            return HEALTH_TIER_HEALTHY
+        return HEALTH_TIER_PROBING
+
+    return HEALTH_TIER_HEALTHY