codexLB-up 1.25.0b10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (827) hide show
  1. app/__init__.py +10 -0
  2. app/admin_cli.py +494 -0
  3. app/cli.py +339 -0
  4. app/codex_sessions_retag.py +487 -0
  5. app/core/__init__.py +0 -0
  6. app/core/audit/__init__.py +0 -0
  7. app/core/audit/service.py +216 -0
  8. app/core/audit/types.py +94 -0
  9. app/core/auth/__init__.py +236 -0
  10. app/core/auth/api_key_cache.py +65 -0
  11. app/core/auth/dashboard_access.py +429 -0
  12. app/core/auth/dashboard_mode.py +171 -0
  13. app/core/auth/dashboard_session_ttl.py +74 -0
  14. app/core/auth/dashboard_users_cache.py +120 -0
  15. app/core/auth/dependencies.py +710 -0
  16. app/core/auth/external_identity.py +78 -0
  17. app/core/auth/guardian.py +374 -0
  18. app/core/auth/models.py +49 -0
  19. app/core/auth/providers/__init__.py +129 -0
  20. app/core/auth/providers/oidc.py +838 -0
  21. app/core/auth/providers/registry.py +118 -0
  22. app/core/auth/refresh.py +394 -0
  23. app/core/auth/step_up.py +117 -0
  24. app/core/auth/totp.py +81 -0
  25. app/core/balancer/__init__.py +71 -0
  26. app/core/balancer/logic.py +1612 -0
  27. app/core/balancer/rendezvous_hash.py +21 -0
  28. app/core/balancer/types.py +21 -0
  29. app/core/bootstrap.py +172 -0
  30. app/core/cache/invalidation.py +426 -0
  31. app/core/clients/__init__.py +0 -0
  32. app/core/clients/codex.py +996 -0
  33. app/core/clients/codex_version.py +187 -0
  34. app/core/clients/codex_version_snapshot.py +2 -0
  35. app/core/clients/files.py +534 -0
  36. app/core/clients/headers.py +29 -0
  37. app/core/clients/http.py +511 -0
  38. app/core/clients/model_fetcher.py +276 -0
  39. app/core/clients/native_egress.py +1304 -0
  40. app/core/clients/oauth.py +477 -0
  41. app/core/clients/proxy.py +6089 -0
  42. app/core/clients/proxy_websocket.py +1411 -0
  43. app/core/clients/rate_limit_reset_credits.py +412 -0
  44. app/core/clients/stream_errors.py +12 -0
  45. app/core/clients/thread_cache_identity.py +347 -0
  46. app/core/clients/usage.py +467 -0
  47. app/core/clock.py +170 -0
  48. app/core/config/__init__.py +0 -0
  49. app/core/config/background_jobs.py +73 -0
  50. app/core/config/context_window_overrides.py +166 -0
  51. app/core/config/dashboard_overrides.py +145 -0
  52. app/core/config/inheritable.py +47 -0
  53. app/core/config/key_fingerprint.py +115 -0
  54. app/core/config/settings.py +872 -0
  55. app/core/config/settings_cache.py +100 -0
  56. app/core/config/spool_retention.py +217 -0
  57. app/core/config/tiers.py +174 -0
  58. app/core/conversation_archive.py +568 -0
  59. app/core/crypto.py +37 -0
  60. app/core/draining.py +11 -0
  61. app/core/errors.py +365 -0
  62. app/core/exceptions.py +155 -0
  63. app/core/handlers/__init__.py +3 -0
  64. app/core/handlers/exceptions.py +478 -0
  65. app/core/http_protocol.py +176 -0
  66. app/core/http_protocol_httptools.py +204 -0
  67. app/core/ingress_limits.py +17 -0
  68. app/core/metadata_store.py +18 -0
  69. app/core/metrics/__init__.py +0 -0
  70. app/core/metrics/middleware.py +74 -0
  71. app/core/metrics/prometheus.py +604 -0
  72. app/core/middleware/__init__.py +25 -0
  73. app/core/middleware/api_firewall.py +111 -0
  74. app/core/middleware/app_version.py +37 -0
  75. app/core/middleware/dashboard_auth_proxy.py +55 -0
  76. app/core/middleware/dashboard_csrf.py +177 -0
  77. app/core/middleware/dashboard_gzip.py +41 -0
  78. app/core/middleware/dashboard_overrides.py +59 -0
  79. app/core/middleware/firewall_cache.py +91 -0
  80. app/core/middleware/inflight.py +126 -0
  81. app/core/middleware/multipart_content_encoding.py +129 -0
  82. app/core/middleware/path_rewrite.py +148 -0
  83. app/core/middleware/request_body_limit.py +196 -0
  84. app/core/middleware/request_decompression.py +209 -0
  85. app/core/middleware/request_id.py +72 -0
  86. app/core/middleware/required_capability_http.py +116 -0
  87. app/core/middleware/trusted_proxy_headers.py +36 -0
  88. app/core/multipart.py +237 -0
  89. app/core/multipart_fields.py +91 -0
  90. app/core/openai/__init__.py +0 -0
  91. app/core/openai/chat_requests.py +493 -0
  92. app/core/openai/chat_responses.py +931 -0
  93. app/core/openai/contracts.py +46 -0
  94. app/core/openai/exceptions.py +24 -0
  95. app/core/openai/host_models.py +14 -0
  96. app/core/openai/images.py +413 -0
  97. app/core/openai/message_coercion.py +376 -0
  98. app/core/openai/model_refresh_scheduler.py +423 -0
  99. app/core/openai/model_registry.py +997 -0
  100. app/core/openai/model_registry_store.py +548 -0
  101. app/core/openai/models.py +172 -0
  102. app/core/openai/parsing.py +97 -0
  103. app/core/openai/requests.py +1980 -0
  104. app/core/openai/strict_schema.py +199 -0
  105. app/core/openai/tool_call_safety.py +99 -0
  106. app/core/openai/v1_requests.py +187 -0
  107. app/core/plan_types.py +90 -0
  108. app/core/prestop.py +202 -0
  109. app/core/rate_limiter/__init__.py +0 -0
  110. app/core/rate_limiter/db_rate_limiter.py +215 -0
  111. app/core/request_locality.py +473 -0
  112. app/core/resilience/__init__.py +0 -0
  113. app/core/resilience/backpressure.py +57 -0
  114. app/core/resilience/bulkhead.py +202 -0
  115. app/core/resilience/circuit_breaker.py +150 -0
  116. app/core/resilience/degradation.py +55 -0
  117. app/core/resilience/loop_lag_monitor.py +60 -0
  118. app/core/resilience/memory_monitor.py +137 -0
  119. app/core/resilience/network_recovery.py +256 -0
  120. app/core/resilience/overload.py +98 -0
  121. app/core/resilience/toggles.py +114 -0
  122. app/core/retention/__init__.py +0 -0
  123. app/core/retention/job.py +240 -0
  124. app/core/retention/scheduler.py +69 -0
  125. app/core/runtime_logging.py +601 -0
  126. app/core/scheduling/__init__.py +0 -0
  127. app/core/scheduling/leader_election.py +993 -0
  128. app/core/scheduling/leader_election_handle.py +25 -0
  129. app/core/server.py +140 -0
  130. app/core/shutdown.py +326 -0
  131. app/core/socket_peer.py +22 -0
  132. app/core/startup.py +41 -0
  133. app/core/timeout_invariants.py +347 -0
  134. app/core/tracing/__init__.py +0 -0
  135. app/core/tracing/otel.py +207 -0
  136. app/core/types.py +6 -0
  137. app/core/upstream_proxy/__init__.py +13 -0
  138. app/core/upstream_proxy/cache.py +130 -0
  139. app/core/upstream_proxy/resolver.py +189 -0
  140. app/core/upstream_proxy/types.py +104 -0
  141. app/core/usage/__init__.py +407 -0
  142. app/core/usage/depletion.py +109 -0
  143. app/core/usage/live_hub.py +41 -0
  144. app/core/usage/live_snapshots.py +174 -0
  145. app/core/usage/logs.py +212 -0
  146. app/core/usage/metadata_scheduler.py +118 -0
  147. app/core/usage/models.py +54 -0
  148. app/core/usage/pricing.py +576 -0
  149. app/core/usage/pricing_catalog.py +241 -0
  150. app/core/usage/pricing_snapshot.json +821 -0
  151. app/core/usage/quota.py +133 -0
  152. app/core/usage/refresh_policy.py +21 -0
  153. app/core/usage/refresh_scheduler.py +713 -0
  154. app/core/usage/reset_credits_refresh_scheduler.py +435 -0
  155. app/core/usage/types.py +168 -0
  156. app/core/utils/__init__.py +0 -0
  157. app/core/utils/json_guards.py +18 -0
  158. app/core/utils/locks.py +33 -0
  159. app/core/utils/masking.py +20 -0
  160. app/core/utils/proxy_env.py +104 -0
  161. app/core/utils/request_id.py +65 -0
  162. app/core/utils/retry.py +77 -0
  163. app/core/utils/shared_future.py +172 -0
  164. app/core/utils/sse.py +286 -0
  165. app/core/utils/time.py +24 -0
  166. app/db/__init__.py +0 -0
  167. app/db/account_identity_lock.py +56 -0
  168. app/db/alembic/__init__.py +1 -0
  169. app/db/alembic/env.py +70 -0
  170. app/db/alembic/revision_ids.py +43 -0
  171. app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
  172. app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
  173. app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
  174. app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
  175. app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
  176. app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
  177. app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
  178. app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
  179. app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
  180. app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
  181. app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
  182. app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
  183. app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
  184. app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
  185. app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
  186. app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
  187. app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
  188. app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
  189. app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
  190. app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
  191. app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
  192. app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
  193. app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
  194. app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
  195. app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
  196. app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
  197. app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
  198. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
  199. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
  200. app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
  201. app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
  202. app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
  203. app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
  204. app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
  205. app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
  206. app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
  207. app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
  208. app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
  209. app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
  210. app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
  211. app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
  212. app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
  213. app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
  214. app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
  215. app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
  216. app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
  217. app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
  218. app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
  219. app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
  220. app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
  221. app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
  222. app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
  223. app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
  224. app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
  225. app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
  226. app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
  227. app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
  228. app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
  229. app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
  230. app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
  231. app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
  232. app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
  233. app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
  234. app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
  235. app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
  236. app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
  237. app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
  238. app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
  239. app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
  240. app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
  241. app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
  242. app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
  243. app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
  244. app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
  245. app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
  246. app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
  247. app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
  248. app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
  249. app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
  250. app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
  251. app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
  252. app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
  253. app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
  254. app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
  255. app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
  256. app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
  257. app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
  258. app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
  259. app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
  260. app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
  261. app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
  262. app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
  263. app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
  264. app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
  265. app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
  266. app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
  267. app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
  268. app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
  269. app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
  270. app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
  271. app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
  272. app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
  273. app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
  274. app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
  275. app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
  276. app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
  277. app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
  278. app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
  279. app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
  280. app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
  281. app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
  282. app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
  283. app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
  284. app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
  285. app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
  286. app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
  287. app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
  288. app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
  289. app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
  290. app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
  291. app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
  292. app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
  293. app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
  294. app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
  295. app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
  296. app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
  297. app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
  298. app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
  299. app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
  300. app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
  301. app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
  302. app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
  303. app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
  304. app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
  305. app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
  306. app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
  307. app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
  308. app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
  309. app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
  310. app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
  311. app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
  312. app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
  313. app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
  314. app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
  315. app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
  316. app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
  317. app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
  318. app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
  319. app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
  320. app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
  321. app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
  322. app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
  323. app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
  324. app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
  325. app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
  326. app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
  327. app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
  328. app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
  329. app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
  330. app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
  331. app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
  332. app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
  333. app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
  334. app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
  335. app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
  336. app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
  337. app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
  338. app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
  339. app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
  340. app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
  341. app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
  342. app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
  343. app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
  344. app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
  345. app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
  346. app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
  347. app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
  348. app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
  349. app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
  350. app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
  351. app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
  352. app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
  353. app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
  354. app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
  355. app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
  356. app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
  357. app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
  358. app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
  359. app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
  360. app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
  361. app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
  362. app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
  363. app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
  364. app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
  365. app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
  366. app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
  367. app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
  368. app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
  369. app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
  370. app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
  371. app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
  372. app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
  373. app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
  374. app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
  375. app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
  376. app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
  377. app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
  378. app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
  379. app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
  380. app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
  381. app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
  382. app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
  383. app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
  384. app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
  385. app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
  386. app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
  387. app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
  388. app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
  389. app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
  390. app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
  391. app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
  392. app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
  393. app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
  394. app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
  395. app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
  396. app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
  397. app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
  398. app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
  399. app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
  400. app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
  401. app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
  402. app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
  403. app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
  404. app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
  405. app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
  406. app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
  407. app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
  408. app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
  409. app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
  410. app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
  411. app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
  412. app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
  413. app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
  414. app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
  415. app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
  416. app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
  417. app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
  418. app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
  419. app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
  420. app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
  421. app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
  422. app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
  423. app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
  424. app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
  425. app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
  426. app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
  427. app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
  428. app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
  429. app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
  430. app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
  431. app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
  432. app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
  433. app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
  434. app/db/alembic/versions/__init__.py +1 -0
  435. app/db/backup.py +64 -0
  436. app/db/migrate.py +1249 -0
  437. app/db/migration_lock.py +160 -0
  438. app/db/migration_url.py +18 -0
  439. app/db/models.py +3100 -0
  440. app/db/recover.py +244 -0
  441. app/db/session.py +1295 -0
  442. app/db/snapshot.py +46 -0
  443. app/db/sqlite_lock_retry.py +190 -0
  444. app/db/sqlite_utils.py +549 -0
  445. app/dependencies.py +462 -0
  446. app/main.py +1240 -0
  447. app/modules/__init__.py +0 -0
  448. app/modules/accounts/__init__.py +0 -0
  449. app/modules/accounts/api.py +426 -0
  450. app/modules/accounts/auth_manager.py +1439 -0
  451. app/modules/accounts/background_repository.py +160 -0
  452. app/modules/accounts/deletion.py +540 -0
  453. app/modules/accounts/mappers.py +654 -0
  454. app/modules/accounts/refresh_claims.py +352 -0
  455. app/modules/accounts/repository.py +1679 -0
  456. app/modules/accounts/schemas.py +308 -0
  457. app/modules/accounts/service.py +809 -0
  458. app/modules/accounts/usage_rollup.py +486 -0
  459. app/modules/accounts/usage_rollup_scheduler.py +75 -0
  460. app/modules/accounts/usage_time_rollup.py +1037 -0
  461. app/modules/accounts/usage_time_rollup_read.py +522 -0
  462. app/modules/api_keys/__init__.py +1 -0
  463. app/modules/api_keys/api.py +344 -0
  464. app/modules/api_keys/last_used_coalescer.py +216 -0
  465. app/modules/api_keys/limit_windows.py +41 -0
  466. app/modules/api_keys/repository.py +1200 -0
  467. app/modules/api_keys/reset_scheduler.py +85 -0
  468. app/modules/api_keys/schemas.py +135 -0
  469. app/modules/api_keys/service.py +2123 -0
  470. app/modules/audit/__init__.py +0 -0
  471. app/modules/audit/api.py +88 -0
  472. app/modules/audit/repository.py +74 -0
  473. app/modules/audit/schemas.py +33 -0
  474. app/modules/audit/service.py +71 -0
  475. app/modules/auth_providers/__init__.py +0 -0
  476. app/modules/auth_providers/api.py +127 -0
  477. app/modules/auth_providers/config.py +343 -0
  478. app/modules/auth_providers/repository.py +101 -0
  479. app/modules/auth_providers/schemas.py +88 -0
  480. app/modules/auth_providers/seed.py +105 -0
  481. app/modules/auth_providers/service.py +289 -0
  482. app/modules/automations/__init__.py +1 -0
  483. app/modules/automations/api.py +369 -0
  484. app/modules/automations/repository.py +1896 -0
  485. app/modules/automations/scheduler.py +107 -0
  486. app/modules/automations/schemas.py +158 -0
  487. app/modules/automations/service.py +2548 -0
  488. app/modules/cache_isolation_probe/__init__.py +0 -0
  489. app/modules/cache_isolation_probe/api.py +188 -0
  490. app/modules/cache_isolation_probe/prefix.py +144 -0
  491. app/modules/cache_isolation_probe/schemas.py +94 -0
  492. app/modules/cache_isolation_probe/sender.py +229 -0
  493. app/modules/cache_isolation_probe/service.py +525 -0
  494. app/modules/conversation_archive/__init__.py +1 -0
  495. app/modules/conversation_archive/api.py +101 -0
  496. app/modules/conversation_archive/schemas.py +28 -0
  497. app/modules/conversation_archive/service.py +165 -0
  498. app/modules/dashboard/__init__.py +1 -0
  499. app/modules/dashboard/api.py +106 -0
  500. app/modules/dashboard/builders.py +107 -0
  501. app/modules/dashboard/repository.py +304 -0
  502. app/modules/dashboard/schemas.py +148 -0
  503. app/modules/dashboard/service.py +534 -0
  504. app/modules/dashboard/timeframes.py +19 -0
  505. app/modules/dashboard/weekly_pace.py +617 -0
  506. app/modules/dashboard_auth/__init__.py +1 -0
  507. app/modules/dashboard_auth/api.py +1344 -0
  508. app/modules/dashboard_auth/oidc_api.py +753 -0
  509. app/modules/dashboard_auth/oidc_flows.py +330 -0
  510. app/modules/dashboard_auth/repository.py +452 -0
  511. app/modules/dashboard_auth/schemas.py +232 -0
  512. app/modules/dashboard_auth/service.py +1384 -0
  513. app/modules/dashboard_roles/__init__.py +0 -0
  514. app/modules/dashboard_roles/api.py +64 -0
  515. app/modules/dashboard_roles/repository.py +54 -0
  516. app/modules/dashboard_roles/schemas.py +29 -0
  517. app/modules/dashboard_roles/seed.py +80 -0
  518. app/modules/dashboard_roles/service.py +142 -0
  519. app/modules/dashboard_users/__init__.py +0 -0
  520. app/modules/dashboard_users/api.py +278 -0
  521. app/modules/dashboard_users/break_glass.py +169 -0
  522. app/modules/dashboard_users/credentials.py +18 -0
  523. app/modules/dashboard_users/identity_resolver.py +665 -0
  524. app/modules/dashboard_users/repository.py +725 -0
  525. app/modules/dashboard_users/schemas.py +117 -0
  526. app/modules/dashboard_users/service.py +1054 -0
  527. app/modules/firewall/__init__.py +1 -0
  528. app/modules/firewall/api.py +78 -0
  529. app/modules/firewall/repository.py +51 -0
  530. app/modules/firewall/schemas.py +28 -0
  531. app/modules/firewall/service.py +93 -0
  532. app/modules/fleet/__init__.py +1 -0
  533. app/modules/fleet/api.py +177 -0
  534. app/modules/fleet/mappers.py +56 -0
  535. app/modules/fleet/observability.py +372 -0
  536. app/modules/fleet/schemas.py +119 -0
  537. app/modules/health/__init__.py +0 -0
  538. app/modules/health/api.py +246 -0
  539. app/modules/health/schemas.py +27 -0
  540. app/modules/limit_warmup/__init__.py +1 -0
  541. app/modules/limit_warmup/repository.py +159 -0
  542. app/modules/limit_warmup/service.py +1001 -0
  543. app/modules/model_sources/__init__.py +1 -0
  544. app/modules/model_sources/api.py +99 -0
  545. app/modules/model_sources/catalog.py +307 -0
  546. app/modules/model_sources/forwarding.py +1435 -0
  547. app/modules/model_sources/projection.py +55 -0
  548. app/modules/model_sources/repository.py +201 -0
  549. app/modules/model_sources/schemas.py +79 -0
  550. app/modules/model_sources/selection.py +166 -0
  551. app/modules/model_sources/service.py +236 -0
  552. app/modules/oauth/__init__.py +0 -0
  553. app/modules/oauth/api.py +101 -0
  554. app/modules/oauth/repository.py +238 -0
  555. app/modules/oauth/schemas.py +49 -0
  556. app/modules/oauth/service.py +1196 -0
  557. app/modules/oauth/templates/oauth_success.html +122 -0
  558. app/modules/proxy/__init__.py +0 -0
  559. app/modules/proxy/_load_balancer/__init__.py +1 -0
  560. app/modules/proxy/_load_balancer/error_rate.py +116 -0
  561. app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
  562. app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
  563. app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
  564. app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
  565. app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
  566. app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
  567. app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
  568. app/modules/proxy/_load_balancer/tunables.py +97 -0
  569. app/modules/proxy/_load_balancer/types.py +96 -0
  570. app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
  571. app/modules/proxy/_service/__init__.py +6 -0
  572. app/modules/proxy/_service/api_key_usage.py +745 -0
  573. app/modules/proxy/_service/clock_budget.py +38 -0
  574. app/modules/proxy/_service/codex_control.py +611 -0
  575. app/modules/proxy/_service/compact.py +2211 -0
  576. app/modules/proxy/_service/file_ops.py +749 -0
  577. app/modules/proxy/_service/http_bridge/__init__.py +3 -0
  578. app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
  579. app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
  580. app/modules/proxy/_service/http_bridge/activity.py +178 -0
  581. app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
  582. app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
  583. app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
  584. app/modules/proxy/_service/http_bridge/protocol.py +167 -0
  585. app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
  586. app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
  587. app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
  588. app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
  589. app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
  590. app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
  591. app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
  592. app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
  593. app/modules/proxy/_service/observability.py +410 -0
  594. app/modules/proxy/_service/rate_limit.py +361 -0
  595. app/modules/proxy/_service/realtime_live.py +655 -0
  596. app/modules/proxy/_service/refresh.py +154 -0
  597. app/modules/proxy/_service/request_log.py +585 -0
  598. app/modules/proxy/_service/response_create.py +930 -0
  599. app/modules/proxy/_service/streaming/__init__.py +3 -0
  600. app/modules/proxy/_service/streaming/helpers.py +1317 -0
  601. app/modules/proxy/_service/streaming/mixin.py +1100 -0
  602. app/modules/proxy/_service/streaming/protocol.py +32 -0
  603. app/modules/proxy/_service/streaming/retry.py +3762 -0
  604. app/modules/proxy/_service/support.py +2122 -0
  605. app/modules/proxy/_service/transcribe.py +434 -0
  606. app/modules/proxy/_service/warmup.py +483 -0
  607. app/modules/proxy/_service/websocket/__init__.py +3 -0
  608. app/modules/proxy/_service/websocket/helpers.py +2360 -0
  609. app/modules/proxy/_service/websocket/mixin.py +7285 -0
  610. app/modules/proxy/_service/websocket/protocol.py +74 -0
  611. app/modules/proxy/account_cache.py +229 -0
  612. app/modules/proxy/account_eligibility.py +87 -0
  613. app/modules/proxy/additional_model_limits.py +40 -0
  614. app/modules/proxy/affinity.py +830 -0
  615. app/modules/proxy/affinity_observation.py +101 -0
  616. app/modules/proxy/api.py +10254 -0
  617. app/modules/proxy/api_key_usage.py +113 -0
  618. app/modules/proxy/cap_partitioning.py +280 -0
  619. app/modules/proxy/capability_lineage.py +65 -0
  620. app/modules/proxy/capability_lineage_repository.py +82 -0
  621. app/modules/proxy/capability_routing.py +253 -0
  622. app/modules/proxy/continuity.py +112 -0
  623. app/modules/proxy/downstream_delivery.py +277 -0
  624. app/modules/proxy/durable_bridge_coordinator.py +1217 -0
  625. app/modules/proxy/durable_bridge_repository.py +4516 -0
  626. app/modules/proxy/durable_bridge_runtime.py +9 -0
  627. app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
  628. app/modules/proxy/fair_share.py +131 -0
  629. app/modules/proxy/file_pin_repository.py +210 -0
  630. app/modules/proxy/helpers.py +439 -0
  631. app/modules/proxy/http_bridge_event_batcher.py +613 -0
  632. app/modules/proxy/http_bridge_forwarding.py +797 -0
  633. app/modules/proxy/http_continuation.py +76 -0
  634. app/modules/proxy/images_observability.py +62 -0
  635. app/modules/proxy/images_service.py +1054 -0
  636. app/modules/proxy/load_balancer.py +3016 -0
  637. app/modules/proxy/rate_limit_cache.py +46 -0
  638. app/modules/proxy/replay_safety.py +1063 -0
  639. app/modules/proxy/repo_bundle.py +31 -0
  640. app/modules/proxy/request_policy.py +1054 -0
  641. app/modules/proxy/ring_membership.py +251 -0
  642. app/modules/proxy/schemas.py +345 -0
  643. app/modules/proxy/selection_errors.py +41 -0
  644. app/modules/proxy/service.py +2600 -0
  645. app/modules/proxy/source_admission.py +125 -0
  646. app/modules/proxy/source_dispatch.py +999 -0
  647. app/modules/proxy/sticky_repository.py +831 -0
  648. app/modules/proxy/thread_anchors.py +491 -0
  649. app/modules/proxy/tool_call_dedupe.py +802 -0
  650. app/modules/proxy/types.py +54 -0
  651. app/modules/proxy/work_admission.py +121 -0
  652. app/modules/quota_planner/__init__.py +1 -0
  653. app/modules/quota_planner/api.py +276 -0
  654. app/modules/quota_planner/logic.py +743 -0
  655. app/modules/quota_planner/repository.py +722 -0
  656. app/modules/quota_planner/scheduler.py +212 -0
  657. app/modules/quota_planner/schemas.py +102 -0
  658. app/modules/quota_planner/warmup.py +693 -0
  659. app/modules/rate_limit_reset_credits/__init__.py +0 -0
  660. app/modules/rate_limit_reset_credits/api.py +601 -0
  661. app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
  662. app/modules/rate_limit_reset_credits/store.py +114 -0
  663. app/modules/reports/__init__.py +1 -0
  664. app/modules/reports/api.py +134 -0
  665. app/modules/reports/cache.py +74 -0
  666. app/modules/reports/filters.py +27 -0
  667. app/modules/reports/repository.py +525 -0
  668. app/modules/reports/rollup.py +150 -0
  669. app/modules/reports/rollup_read.py +148 -0
  670. app/modules/reports/schemas.py +123 -0
  671. app/modules/reports/service.py +287 -0
  672. app/modules/reports/thread_identity.py +256 -0
  673. app/modules/request_logs/__init__.py +0 -0
  674. app/modules/request_logs/api.py +224 -0
  675. app/modules/request_logs/cost_backfill.py +154 -0
  676. app/modules/request_logs/mappers.py +97 -0
  677. app/modules/request_logs/repository.py +1765 -0
  678. app/modules/request_logs/schemas.py +145 -0
  679. app/modules/request_logs/service.py +348 -0
  680. app/modules/role_mappings/__init__.py +0 -0
  681. app/modules/role_mappings/api.py +206 -0
  682. app/modules/role_mappings/matching.py +55 -0
  683. app/modules/role_mappings/repository.py +106 -0
  684. app/modules/role_mappings/schemas.py +67 -0
  685. app/modules/role_mappings/service.py +334 -0
  686. app/modules/runtime/__init__.py +1 -0
  687. app/modules/runtime/api.py +18 -0
  688. app/modules/runtime/schemas.py +14 -0
  689. app/modules/runtime/service.py +218 -0
  690. app/modules/scim/__init__.py +1 -0
  691. app/modules/scim/api.py +194 -0
  692. app/modules/scim/dependencies.py +112 -0
  693. app/modules/scim/errors.py +77 -0
  694. app/modules/scim/management_api.py +156 -0
  695. app/modules/scim/repository.py +212 -0
  696. app/modules/scim/schemas.py +145 -0
  697. app/modules/scim/service.py +786 -0
  698. app/modules/scim/tokens.py +43 -0
  699. app/modules/settings/__init__.py +1 -0
  700. app/modules/settings/api.py +1677 -0
  701. app/modules/settings/repository.py +606 -0
  702. app/modules/settings/schemas.py +496 -0
  703. app/modules/settings/service.py +769 -0
  704. app/modules/shared/__init__.py +0 -0
  705. app/modules/shared/schemas.py +22 -0
  706. app/modules/sticky_sessions/__init__.py +1 -0
  707. app/modules/sticky_sessions/api.py +118 -0
  708. app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
  709. app/modules/sticky_sessions/schemas.py +78 -0
  710. app/modules/sticky_sessions/service.py +196 -0
  711. app/modules/telemetry/__init__.py +5 -0
  712. app/modules/telemetry/api.py +115 -0
  713. app/modules/telemetry/clients.py +62 -0
  714. app/modules/telemetry/consent.py +115 -0
  715. app/modules/telemetry/scheduler.py +85 -0
  716. app/modules/telemetry/schemas.py +163 -0
  717. app/modules/telemetry/sender.py +221 -0
  718. app/modules/telemetry/snapshot.py +490 -0
  719. app/modules/usage/__init__.py +0 -0
  720. app/modules/usage/additional_quota_keys.py +379 -0
  721. app/modules/usage/api.py +45 -0
  722. app/modules/usage/background_repository.py +147 -0
  723. app/modules/usage/builders.py +559 -0
  724. app/modules/usage/depletion_service.py +277 -0
  725. app/modules/usage/live_ingest.py +356 -0
  726. app/modules/usage/mappers.py +29 -0
  727. app/modules/usage/plan_downgrade_observations.py +488 -0
  728. app/modules/usage/repository.py +1890 -0
  729. app/modules/usage/schemas.py +68 -0
  730. app/modules/usage/service.py +102 -0
  731. app/modules/usage/updater.py +1488 -0
  732. app/static/assets/Trans-Cj-J9mVA.js +1 -0
  733. app/static/assets/access-page-DHzZiXZi.js +1 -0
  734. app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
  735. app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
  736. app/static/assets/account-status-DIsU5sUS.js +1 -0
  737. app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
  738. app/static/assets/accounts-page-Bbai8ift.js +3 -0
  739. app/static/assets/api-Dwa7XxPe.js +1 -0
  740. app/static/assets/api-client-dugX1Pti.js +55 -0
  741. app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
  742. app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
  743. app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
  744. app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
  745. app/static/assets/apis-page-BvXz9c6i.js +2 -0
  746. app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
  747. app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
  748. app/static/assets/automations-page-DQgNAJOR.js +3 -0
  749. app/static/assets/badge-DqMb9es0.js +1 -0
  750. app/static/assets/blur-email-HEvKagG2.js +1 -0
  751. app/static/assets/button-BjVmmiGW.js +1 -0
  752. app/static/assets/checkbox-apsl9vzk.js +1 -0
  753. app/static/assets/chevron-left-1d_pHVa5.js +1 -0
  754. app/static/assets/circle-alert-BS0xd_o1.js +1 -0
  755. app/static/assets/clsx-DB0hHKMi.js +1 -0
  756. app/static/assets/colors-BBy8GEtr.js +1 -0
  757. app/static/assets/copy-button-D7nAvOKR.js +1 -0
  758. app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
  759. app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
  760. app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
  761. app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
  762. app/static/assets/donut-chart-Bxc69cra.js +1 -0
  763. app/static/assets/download-kQteIcsB.js +1 -0
  764. app/static/assets/ellipsis-B_5OgIre.js +1 -0
  765. app/static/assets/empty-state-JcA7tc8m.js +1 -0
  766. app/static/assets/es2015-bMH8YKZT.js +41 -0
  767. app/static/assets/es6-B264ni6W.js +69 -0
  768. app/static/assets/external-link-UVAOgMQ0.js +1 -0
  769. app/static/assets/i18n-BWqZ9Ua2.js +1 -0
  770. app/static/assets/inbox-B_RM2s1J.js +1 -0
  771. app/static/assets/index-DIpd264Z.js +23 -0
  772. app/static/assets/index-DMeLKRJx.css +2 -0
  773. app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
  774. app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
  775. app/static/assets/iso-Bpe5Ihbb.js +1 -0
  776. app/static/assets/label-CGaLmLNf.js +1 -0
  777. app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
  778. app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
  779. app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
  780. app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
  781. app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
  782. app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
  783. app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
  784. app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
  785. app/static/assets/payload-CIYnGZz9.js +1 -0
  786. app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
  787. app/static/assets/report-chart-card-BXDliwcc.js +1 -0
  788. app/static/assets/reports-page-CGCJeMHG.js +3 -0
  789. app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
  790. app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
  791. app/static/assets/search-HfhqwERt.js +1 -0
  792. app/static/assets/select-Cl33bz-g.js +1 -0
  793. app/static/assets/settings-page-mZj01HYl.js +2 -0
  794. app/static/assets/shim-CZwyM4Q7.js +1 -0
  795. app/static/assets/skeleton-C_EUc4Rp.js +1 -0
  796. app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
  797. app/static/assets/status-badge-CHHA5X6c.js +1 -0
  798. app/static/assets/switch-CzxgokcM.js +1 -0
  799. app/static/assets/table-DXaNSQ0G.js +1 -0
  800. app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
  801. app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
  802. app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
  803. app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
  804. app/static/assets/trash-D3KyWr1h.js +1 -0
  805. app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
  806. app/static/assets/use-accounts-PWlVGw3z.js +1 -0
  807. app/static/assets/use-auth-CuDlW3iC.js +1 -0
  808. app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
  809. app/static/assets/use-model-sources-Bul24JBu.js +1 -0
  810. app/static/assets/use-models-DUCPUuZg.js +1 -0
  811. app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
  812. app/static/assets/useTranslation--56nHhlH.js +1 -0
  813. app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
  814. app/static/assets/users-uIC3TIlE.js +1 -0
  815. app/static/assets/vendor-query-_bHZuxEx.js +1 -0
  816. app/static/assets/vendor-react-BVL0YZPW.js +9 -0
  817. app/static/favicon.svg +10 -0
  818. app/static/fonts/GeistSans-Variable.woff2 +0 -0
  819. app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
  820. app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
  821. app/static/index.html +29 -0
  822. codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
  823. codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
  824. codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
  825. codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
  826. codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
  827. config/additional_quota_registry.json +11 -0
@@ -0,0 +1,3762 @@
1
+ # pyright: reportGeneralTypeIssues=false
2
+ from __future__ import annotations
3
+
4
+ import asyncio
5
+ import json
6
+ import logging
7
+ import sys
8
+ from dataclasses import replace
9
+ from typing import Any, AsyncGenerator, AsyncIterator, Mapping, cast
10
+
11
+ import aiohttp
12
+
13
+ from app.core.auth.refresh import RefreshError, is_transient_refresh_contention, refresh_contention_kind
14
+ from app.core.balancer import failover_decision
15
+ from app.core.balancer.logic import (
16
+ BURST_SAME_ACCOUNT_MAX_RETRIES,
17
+ BURST_SURFACE_RETRY_AFTER_SECONDS,
18
+ burst_same_account_backoff_seconds,
19
+ )
20
+ from app.core.balancer.types import ClassifiedFailure, UpstreamError
21
+ from app.core.clients.proxy import (
22
+ ProxyResponseError,
23
+ _is_native_codex_request,
24
+ _resolve_stream_transport,
25
+ is_confirmed_pre_dispatch_transport_error,
26
+ pop_stream_timeout_overrides,
27
+ )
28
+ from app.core.clients.thread_cache_identity import (
29
+ ThreadCacheIdentity,
30
+ cache_scope_payload_overhead_bytes,
31
+ )
32
+ from app.core.clients.thread_cache_identity import (
33
+ effective_thread_cache_identity_mode as _effective_thread_cache_identity_mode,
34
+ )
35
+ from app.core.clock import REAL_CLOCK, REAL_SCHEDULER, Clock, Scheduler, clock_for, scheduler_for
36
+ from app.core.config.dashboard_overrides import with_dashboard_overrides
37
+ from app.core.errors import (
38
+ SYNTHETIC_TRANSPORT_FAILURE_CODES,
39
+ openai_error,
40
+ synthetic_transport_failure_event,
41
+ )
42
+ from app.core.errors import (
43
+ synthetic_stream_failure_event as response_failed_event,
44
+ )
45
+ from app.core.openai.requests import ResponsesRequest, extract_input_file_ids
46
+ from app.core.resilience.network_recovery import (
47
+ NetworkRecoveryDecision,
48
+ ProcessNetworkRecovery,
49
+ )
50
+ from app.core.resilience.toggles import bind_resilience_toggles, set_resilience_toggles
51
+ from app.core.upstream_proxy import UpstreamProxyRouteError
52
+ from app.core.utils.request_id import ensure_request_id
53
+ from app.core.utils.retry import backoff_seconds
54
+ from app.core.utils.shared_future import _await_task_deferring_cancellation
55
+ from app.core.utils.sse import format_sse_event
56
+ from app.db.models import Account
57
+ from app.modules.api_keys.service import ApiKeyData, ApiKeyUsageReservationData
58
+ from app.modules.proxy._load_balancer.overload_backoff import (
59
+ UPSTREAM_OVERLOAD_CODES,
60
+ record_upstream_burst_rejection,
61
+ )
62
+ from app.modules.proxy._service.observability import (
63
+ _maybe_log_proxy_request_shape,
64
+ _record_continuity_fail_closed,
65
+ _record_upstream_transport_decision,
66
+ record_http_bridge_routing,
67
+ )
68
+ from app.modules.proxy._service.streaming.protocol import _StreamingServiceProtocol
69
+ from app.modules.proxy._service.support import (
70
+ _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE,
71
+ _ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS,
72
+ _LOCAL_ACCOUNT_CAP_ERROR_CODES,
73
+ _account_capacity_wait_payload,
74
+ _account_selection_recovery_sleep_seconds,
75
+ _request_log_client_fields,
76
+ _RetryableStreamError,
77
+ _signal_propagated_capacity_startup_wait,
78
+ _signal_propagated_responses_service_cleanup_ready,
79
+ _stream_settlement_error_payload,
80
+ _StreamSettlement,
81
+ _TerminalStreamError,
82
+ _TransientStreamError,
83
+ _WebSocketUpstreamControl,
84
+ configured_upstream_stream_transport,
85
+ upstream_websocket_transport_recently_failed,
86
+ )
87
+ from app.modules.proxy._service.websocket.helpers import (
88
+ _websocket_input_items_are_self_contained_fresh_replay,
89
+ )
90
+ from app.modules.proxy.affinity import (
91
+ _is_synthesized_turn_state,
92
+ _owner_lookup_session_id_from_headers,
93
+ _prompt_cache_key_from_request_model,
94
+ _sticky_key_for_responses_request,
95
+ _sticky_key_from_turn_state_header,
96
+ _websocket_continuity_key_from_headers,
97
+ )
98
+ from app.modules.proxy.affinity_observation import AffinityObservation
99
+ from app.modules.proxy.api_key_usage import estimate_api_key_request_usage
100
+ from app.modules.proxy.continuity import resolve_required_account_id
101
+ from app.modules.proxy.helpers import (
102
+ _apply_error_metadata,
103
+ _is_account_model_unsupported_error,
104
+ _normalize_error_code,
105
+ _parse_openai_error,
106
+ _upstream_error_from_openai,
107
+ classify_upstream_failure,
108
+ is_upstream_burst_rejection,
109
+ is_upstream_model_capacity_error,
110
+ )
111
+ from app.modules.proxy.http_continuation import http_continuation_signal
112
+ from app.modules.proxy.load_balancer import AccountLease, AccountSelection
113
+ from app.modules.proxy.replay_safety import responses_payload_is_account_neutral_fresh_replay
114
+ from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response
115
+
116
+ _REQUEST_TRANSPORT_HTTP = "http"
117
+ _REQUEST_TRANSPORT_WEBSOCKET = "websocket"
118
+ _HTTP_DOWNSTREAM_TRANSPORT_POLICY_DEFAULT = "smart"
119
+ _HTTP_DOWNSTREAM_TRANSPORT_POLICIES = frozenset({"smart", "always_http", "always_websocket", "pinned"})
120
+ logger = logging.getLogger(__name__)
121
+
122
+
123
+ def _facade() -> Any:
124
+ return sys.modules["app.modules.proxy.service"]
125
+
126
+
127
+ def _http_downstream_request_is_sticky(payload: ResponsesRequest, headers: Mapping[str, str]) -> bool:
128
+ return http_continuation_signal(payload, headers) is not None
129
+
130
+
131
+ _POST_REFRESH_TRANSIENT_EXHAUSTED_ATTR = "_codex_lb_post_refresh_transient_exhausted"
132
+ # Set on a surfaced pre-visible ``ProxyResponseError`` whose account health was
133
+ # already written where the failover decision was made, so the outer terminal
134
+ # handler does not write it a second time. Exactly-once matters most for an
135
+ # exhausted owner-bound burst 429 (its same-account retries engage only the
136
+ # replica-local burst cooldown, never ``record_error``), but every surfaced
137
+ # pre-visible failure is penalized once by construction.
138
+ _STREAM_HEALTH_RECORDED_ATTR = "_codex_lb_stream_health_recorded"
139
+
140
+
141
+ def _resolve_http_downstream_transport(policy: str, *, payload: ResponsesRequest, headers: Mapping[str, str]) -> str:
142
+ normalized_policy = policy.strip().lower()
143
+ if normalized_policy not in _HTTP_DOWNSTREAM_TRANSPORT_POLICIES:
144
+ raise ValueError(f"Unsupported HTTP downstream transport policy: {policy}")
145
+ if normalized_policy in ("always_http", "pinned"):
146
+ return "http"
147
+ if normalized_policy == "always_websocket":
148
+ return "websocket"
149
+ return "websocket" if _http_downstream_request_is_sticky(payload, headers) else "http"
150
+
151
+
152
+ def _verified_cross_transport_fresh_replay(
153
+ proxy: _StreamingServiceProtocol,
154
+ *,
155
+ payload: ResponsesRequest,
156
+ headers: Mapping[str, str],
157
+ api_key: ApiKeyData | None,
158
+ ) -> ResponsesRequest | None:
159
+ """Return an unanchored body only when local WS continuity proves its prefix."""
160
+ previous_response_id = payload.previous_response_id
161
+ input_value = payload.input
162
+ if previous_response_id is None or not isinstance(input_value, list):
163
+ return None
164
+ if extract_input_file_ids(input_value):
165
+ return None
166
+ input_items = cast(list[Any], input_value)
167
+ if not _websocket_input_items_are_self_contained_fresh_replay(input_items):
168
+ return None
169
+ session_id = _websocket_continuity_key_from_headers(headers)
170
+ if session_id is None:
171
+ return None
172
+ api_key_id = api_key.id if api_key is not None else None
173
+ continuity_state = proxy._websocket_continuity_index.get((session_id, api_key_id))
174
+ if continuity_state is None or continuity_state.last_completed_response_id != previous_response_id:
175
+ # HTTP and WebSocket entry points synthesize different turn-state
176
+ # headers. The response id remains globally specific within the API-key
177
+ # scope, so use its unique retained state when the direct key differs.
178
+ matching_states = [
179
+ state
180
+ for (_continuity_key, continuity_api_key_id), state in proxy._websocket_continuity_index.items()
181
+ if continuity_api_key_id == api_key_id and state.last_completed_response_id == previous_response_id
182
+ ]
183
+ if len(matching_states) != 1:
184
+ return None
185
+ continuity_state = matching_states[0]
186
+ if not _facade()._input_prefix_matches_stored_context(
187
+ input_value,
188
+ stored_count=continuity_state.last_completed_input_count,
189
+ stored_fingerprint=continuity_state.last_completed_input_prefix_fingerprint,
190
+ ):
191
+ return None
192
+ fresh_payload = payload.model_copy(update={"previous_response_id": None})
193
+ if not responses_payload_is_account_neutral_fresh_replay(fresh_payload.to_replay_safety_payload()):
194
+ return None
195
+ return fresh_payload
196
+
197
+
198
+ def _effective_http_downstream_transport_policy(
199
+ api_key: ApiKeyData | None,
200
+ dashboard_settings: Any,
201
+ ) -> tuple[str, bool]:
202
+ override = getattr(api_key, "transport_policy_override", None) if api_key is not None else None
203
+ if override is not None:
204
+ return override, True
205
+ dashboard_policy = getattr(dashboard_settings, "http_downstream_transport_policy", None)
206
+ if isinstance(dashboard_policy, str) and dashboard_policy:
207
+ return dashboard_policy, False
208
+ return _HTTP_DOWNSTREAM_TRANSPORT_POLICY_DEFAULT, False
209
+
210
+
211
+ def _resolved_configured_stream_transport(dashboard_settings: Any) -> tuple[str, bool]:
212
+ configured = configured_upstream_stream_transport(dashboard_settings)
213
+ return configured, configured in ("http", "websocket")
214
+
215
+
216
+ def _http_bridge_allowed_by_transport_policy(
217
+ payload: ResponsesRequest,
218
+ headers: Mapping[str, str],
219
+ *,
220
+ api_key: ApiKeyData | None,
221
+ dashboard_settings: Any,
222
+ base_settings: Any,
223
+ ) -> bool:
224
+ """Apply ordinary HTTP transport precedence before entering the WS bridge."""
225
+
226
+ configured_transport, explicit_transport = _resolved_configured_stream_transport(dashboard_settings)
227
+ if upstream_websocket_transport_recently_failed():
228
+ reason, allowed = "recent_ws_failure", False
229
+ elif explicit_transport:
230
+ reason, allowed = f"explicit_{configured_transport}", configured_transport == "websocket"
231
+ else:
232
+ policy, _override_applied = _effective_http_downstream_transport_policy(api_key, dashboard_settings)
233
+ allowed = _resolve_http_downstream_transport(policy, payload=payload, headers=headers) == "websocket"
234
+ signal = http_continuation_signal(payload, headers)
235
+ reason = f"smart_{signal or 'single_turn'}" if policy == "smart" else policy
236
+ record_http_bridge_routing(stage="admission", reason=reason)
237
+ return allowed
238
+
239
+
240
+ async def _iter_account_capacity_recovery_wait(
241
+ *,
242
+ request_id: str,
243
+ model: str | None,
244
+ account_id: str | None,
245
+ error_message: str | None,
246
+ recovery_sleep_seconds: float,
247
+ remaining_budget_seconds: float,
248
+ emit_keepalives: bool,
249
+ stage: str,
250
+ scheduler: Scheduler = REAL_SCHEDULER,
251
+ clock: Clock = REAL_CLOCK,
252
+ ) -> AsyncIterator[str]:
253
+ # ``remaining_budget_seconds`` is sampled by the caller through the owner's
254
+ # budget seam (``proxy._remaining_budget_seconds``), so this owner-less
255
+ # helper never reads a clock of its own for budget math; ``clock`` only
256
+ # stamps the wait start and ``scheduler`` owns the heartbeat sleeps.
257
+ if not emit_keepalives:
258
+ _signal_propagated_capacity_startup_wait()
259
+ if remaining_budget_seconds <= 0:
260
+ return
261
+ wait_started_at = clock.monotonic()
262
+ remaining_sleep_seconds = min(recovery_sleep_seconds, remaining_budget_seconds)
263
+ _facade().logger.info(
264
+ "Waiting for account capacity before retrying stream request_id=%s model=%s account_id=%s "
265
+ "stage=%s sleep_seconds=%.1f recovery_hint_seconds=%.1f error=%s",
266
+ request_id,
267
+ model,
268
+ account_id,
269
+ stage,
270
+ remaining_sleep_seconds,
271
+ recovery_sleep_seconds,
272
+ error_message,
273
+ )
274
+ while remaining_sleep_seconds > 0:
275
+ if emit_keepalives:
276
+ yield format_sse_event(
277
+ cast(
278
+ Mapping[str, Any],
279
+ _account_capacity_wait_payload(
280
+ None,
281
+ request_id=request_id,
282
+ reason=error_message,
283
+ retry_after_seconds=remaining_sleep_seconds,
284
+ started_at=wait_started_at,
285
+ now=clock.monotonic(),
286
+ ),
287
+ )
288
+ )
289
+ chunk_seconds = min(
290
+ remaining_sleep_seconds,
291
+ _ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS,
292
+ )
293
+ await scheduler.sleep(chunk_seconds)
294
+ remaining_sleep_seconds -= chunk_seconds
295
+
296
+
297
+ def _retry_after_kwargs(retry_after_seconds: float | None) -> dict[str, float]:
298
+ """Forward upstream ``Retry-After`` to ``_handle_stream_error`` only when one was sent.
299
+
300
+ The kw-only ``retry_after_seconds`` is additive; omitting it when there is
301
+ nothing to convey keeps partial doubles with the historical positional
302
+ signature (``account, error, code, http_status``) working.
303
+ """
304
+ return {} if retry_after_seconds is None else {"retry_after_seconds": retry_after_seconds}
305
+
306
+
307
+ def _payload_size_estimate_bytes(payload: ResponsesRequest) -> int:
308
+ return len(json.dumps(payload.to_payload(), ensure_ascii=True, separators=(",", ":")).encode("utf-8"))
309
+
310
+
311
+ def _transient_retry_error_code(tex: BaseException) -> str:
312
+ """Error code the same-account transient retry loop aggregates under.
313
+
314
+ Stream-framed failures keep their own code. HTTP-status failures used to
315
+ collapse to ``server_error``; an upstream 5xx whose body says the account
316
+ is overloaded keeps that code so the health write after retry exhaustion
317
+ still counts as an admission rejection (the overload backoff feeds on the
318
+ code, and ``server_is_overloaded`` classifies as the same transient class).
319
+ """
320
+ if isinstance(tex, _TransientStreamError):
321
+ return tex.code
322
+ payload = getattr(tex, "payload", None)
323
+ if isinstance(payload, Mapping):
324
+ parsed = _parse_openai_error(payload)
325
+ if parsed is not None and isinstance(parsed.code, str) and parsed.code in UPSTREAM_OVERLOAD_CODES:
326
+ return parsed.code
327
+ return "server_error"
328
+
329
+
330
+ class _StreamingRetryMixin:
331
+ async def _stream_with_retry(
332
+ self,
333
+ payload: ResponsesRequest,
334
+ headers: Mapping[str, str],
335
+ *,
336
+ codex_session_affinity: bool,
337
+ propagate_http_errors: bool,
338
+ openai_cache_affinity: bool,
339
+ api_key: ApiKeyData | None,
340
+ api_key_reservation: ApiKeyUsageReservationData | None,
341
+ suppress_text_done_events: bool,
342
+ request_transport: str,
343
+ rewritten_file_account_id: str | None = None,
344
+ file_account_resolution_complete: bool = False,
345
+ upstream_stream_transport_override: str | None = None,
346
+ client_ip: str | None = None,
347
+ enforce_openai_sdk_contract: bool = True,
348
+ ) -> AsyncIterator[str]:
349
+ proxy = cast(_StreamingServiceProtocol, self)
350
+ scheduler = scheduler_for(proxy)
351
+ clock = clock_for(proxy)
352
+ useragent, useragent_group, conversation_id = _request_log_client_fields(headers)
353
+ request_id = ensure_request_id()
354
+ start = clock.monotonic()
355
+ base_settings = _facade().get_settings()
356
+ settings = await _facade().get_settings_cache().get()
357
+ # C2-3 resilience toggles: resolved from this request's snapshot and
358
+ # bound to the task so the upstream client gates its breaker the same
359
+ # way; rebound before every upstream attempt because a keepalive yield
360
+ # can move this generator to another task (ContextVars follow tasks).
361
+ resilience = bind_resilience_toggles(settings, startup_settings=base_settings)
362
+ concurrency_caps = _facade().effective_account_concurrency_caps(settings)
363
+ deadline = start + _facade()._stream_request_budget_seconds(
364
+ base_settings,
365
+ request_transport=request_transport,
366
+ )
367
+ prefer_earlier_reset = settings.prefer_earlier_reset_accounts
368
+ # Resolved once per request: the API-key override wins, then the fleet
369
+ # value. ``shared`` is a strict no-op downstream. The fleet value is read
370
+ # off the *overlaid* settings, not the raw dashboard row: the row's column
371
+ # is NULL until an operator sets it, so reading it directly would keep the
372
+ # proxy on ``shared`` while ``GET /api/settings`` reported the environment
373
+ # value as effective. ``DASHBOARD_MODE_SETTINGS`` puts the column back on
374
+ # top where it belongs -- dashboard over environment over default.
375
+ thread_cache_identity_mode, thread_cache_identity_from_key = _effective_thread_cache_identity_mode(
376
+ api_key, with_dashboard_overrides(base_settings)
377
+ )
378
+ upstream_transport_policy_label = "explicit" if upstream_stream_transport_override is not None else "configured"
379
+ upstream_transport_sticky = _http_downstream_request_is_sticky(payload, headers)
380
+ preserve_native_failure_lifecycle = not enforce_openai_sdk_contract and _is_native_codex_request(headers)
381
+
382
+ def _must_abort_native_transport_failure(
383
+ exc: ProxyResponseError,
384
+ *,
385
+ downstream_visible: bool,
386
+ ) -> bool:
387
+ if not preserve_native_failure_lifecycle or downstream_visible:
388
+ return False
389
+ error = _parse_openai_error(exc.payload)
390
+ error_code = _normalize_error_code(
391
+ error.code if error else None,
392
+ error.type if error else None,
393
+ )
394
+ return error_code in SYNTHETIC_TRANSPORT_FAILURE_CODES
395
+
396
+ upstream_stream_transport = upstream_stream_transport_override
397
+ if upstream_stream_transport is None:
398
+ configured_transport, explicit_transport = _resolved_configured_stream_transport(settings)
399
+ # ``has_image_generation_tool`` means exactly that: folding
400
+ # ``input_image`` into it pinned every image turn to upstream HTTP
401
+ # from inside _resolve_stream_transport, where no log or counter
402
+ # records the decision (#2363). An ``input_image`` request is pinned
403
+ # only by the narrow predicate below.
404
+ image_generation_bypass = _facade()._responses_request_uses_image_generation(payload)
405
+ payload_size_estimate = _payload_size_estimate_bytes(payload)
406
+ resolved_base_transport = _resolve_stream_transport(
407
+ transport=configured_transport,
408
+ transport_override=None,
409
+ model=payload.model,
410
+ headers=headers,
411
+ has_image_generation_tool=image_generation_bypass,
412
+ # Include what isolated mode will inject: that choice is passed
413
+ # down as an explicit override, which short-circuits the
414
+ # post-injection size check, so a request just under the
415
+ # websocket budget has to be measured at its egress size here.
416
+ payload_size_estimate_bytes=(
417
+ payload_size_estimate + cache_scope_payload_overhead_bytes(thread_cache_identity_mode)
418
+ ),
419
+ )
420
+ upstream_stream_transport = resolved_base_transport
421
+ if not explicit_transport and (
422
+ image_generation_bypass
423
+ or _facade()._input_image_request_requires_http_upstream(
424
+ payload, payload_size_estimate_bytes=payload_size_estimate
425
+ )
426
+ ):
427
+ upstream_stream_transport = "http"
428
+ if (
429
+ not explicit_transport
430
+ and request_transport == _REQUEST_TRANSPORT_HTTP
431
+ and upstream_stream_transport == "websocket"
432
+ ):
433
+ sticky = upstream_transport_sticky
434
+ if upstream_websocket_transport_recently_failed():
435
+ policy = "recent_ws_failure"
436
+ override_applied = False
437
+ upstream_transport_policy_label = policy
438
+ upstream_stream_transport = "http"
439
+ else:
440
+ policy, override_applied = _effective_http_downstream_transport_policy(api_key, settings)
441
+ upstream_transport_policy_label = policy
442
+ policy_transport = _resolve_http_downstream_transport(policy, payload=payload, headers=headers)
443
+ upstream_stream_transport = "http" if policy_transport == "http" else configured_transport
444
+ logger.info(
445
+ "http_downstream_transport_decision policy=%s override_applied=%s sticky=%s "
446
+ "upstream_stream_transport=%s request_id=%s",
447
+ policy,
448
+ override_applied,
449
+ sticky,
450
+ upstream_stream_transport,
451
+ request_id,
452
+ )
453
+ elif request_transport == _REQUEST_TRANSPORT_HTTP:
454
+ logger.info(
455
+ "http_downstream_transport_decision policy=explicit override_applied=%s sticky=%s "
456
+ "upstream_stream_transport=%s request_id=%s",
457
+ False,
458
+ _http_downstream_request_is_sticky(payload, headers),
459
+ upstream_stream_transport,
460
+ request_id,
461
+ )
462
+ if request_transport == _REQUEST_TRANSPORT_HTTP and upstream_websocket_transport_recently_failed():
463
+ upstream_stream_transport = "http"
464
+ upstream_transport_policy_label = "recent_ws_failure"
465
+ if rewritten_file_account_id is None and not file_account_resolution_complete:
466
+ proxy._raise_for_unsupported_input_image_references(payload)
467
+ rewritten_file_account_id = await proxy._resolve_file_account_for_responses(payload, headers)
468
+ affinity = _sticky_key_for_responses_request(
469
+ payload,
470
+ headers,
471
+ codex_session_affinity=codex_session_affinity,
472
+ openai_cache_affinity=openai_cache_affinity,
473
+ openai_cache_affinity_max_age_seconds=settings.openai_cache_affinity_max_age_seconds,
474
+ sticky_threads_enabled=settings.sticky_threads_enabled,
475
+ api_key=api_key,
476
+ )
477
+ turn_state_owner_account_id: str | None = None
478
+ turn_state = _sticky_key_from_turn_state_header(headers)
479
+ if turn_state is not None:
480
+ # HTTP and WebSocket transports share the bridge turn-state index;
481
+ # treating this as ordinary sticky input would cross replicas or
482
+ # accounts when the token was minted by an HTTP bridge session.
483
+ turn_state_owner_account_id = await proxy._resolve_compact_turn_state_owner(
484
+ turn_state=turn_state,
485
+ api_key=api_key,
486
+ fail_on_missing=not _is_synthesized_turn_state(turn_state),
487
+ )
488
+ affinity_observation = AffinityObservation.from_policy(affinity)
489
+ _maybe_log_proxy_request_shape(
490
+ "stream",
491
+ payload,
492
+ headers,
493
+ sticky_kind=affinity_observation.kind,
494
+ sticky_key_source=affinity_observation.source,
495
+ derivation_outcome=affinity.prompt_cache_derivation_outcome,
496
+ prompt_cache_key_set=_prompt_cache_key_from_request_model(payload) is not None,
497
+ thread_cache_identity_mode=thread_cache_identity_mode,
498
+ thread_cache_identity_from_key=thread_cache_identity_from_key,
499
+ )
500
+ routing_strategy = _facade()._routing_strategy(settings)
501
+ max_attempts = _facade()._STREAM_MAX_ACCOUNT_ATTEMPTS
502
+ settled = False
503
+ any_attempt_logged = False
504
+ upstream_transport_metric_status: str | None = None
505
+ upstream_transport_metric_recorded = False
506
+ network_recovery = ProcessNetworkRecovery(transport="stream", request_id=request_id)
507
+ settlement = _StreamSettlement()
508
+ last_transient_exc: ProxyResponseError | None = None
509
+ last_pre_dispatch_transport_error: ProxyResponseError | None = None
510
+ last_account_model_rejection: ProxyResponseError | None = None
511
+ last_account_model_rejection_account_id: str | None = None
512
+ account_model_replacement_account_id: str | None = None
513
+ account_model_replay_attempted = False
514
+ current_account_lease: AccountLease | None = None
515
+ last_security_work_retry_error: _RetryableStreamError | None = None
516
+ excluded_account_ids: set[str] = set()
517
+ transient_failed_account_id: str | None = None
518
+ hard_affinity_same_owner_retry_attempted = False
519
+ # Owner-bound burst 429 same-account retries; reset when the counter's
520
+ # account differs from the one being retried (see the closures below).
521
+ burst_same_account_retries = 0
522
+ burst_same_account_retry_account_id: str | None = None
523
+ deferred_capacity_account: Account | None = None
524
+ deferred_capacity_lease: AccountLease | None = None
525
+ preferred_account_id: str | None = None
526
+ payload_replay_required_account_id: str | None = None
527
+ file_preferred_account_id: str | None = rewritten_file_account_id
528
+ require_preferred_account = False
529
+ last_retryable_stream_error: _RetryableStreamError | None = None
530
+ # (account, error, code, http_status, retry_count, retry_after_seconds,
531
+ # burst_cooldown_recorded): a queued keyed health write. The burst
532
+ # cooldown is replica-local runtime state with no reservation-ordering
533
+ # dependency, so it is engaged at rejection time and the flag stops the
534
+ # deferred write from re-engaging it after the stream settled.
535
+ pending_post_refresh_transient_penalties: list[
536
+ tuple[Account, UpstreamError, str, int | None, int, float | None, bool]
537
+ ] = []
538
+ deferred_account_error_backoffs: dict[str, Account] = {}
539
+ post_refresh_transient_replacement_selected = False
540
+ require_security_work_authorized = False
541
+ account_leases: list[AccountLease] = []
542
+ estimated_lease_tokens = _facade()._estimated_lease_tokens_from_request_usage_budget(
543
+ estimate_api_key_request_usage(payload)
544
+ )
545
+ verified_fresh_replay_payload = _verified_cross_transport_fresh_replay(
546
+ proxy,
547
+ payload=payload,
548
+ headers=headers,
549
+ api_key=api_key,
550
+ )
551
+
552
+ async def _release_tracked_stream_lease(lease: AccountLease | None) -> None:
553
+ if lease is None:
554
+ return
555
+ try:
556
+ account_leases.remove(lease)
557
+ except ValueError:
558
+ pass
559
+ await proxy._load_balancer.release_account_lease(lease)
560
+
561
+ def _render_dispatch_transport_error(exc: ProxyResponseError) -> str:
562
+ # Terminal render of the preserved sanitized transport failure:
563
+ # the client sees the original upstream-unavailable error instead
564
+ # of a misleading generated ``no_accounts`` response.
565
+ error = _parse_openai_error(exc.payload)
566
+ error_code = (
567
+ _normalize_error_code(
568
+ error.code if error else None,
569
+ error.type if error else None,
570
+ )
571
+ or "upstream_unavailable"
572
+ )
573
+ error_message = error.message if error and error.message else "Upstream transport failed"
574
+ event = response_failed_event(
575
+ error_code,
576
+ error_message,
577
+ error_type=(error.type if error else None) or "server_error",
578
+ response_id=request_id,
579
+ error_param=error.param_state if error else None,
580
+ )
581
+ _apply_error_metadata(event["response"]["error"], error)
582
+ return format_sse_event(event)
583
+
584
+ async def _flush_pending_post_refresh_penalties() -> None:
585
+ # Consume one queued penalty at a time only after that entry's
586
+ # health write finishes (success or logged failure). Clearing the
587
+ # whole queue first would drop later accounts if one write raises
588
+ # or cancellation lands on an await (compact flush isolates entries
589
+ # the same way). Each entry runs in an owned cancellation-deferring
590
+ # task so CancelledError mid-write cannot leave a half-applied
591
+ # tuple for cleanup to replay (double error_count).
592
+ while pending_post_refresh_transient_penalties:
593
+ (
594
+ failed_account,
595
+ transient_error_payload,
596
+ transient_error_code,
597
+ transient_http_status,
598
+ transient_retry_count,
599
+ transient_retry_after_seconds,
600
+ transient_burst_cooldown_recorded,
601
+ ) = pending_post_refresh_transient_penalties[0]
602
+
603
+ async def _apply_deferred_penalty(
604
+ account: Account = failed_account,
605
+ error_payload: UpstreamError = transient_error_payload,
606
+ error_code: str = transient_error_code,
607
+ http_status: int | None = transient_http_status,
608
+ retry_count: int = transient_retry_count,
609
+ retry_after_seconds: float | None = transient_retry_after_seconds,
610
+ burst_cooldown_recorded: bool = transient_burst_cooldown_recorded,
611
+ ) -> None:
612
+ try:
613
+ await proxy._handle_stream_error(
614
+ account,
615
+ error_payload,
616
+ error_code,
617
+ http_status=http_status,
618
+ **_retry_after_kwargs(retry_after_seconds),
619
+ **({"burst_cooldown_recorded": True} if burst_cooldown_recorded else {}),
620
+ )
621
+ if retry_count > 1:
622
+ await proxy._load_balancer.record_errors(account, retry_count - 1)
623
+ except Exception:
624
+ logger.warning(
625
+ "Failed to flush deferred keyed stream health account_id=%s request_id=%s",
626
+ account.id,
627
+ request_id,
628
+ exc_info=True,
629
+ )
630
+
631
+ apply_task = scheduler.create_task(
632
+ _apply_deferred_penalty(),
633
+ name=f"flush-deferred-keyed-stream-health-{failed_account.id}-{request_id}",
634
+ )
635
+ _, cancellation = await _await_task_deferring_cancellation(apply_task)
636
+ del pending_post_refresh_transient_penalties[0]
637
+ if cancellation is not None:
638
+ raise cancellation
639
+
640
+ async def _flush_or_schedule_pending_post_refresh_penalties() -> None:
641
+ if not pending_post_refresh_transient_penalties:
642
+ return
643
+ flush_coro = _flush_pending_post_refresh_penalties()
644
+ current_task = asyncio.current_task()
645
+ if current_task is not None and current_task.cancelling():
646
+ proxy._schedule_cancel_safe_cleanup(
647
+ flush_coro,
648
+ action="flush_deferred_keyed_stream_health",
649
+ request_id=request_id,
650
+ )
651
+ return
652
+ await flush_coro
653
+
654
+ async def _settle_stream_usage_before_pending_penalty(
655
+ current_settlement: _StreamSettlement,
656
+ *,
657
+ settlement_order_required: bool = False,
658
+ ) -> bool:
659
+ nonlocal settled
660
+ apply_pending_penalty = post_refresh_transient_replacement_selected and bool(
661
+ pending_post_refresh_transient_penalties
662
+ )
663
+ wait_for_health_write = (
664
+ settlement_order_required
665
+ or current_settlement.settlement_order_required
666
+ or apply_pending_penalty
667
+ or bool(deferred_account_error_backoffs)
668
+ )
669
+ settle_kwargs = {"wait_for_settlement": True} if wait_for_health_write else {}
670
+ settled_result = await proxy._settle_stream_api_key_usage(
671
+ api_key,
672
+ api_key_reservation,
673
+ current_settlement,
674
+ request_id,
675
+ **settle_kwargs,
676
+ )
677
+ if not settled_result:
678
+ return False
679
+ # Commit settlement visibility before any cancellable health flush.
680
+ # CancelledError during flush must not leave ``settled`` false while
681
+ # ``usage_settlement_transferred`` is already true — that combination
682
+ # skips both reservation cleanup and retained-queue flush.
683
+ settled = True
684
+ try:
685
+ await proxy._drain_deferred_account_error_backoffs(deferred_account_error_backoffs)
686
+ finally:
687
+ # Route backoffs and queued stream penalties have independent
688
+ # ownership after settlement. A failed backoff write must not
689
+ # orphan the penalty queue in detached cancellation cleanup.
690
+ if apply_pending_penalty:
691
+ await _flush_pending_post_refresh_penalties()
692
+ return settled_result
693
+
694
+ async def _record_or_defer_confirmed_route_backoff(account: Account) -> None:
695
+ if api_key is not None and api_key_reservation is not None:
696
+ deferred_account_error_backoffs.setdefault(account.id, account)
697
+ return
698
+ await proxy._load_balancer.record_error_backoff(account)
699
+
700
+ async def _handle_or_defer_keyed_stream_health(
701
+ failed_account: Account,
702
+ failed_error: UpstreamError,
703
+ failed_code: str,
704
+ *,
705
+ http_status: int | None = None,
706
+ transient_retry_count: int = 1,
707
+ retry_after_seconds: float | None = None,
708
+ ) -> ClassifiedFailure:
709
+ """Classify and either write health now or defer until after settle.
710
+
711
+ Keyed streams keep a shared ``api_key_reservation`` open across
712
+ mid-loop failover ``continue``/``break`` paths. Immediate
713
+ ``_handle_stream_error`` / ``record_errors`` on those paths would
714
+ run while the reservation is still open; queue into
715
+ ``pending_post_refresh_transient_penalties`` instead so
716
+ ``_settle_stream_usage_before_pending_penalty`` flushes health
717
+ after settlement (same ordering as compact keyed mid-loop health).
718
+ """
719
+ if api_key is not None and api_key_reservation is not None:
720
+ classified = classify_upstream_failure(
721
+ error_code=failed_code,
722
+ error=failed_error,
723
+ http_status=http_status,
724
+ phase="first_event",
725
+ )
726
+ burst_cooldown_recorded = is_upstream_burst_rejection(
727
+ failure_class=classified["failure_class"],
728
+ http_status=http_status,
729
+ )
730
+ if burst_cooldown_recorded:
731
+ # Replica-local runtime state only (no DB write, no
732
+ # reservation ordering): engage it now so other requests
733
+ # are steered away during the burst, not after settlement.
734
+ await record_upstream_burst_rejection(
735
+ proxy._load_balancer,
736
+ failed_account,
737
+ retry_after_seconds=retry_after_seconds,
738
+ )
739
+ pending_post_refresh_transient_penalties.append(
740
+ (
741
+ failed_account,
742
+ failed_error,
743
+ failed_code,
744
+ http_status,
745
+ transient_retry_count,
746
+ retry_after_seconds,
747
+ burst_cooldown_recorded,
748
+ )
749
+ )
750
+ return classified
751
+ classified = await proxy._handle_stream_error(
752
+ failed_account,
753
+ failed_error,
754
+ failed_code,
755
+ http_status=http_status,
756
+ **_retry_after_kwargs(retry_after_seconds),
757
+ )
758
+ if transient_retry_count > 1:
759
+ await proxy._load_balancer.record_errors(failed_account, transient_retry_count - 1)
760
+ return classified
761
+
762
+ def _stream_owner_bound_to(account: Account) -> bool:
763
+ """True when this request cannot move off ``account``.
764
+
765
+ Mirrors the loop's ``can_try_other_account`` predicates: single-account
766
+ routing, a payload already dispatched to the account (account-bound
767
+ items such as ``reasoning``), a required previous-response /
768
+ turn-state owner, or an input-file owner. A required owner whose
769
+ locally verified full-input replay may still move
770
+ (``_move_verified_fresh_replay_from_owner``) is *not* bound.
771
+ """
772
+ if routing_strategy == "single_account":
773
+ return True
774
+ if (
775
+ require_preferred_account
776
+ and preferred_account_id == account.id
777
+ and verified_fresh_replay_payload is not None
778
+ ):
779
+ return False
780
+ return (
781
+ payload_replay_required_account_id == account.id
782
+ or (require_preferred_account and preferred_account_id == account.id)
783
+ or account.id == file_preferred_account_id
784
+ or turn_state_owner_account_id is not None
785
+ )
786
+
787
+ def _burst_same_account_retry_available(account: Account, *, burst: bool) -> bool:
788
+ if not burst:
789
+ return False
790
+ retries = burst_same_account_retries if burst_same_account_retry_account_id == account.id else 0
791
+ return retries < BURST_SAME_ACCOUNT_MAX_RETRIES and proxy._remaining_budget_seconds(deadline) > 0
792
+
793
+ def _next_burst_same_account_retry(account: Account) -> int:
794
+ nonlocal burst_same_account_retries, burst_same_account_retry_account_id
795
+ if burst_same_account_retry_account_id != account.id:
796
+ burst_same_account_retry_account_id = account.id
797
+ burst_same_account_retries = 0
798
+ burst_same_account_retries += 1
799
+ return burst_same_account_retries
800
+
801
+ def _stamp_surfaced_burst_retry_after(exc: ProxyResponseError) -> None:
802
+ # A surfaced burst 429 that carried no upstream Retry-After still
803
+ # tells the client when to come back (api.py emits the header).
804
+ if exc.retry_after_seconds is None:
805
+ exc.retry_after_seconds = BURST_SURFACE_RETRY_AFTER_SECONDS
806
+
807
+ async def _drain_pending_post_refresh_penalty_on_terminal(
808
+ current_settlement: _StreamSettlement,
809
+ ) -> bool:
810
+ nonlocal post_refresh_transient_replacement_selected, settled
811
+ if pending_post_refresh_transient_penalties or deferred_account_error_backoffs:
812
+ # A failed replacement selection still ends the request. Mark
813
+ # it as terminal so the deferred failure is settled and
814
+ # recorded before this path returns or re-raises.
815
+ if pending_post_refresh_transient_penalties:
816
+ post_refresh_transient_replacement_selected = True
817
+ settled = await _settle_stream_usage_before_pending_penalty(current_settlement)
818
+ return settled
819
+ return True
820
+
821
+ async def _finalize_terminal_settlement_after_downstream_close(
822
+ current_settlement: _StreamSettlement,
823
+ account: Account,
824
+ *,
825
+ settlement_order_required: bool = False,
826
+ upstream_http_status: int | None = None,
827
+ ) -> None:
828
+ """Settle usage then write account health after the client went away.
829
+
830
+ ``_StreamSettlement`` has no HTTP-status field, so a caller that
831
+ reached here from an HTTP-coded failure must pass
832
+ ``upstream_http_status`` for the soft-overload window to see that
833
+ the settlement's ``error_code`` was HTTP-derived rather than a
834
+ status-less stream terminal.
835
+ """
836
+ nonlocal settled
837
+
838
+ async def _finalize() -> None:
839
+ nonlocal settled
840
+ if not settled:
841
+ settled = await _settle_stream_usage_before_pending_penalty(
842
+ current_settlement,
843
+ settlement_order_required=settlement_order_required,
844
+ )
845
+ if not settled:
846
+ return
847
+ if current_settlement.account_health_error:
848
+ await proxy._handle_stream_error(
849
+ account,
850
+ _stream_settlement_error_payload(current_settlement),
851
+ current_settlement.error_code or "upstream_error",
852
+ upstream_http_status=upstream_http_status,
853
+ )
854
+ elif current_settlement.record_success:
855
+ await proxy._load_balancer.record_success(account)
856
+
857
+ finalize_task = scheduler.create_task(_finalize(), name=f"stream-terminal-settlement-{request_id}")
858
+ _, cancellation = await _await_task_deferring_cancellation(finalize_task)
859
+ if cancellation is not None:
860
+ raise cancellation
861
+
862
+ async def _wait_for_process_network_recovery(
863
+ account: Account,
864
+ *,
865
+ error_code: str | None,
866
+ retryable_same_contract: bool,
867
+ failed_session: aiohttp.ClientSession | None = None,
868
+ ) -> NetworkRecoveryDecision:
869
+ network_recovery.account_id = account.id
870
+ return await network_recovery.wait(
871
+ error_code=error_code,
872
+ retryable_same_contract=retryable_same_contract,
873
+ deadline=deadline,
874
+ rotate_shared_client=True,
875
+ failed_session=failed_session,
876
+ )
877
+
878
+ async def _settle_process_network_budget_exhaustion(
879
+ account: Account,
880
+ settlement: _StreamSettlement,
881
+ ) -> None:
882
+ nonlocal settled
883
+ settlement.status = "error"
884
+ settlement.record_success = False
885
+ settlement.account_health_error = False
886
+ settlement.error_code = "upstream_request_timeout"
887
+ settlement.error_message = "Proxy request budget exhausted"
888
+ settlement.error = {"message": "Proxy request budget exhausted"}
889
+ await proxy._write_stream_preflight_error(
890
+ affinity_observation=affinity_observation,
891
+ account_id=account.id,
892
+ api_key=api_key,
893
+ request_id=request_id,
894
+ model=payload.model,
895
+ start=start,
896
+ error_code="upstream_request_timeout",
897
+ error_message="Proxy request budget exhausted",
898
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
899
+ service_tier=payload.service_tier,
900
+ transport=request_transport,
901
+ upstream_transport=upstream_stream_transport,
902
+ useragent=useragent,
903
+ useragent_group=useragent_group,
904
+ conversation_id=conversation_id,
905
+ client_ip=client_ip,
906
+ )
907
+ settled = await _settle_stream_usage_before_pending_penalty(settlement)
908
+
909
+ def _authorize_payload_dispatch(account: Account) -> bool:
910
+ required_account_id = payload_replay_required_account_id
911
+ if required_account_id is not None and required_account_id != account.id:
912
+ raise ProxyResponseError(
913
+ 502,
914
+ openai_error(
915
+ "previous_response_owner_unavailable",
916
+ "Request payload owner account is unavailable; retry later.",
917
+ error_type="server_error",
918
+ ),
919
+ )
920
+ return required_account_id is None and not responses_payload_is_account_neutral_fresh_replay(
921
+ payload.to_replay_safety_payload()
922
+ )
923
+
924
+ def _move_verified_fresh_replay_from_owner(*, account_id: str, outcome: str) -> bool:
925
+ # Only a proxy-injected owner anchor with locally verified full
926
+ # input may move; the failed owner stays excluded so sticky
927
+ # selection cannot immediately loop back to it.
928
+ nonlocal affinity, payload, payload_replay_required_account_id
929
+ nonlocal preferred_account_id, require_preferred_account, verified_fresh_replay_payload
930
+ if not (
931
+ require_preferred_account
932
+ and preferred_account_id == account_id
933
+ and verified_fresh_replay_payload is not None
934
+ ):
935
+ return False
936
+ if not responses_payload_is_account_neutral_fresh_replay(
937
+ verified_fresh_replay_payload.to_replay_safety_payload()
938
+ ):
939
+ return False
940
+ payload = verified_fresh_replay_payload
941
+ payload_replay_required_account_id = None
942
+ verified_fresh_replay_payload = None
943
+ excluded_account_ids.add(account_id)
944
+ preferred_account_id = None
945
+ require_preferred_account = False
946
+ affinity = replace(affinity, reallocate_sticky=True)
947
+ logger.info(
948
+ "cross_transport_verified_fresh_replay request_id=%s outcome=%s account_id=%s",
949
+ request_id,
950
+ outcome,
951
+ account_id,
952
+ )
953
+ return True
954
+
955
+ async def _stream_post_refresh_with_capacity_recovery(
956
+ account: Account,
957
+ *,
958
+ settlement: _StreamSettlement,
959
+ can_try_other_account: bool,
960
+ tool_call_dedupe: _WebSocketUpstreamControl,
961
+ ) -> AsyncGenerator[str, None]:
962
+ nonlocal last_transient_exc
963
+ transient_retries = 0
964
+
965
+ async def _iter_stream_once() -> AsyncGenerator[str, None]:
966
+ inner_stream = proxy._stream_once(
967
+ account,
968
+ payload,
969
+ headers,
970
+ request_id,
971
+ False,
972
+ request_started_at=start,
973
+ affinity_observation=affinity_observation,
974
+ allow_transient_retry=True,
975
+ api_key=api_key,
976
+ api_key_reservation=api_key_reservation,
977
+ settlement=settlement,
978
+ suppress_text_done_events=suppress_text_done_events,
979
+ upstream_stream_transport=upstream_stream_transport,
980
+ request_transport=request_transport,
981
+ concurrency_caps=concurrency_caps,
982
+ useragent=useragent,
983
+ useragent_group=useragent_group,
984
+ conversation_id=conversation_id,
985
+ client_ip=client_ip,
986
+ tool_call_dedupe=tool_call_dedupe,
987
+ enforce_openai_sdk_contract=enforce_openai_sdk_contract,
988
+ thread_cache_identity=ThreadCacheIdentity(thread_cache_identity_mode, account.id),
989
+ )
990
+ try:
991
+ try:
992
+ async for line in inner_stream:
993
+ yield line
994
+ finally:
995
+ close_task = scheduler.create_task(
996
+ inner_stream.aclose(),
997
+ name=f"stream-post-refresh-inner-close-{request_id}",
998
+ )
999
+ _, close_cancellation = await _await_task_deferring_cancellation(close_task)
1000
+ if close_cancellation is not None:
1001
+ raise close_cancellation
1002
+ except ProxyResponseError as exc:
1003
+ if is_confirmed_pre_dispatch_transport_error(exc):
1004
+ # Keep dispatch provenance intact for the outer account
1005
+ # failover handler. Converting this into the generic
1006
+ # transient wrapper would authorize same-account replay
1007
+ # and lose the confirmed dead-route backoff semantics.
1008
+ raise
1009
+ error = _parse_openai_error(exc.payload)
1010
+ error_code = _normalize_error_code(
1011
+ error.code if error else None,
1012
+ error.type if error else None,
1013
+ )
1014
+ error_message = error.message if error else None
1015
+ retryable_connect_failure = bool(
1016
+ exc.failure_phase == "connect"
1017
+ and _facade()._should_retry_transient_stream_error(error_code, error_message)
1018
+ )
1019
+ retryable_capacity_rejection = bool(
1020
+ is_upstream_model_capacity_error(error_message)
1021
+ and _facade()._should_retry_transient_stream_error(error_code, error_message)
1022
+ )
1023
+ if retryable_connect_failure or retryable_capacity_rejection:
1024
+ raise _TransientStreamError(
1025
+ error_code or "upstream_error",
1026
+ _upstream_error_from_openai(error),
1027
+ account_health_error=retryable_connect_failure,
1028
+ ) from exc
1029
+ raise
1030
+
1031
+ while True:
1032
+ settlement.reset()
1033
+ # Rebind per attempt: a capacity keepalive may have handed this
1034
+ # generator to another task since the request-entry binding.
1035
+ set_resilience_toggles(resilience)
1036
+ stream_timeout_tokens = _facade()._push_stream_attempt_timeout_overrides(
1037
+ proxy._remaining_budget_seconds(deadline)
1038
+ )
1039
+ try:
1040
+ attempt_stream = _iter_stream_once()
1041
+ try:
1042
+ try:
1043
+ async for line in attempt_stream:
1044
+ yield line
1045
+ finally:
1046
+ close_task = scheduler.create_task(
1047
+ attempt_stream.aclose(),
1048
+ name=f"stream-post-refresh-close-{request_id}",
1049
+ )
1050
+ _, close_cancellation = await _await_task_deferring_cancellation(close_task)
1051
+ if close_cancellation is not None:
1052
+ raise close_cancellation
1053
+ except (asyncio.CancelledError, GeneratorExit):
1054
+ # A terminal frame may already have been yielded when
1055
+ # downstream cancellation is delivered on the next
1056
+ # generator resume. Finalize that terminal usage and
1057
+ # health result before propagating cancellation so the
1058
+ # reservation is not released as abandoned.
1059
+ if settlement.status in {"success", "error"} and not settled:
1060
+ await _finalize_terminal_settlement_after_downstream_close(
1061
+ settlement,
1062
+ account,
1063
+ settlement_order_required=True,
1064
+ )
1065
+ raise
1066
+ network_recovery.log_recovered()
1067
+ return
1068
+ except _TerminalStreamError:
1069
+ # `_stream_once()` has already yielded the terminal event.
1070
+ # Returning preserves fail-closed delivery: replaying here
1071
+ # could duplicate a request that reached the upstream.
1072
+ return
1073
+ except _TransientStreamError as exc:
1074
+ recovery_decision = await _wait_for_process_network_recovery(
1075
+ account,
1076
+ error_code=exc.code,
1077
+ retryable_same_contract=False,
1078
+ failed_session=None,
1079
+ )
1080
+ if recovery_decision == "retry":
1081
+ continue
1082
+ if recovery_decision == "exhausted":
1083
+ raise ProxyResponseError(
1084
+ 502,
1085
+ openai_error("upstream_request_timeout", "Proxy request budget exhausted"),
1086
+ ) from exc
1087
+ transient_retries += 1
1088
+ if (
1089
+ transient_retries < _facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES
1090
+ and proxy._remaining_budget_seconds(deadline) > 0
1091
+ and not settlement.downstream_visible
1092
+ ):
1093
+ delay = backoff_seconds(transient_retries)
1094
+ _facade().logger.info(
1095
+ "Transient post-refresh stream error, retrying same account "
1096
+ "request_id=%s account_id=%s retry=%s/%s delay=%.2fs code=%s",
1097
+ request_id,
1098
+ account.id,
1099
+ transient_retries,
1100
+ _facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES,
1101
+ delay,
1102
+ exc.code,
1103
+ )
1104
+ await scheduler.sleep(delay)
1105
+ continue
1106
+ error_message = str(exc.error.get("message") or "Upstream error")
1107
+ settlement.record_success = False
1108
+ settlement.error_code = exc.code
1109
+ settlement.error_message = error_message
1110
+ settlement.error = exc.error
1111
+ settlement.account_health_error = (
1112
+ _facade()._should_penalize_stream_error(exc.code, error_message)
1113
+ or is_upstream_model_capacity_error(error_message)
1114
+ or exc.account_health_error
1115
+ )
1116
+ if settlement.account_health_error:
1117
+ pending_post_refresh_transient_penalties.append(
1118
+ (
1119
+ account,
1120
+ _stream_settlement_error_payload(settlement),
1121
+ settlement.error_code or "upstream_error",
1122
+ 502,
1123
+ transient_retries,
1124
+ None,
1125
+ False,
1126
+ )
1127
+ )
1128
+ if can_try_other_account:
1129
+ retry_exc = ProxyResponseError(502, openai_error(exc.code, error_message))
1130
+ setattr(retry_exc, _POST_REFRESH_TRANSIENT_EXHAUSTED_ATTR, True)
1131
+ raise retry_exc from exc
1132
+ yield format_sse_event(
1133
+ response_failed_event(
1134
+ exc.code,
1135
+ error_message,
1136
+ response_id=settlement.response_id or request_id,
1137
+ )
1138
+ )
1139
+ return
1140
+ except ProxyResponseError as exc:
1141
+ if _must_abort_native_transport_failure(
1142
+ exc,
1143
+ downstream_visible=settlement.downstream_visible,
1144
+ ):
1145
+ raise
1146
+ error = _parse_openai_error(exc.payload)
1147
+ error_code = _normalize_error_code(
1148
+ error.code if error else None,
1149
+ error.type if error else None,
1150
+ )
1151
+ recovery_decision = await _wait_for_process_network_recovery(
1152
+ account,
1153
+ error_code=error_code,
1154
+ retryable_same_contract=exc.retryable_same_contract,
1155
+ failed_session=exc.failed_session,
1156
+ )
1157
+ if recovery_decision == "retry":
1158
+ continue
1159
+ if recovery_decision == "exhausted":
1160
+ raise ProxyResponseError(
1161
+ 502,
1162
+ openai_error("upstream_request_timeout", "Proxy request budget exhausted"),
1163
+ ) from exc
1164
+ if error_code != "account_response_create_cap":
1165
+ raise
1166
+ last_transient_exc = exc
1167
+ if can_try_other_account:
1168
+ raise
1169
+ recovery_sleep_seconds = _account_selection_recovery_sleep_seconds(
1170
+ AccountSelection(
1171
+ account=None,
1172
+ error_message=error.message if error else None,
1173
+ error_code=error_code,
1174
+ )
1175
+ )
1176
+ if recovery_sleep_seconds is None or proxy._remaining_budget_seconds(deadline) <= 0:
1177
+ raise
1178
+ async for wait_event in _iter_account_capacity_recovery_wait(
1179
+ request_id=request_id,
1180
+ model=payload.model,
1181
+ account_id=account.id,
1182
+ error_message=error.message if error else None,
1183
+ recovery_sleep_seconds=recovery_sleep_seconds,
1184
+ remaining_budget_seconds=proxy._remaining_budget_seconds(deadline),
1185
+ emit_keepalives=not propagate_http_errors or not enforce_openai_sdk_contract,
1186
+ stage="post_refresh_response_create",
1187
+ scheduler=scheduler,
1188
+ clock=clock,
1189
+ ):
1190
+ yield wait_event
1191
+ if proxy._remaining_budget_seconds(deadline) <= 0:
1192
+ raise
1193
+ finally:
1194
+ pop_stream_timeout_overrides(stream_timeout_tokens)
1195
+
1196
+ def _record_upstream_transport_metric_once(status: str) -> None:
1197
+ nonlocal upstream_transport_metric_recorded
1198
+ if upstream_transport_metric_recorded:
1199
+ return
1200
+ upstream_transport_metric_recorded = True
1201
+ _record_upstream_transport_decision(
1202
+ downstream_transport=request_transport,
1203
+ upstream_transport=upstream_stream_transport,
1204
+ policy=upstream_transport_policy_label,
1205
+ sticky=upstream_transport_sticky,
1206
+ status=status,
1207
+ )
1208
+
1209
+ async def _render_account_model_rejection(
1210
+ exc: ProxyResponseError,
1211
+ *,
1212
+ account_id: str | None,
1213
+ ) -> str:
1214
+ error = _parse_openai_error(exc.payload)
1215
+ error_code = (
1216
+ _normalize_error_code(
1217
+ error.code if error else None,
1218
+ error.type if error else None,
1219
+ )
1220
+ or "invalid_request_error"
1221
+ )
1222
+ error_message = error.message if error and error.message else "Upstream rejected the requested model"
1223
+ event = response_failed_event(
1224
+ error_code,
1225
+ error_message,
1226
+ error_type=(error.type if error else None) or "invalid_request_error",
1227
+ response_id=request_id,
1228
+ error_param=error.param_state if error else None,
1229
+ )
1230
+ _apply_error_metadata(event["response"]["error"], error)
1231
+ if not any_attempt_logged:
1232
+ await proxy._write_request_log(
1233
+ affinity_observation=affinity_observation,
1234
+ account_id=account_id,
1235
+ api_key=api_key,
1236
+ request_id=request_id,
1237
+ model=payload.model,
1238
+ latency_ms=int((clock.monotonic() - start) * 1000),
1239
+ status="error",
1240
+ error_code=error_code,
1241
+ error_message=error_message,
1242
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1243
+ transport=request_transport,
1244
+ upstream_transport=upstream_stream_transport,
1245
+ service_tier=payload.service_tier,
1246
+ requested_service_tier=payload.service_tier,
1247
+ useragent=useragent,
1248
+ useragent_group=useragent_group,
1249
+ conversation_id=conversation_id,
1250
+ client_ip=client_ip,
1251
+ )
1252
+ return format_sse_event(event)
1253
+
1254
+ async def _retry_account_model_rejection(
1255
+ exc: ProxyResponseError,
1256
+ account: Account,
1257
+ *,
1258
+ outcome: str,
1259
+ ) -> bool | None:
1260
+ nonlocal affinity, current_account_lease
1261
+ nonlocal account_model_replay_attempted
1262
+ nonlocal last_account_model_rejection, last_account_model_rejection_account_id
1263
+ error = _parse_openai_error(exc.payload)
1264
+ error_code = _normalize_error_code(
1265
+ error.code if error else None,
1266
+ error.type if error else None,
1267
+ )
1268
+ if not _is_account_model_unsupported_error(
1269
+ code=error_code,
1270
+ message=error.message if error else None,
1271
+ model=payload.model,
1272
+ ):
1273
+ return None
1274
+ can_move_verified_owner = bool(
1275
+ require_preferred_account
1276
+ and preferred_account_id == account.id
1277
+ and verified_fresh_replay_payload is not None
1278
+ )
1279
+ can_try_other_account = bool(
1280
+ not account_model_replay_attempted
1281
+ and attempt < max_attempts - 1
1282
+ and (
1283
+ can_move_verified_owner
1284
+ or (not require_preferred_account and account.id != file_preferred_account_id)
1285
+ )
1286
+ )
1287
+ if not can_try_other_account:
1288
+ return False
1289
+ logger.info(
1290
+ "Retrying stream after account/model rejection request_id=%s account_id=%s model=%s phase=%s reason=%s",
1291
+ request_id,
1292
+ account.id,
1293
+ payload.model,
1294
+ outcome,
1295
+ _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE,
1296
+ )
1297
+ account_model_replay_attempted = True
1298
+ last_account_model_rejection = exc
1299
+ last_account_model_rejection_account_id = account.id
1300
+ await _release_tracked_stream_lease(current_account_lease)
1301
+ current_account_lease = None
1302
+ if not _move_verified_fresh_replay_from_owner(
1303
+ account_id=account.id,
1304
+ outcome=outcome,
1305
+ ):
1306
+ excluded_account_ids.add(account.id)
1307
+ affinity = replace(affinity, reallocate_sticky=True)
1308
+ return True
1309
+
1310
+ try:
1311
+ # From this exact point the service finalizer below owns reservation
1312
+ # settlement/release. Preflight failures before this boundary are
1313
+ # still owned by the originating API startup guard.
1314
+ _signal_propagated_responses_service_cleanup_ready()
1315
+ if payload.previous_response_id is not None:
1316
+ previous_response_lookup_session_id = _owner_lookup_session_id_from_headers(headers)
1317
+ preferred_account_id = await proxy._resolve_websocket_previous_response_owner(
1318
+ previous_response_id=payload.previous_response_id,
1319
+ api_key=api_key,
1320
+ session_id=previous_response_lookup_session_id,
1321
+ surface="http_stream",
1322
+ )
1323
+ require_preferred_account = preferred_account_id is not None
1324
+ # `previous_response_id` is a stored-object continuation, so it
1325
+ # remains hard owner-bound even when the request also carries a
1326
+ # soft prompt-cache affinity key. A different account may have a
1327
+ # warmer cache, but it cannot safely resolve the stored response.
1328
+ if preferred_account_id is None:
1329
+ selection_inputs = await proxy._load_balancer._load_selection_inputs(
1330
+ model=payload.model,
1331
+ additional_limit_name=None,
1332
+ account_ids=None,
1333
+ )
1334
+ if len(selection_inputs.accounts) != 1:
1335
+ message = "Previous response owner account is unavailable; retry later."
1336
+ _record_continuity_fail_closed(
1337
+ surface="http_stream",
1338
+ reason="owner_account_unavailable",
1339
+ previous_response_id=payload.previous_response_id,
1340
+ session_id=previous_response_lookup_session_id,
1341
+ upstream_error_code="owner_lookup_miss",
1342
+ )
1343
+ event = response_failed_event(
1344
+ "previous_response_owner_unavailable",
1345
+ message,
1346
+ response_id=request_id,
1347
+ )
1348
+ yield format_sse_event(event)
1349
+ await proxy._write_request_log(
1350
+ affinity_observation=affinity_observation,
1351
+ account_id=None,
1352
+ api_key=api_key,
1353
+ request_id=request_id,
1354
+ model=payload.model,
1355
+ latency_ms=int((clock.monotonic() - start) * 1000),
1356
+ status="error",
1357
+ error_code="previous_response_owner_unavailable",
1358
+ error_message=message,
1359
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1360
+ transport=request_transport,
1361
+ upstream_transport=upstream_stream_transport,
1362
+ service_tier=payload.service_tier,
1363
+ requested_service_tier=payload.service_tier,
1364
+ useragent=useragent,
1365
+ useragent_group=useragent_group,
1366
+ conversation_id=conversation_id,
1367
+ client_ip=client_ip,
1368
+ )
1369
+ return
1370
+ # File and previous-response ownership are peers, not fallback
1371
+ # preferences. Resolve both before selection so a conflict cannot
1372
+ # be hidden by whichever source happened to run first. A hard turn
1373
+ # state is checked against this required owner by the balancer.
1374
+ preferred_account_id = resolve_required_account_id(
1375
+ ("turn state", turn_state_owner_account_id),
1376
+ ("previous response", preferred_account_id),
1377
+ ("input file", rewritten_file_account_id),
1378
+ )
1379
+ require_preferred_account = require_preferred_account or turn_state_owner_account_id is not None
1380
+ file_required_preferred_account = rewritten_file_account_id is not None
1381
+ for attempt in range(max_attempts):
1382
+ remaining_budget = proxy._remaining_budget_seconds(deadline)
1383
+ if remaining_budget <= 0:
1384
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
1385
+ _facade().logger.warning(
1386
+ "Proxy request budget exhausted before retry request_id=%s attempt=%s",
1387
+ request_id,
1388
+ attempt + 1,
1389
+ )
1390
+ await proxy._write_stream_preflight_error(
1391
+ affinity_observation=affinity_observation,
1392
+ account_id=None,
1393
+ api_key=api_key,
1394
+ request_id=request_id,
1395
+ model=payload.model,
1396
+ start=start,
1397
+ error_code="upstream_request_timeout",
1398
+ error_message="Proxy request budget exhausted",
1399
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1400
+ service_tier=payload.service_tier,
1401
+ transport=request_transport,
1402
+ upstream_transport=upstream_stream_transport,
1403
+ useragent=useragent,
1404
+ useragent_group=useragent_group,
1405
+ conversation_id=conversation_id,
1406
+ client_ip=client_ip,
1407
+ )
1408
+ if preserve_native_failure_lifecycle:
1409
+ return
1410
+ yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
1411
+ return
1412
+ while True:
1413
+ effective_preferred_account_id = resolve_required_account_id(
1414
+ ("continuation", preferred_account_id),
1415
+ ("dispatched payload", payload_replay_required_account_id),
1416
+ )
1417
+ effective_require_preferred_account = (
1418
+ require_preferred_account or payload_replay_required_account_id is not None
1419
+ )
1420
+ try:
1421
+ selection = await proxy._select_account_with_budget_compatible(
1422
+ deadline,
1423
+ request_id=request_id,
1424
+ kind="stream",
1425
+ api_key=api_key,
1426
+ affinity_policy=affinity,
1427
+ prefer_earlier_reset_accounts=prefer_earlier_reset,
1428
+ prefer_earlier_reset_window=_facade()._prefer_earlier_reset_window(settings),
1429
+ routing_strategy=routing_strategy,
1430
+ model=payload.model,
1431
+ service_tier=payload.service_tier,
1432
+ exclude_account_ids=excluded_account_ids,
1433
+ preferred_account_id=effective_preferred_account_id,
1434
+ require_security_work_authorized=require_security_work_authorized,
1435
+ lease_kind="stream",
1436
+ estimated_lease_tokens=estimated_lease_tokens,
1437
+ # Keep stored-object and file ownership strict. The
1438
+ # verified-fresh replay branch below removes its
1439
+ # anchor before it permits cross-account movement.
1440
+ fallback_on_preferred_account_unavailable=not (
1441
+ effective_require_preferred_account or file_required_preferred_account
1442
+ ),
1443
+ )
1444
+ except ProxyResponseError as exc:
1445
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
1446
+ error = _parse_openai_error(exc.payload)
1447
+ error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
1448
+ error_message = error.message if error else None
1449
+ if _facade()._is_proxy_budget_exhausted_error(exc):
1450
+ await proxy._write_stream_preflight_error(
1451
+ affinity_observation=affinity_observation,
1452
+ account_id=None,
1453
+ api_key=api_key,
1454
+ request_id=request_id,
1455
+ model=payload.model,
1456
+ start=start,
1457
+ error_code="upstream_request_timeout",
1458
+ error_message="Proxy request budget exhausted",
1459
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1460
+ service_tier=payload.service_tier,
1461
+ transport=request_transport,
1462
+ upstream_transport=upstream_stream_transport,
1463
+ useragent=useragent,
1464
+ useragent_group=useragent_group,
1465
+ conversation_id=conversation_id,
1466
+ client_ip=client_ip,
1467
+ )
1468
+ if preserve_native_failure_lifecycle:
1469
+ return
1470
+ yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
1471
+ return
1472
+ event = response_failed_event(
1473
+ error_code,
1474
+ error_message or "Upstream unavailable",
1475
+ error_type=(error.type or "server_error") if error else "server_error",
1476
+ response_id=request_id,
1477
+ )
1478
+ _apply_error_metadata(event["response"]["error"], error)
1479
+ yield format_sse_event(event)
1480
+ return
1481
+ account = selection.account
1482
+ current_account_lease = selection.lease
1483
+ if selection.lease is not None:
1484
+ account_leases.append(selection.lease)
1485
+ if (
1486
+ not account
1487
+ and require_security_work_authorized
1488
+ and selection.error_code == _facade()._NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE
1489
+ ):
1490
+ _facade().logger.info(
1491
+ "No security-work-authorized account available for stream retry; "
1492
+ "continuing normal account failover request_id=%s",
1493
+ request_id,
1494
+ )
1495
+ yield format_sse_event(
1496
+ _facade()._security_work_advisory_event(
1497
+ code=_facade()._NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE,
1498
+ message=_facade()._SECURITY_WORK_NO_AUTHORIZED_ACCOUNTS_MESSAGE,
1499
+ request_id=request_id,
1500
+ action="continue_normal_selection",
1501
+ )
1502
+ )
1503
+ require_security_work_authorized = False
1504
+ continue
1505
+ if not account and deferred_capacity_account is not None:
1506
+ deferred_error = _parse_openai_error(last_transient_exc.payload) if last_transient_exc else None
1507
+ recovery_sleep_seconds = _account_selection_recovery_sleep_seconds(
1508
+ AccountSelection(
1509
+ account=None,
1510
+ error_message=deferred_error.message if deferred_error else None,
1511
+ error_code="account_response_create_cap",
1512
+ )
1513
+ )
1514
+ if recovery_sleep_seconds is not None:
1515
+ remaining_budget_seconds = proxy._remaining_budget_seconds(deadline)
1516
+ if remaining_budget_seconds <= 0:
1517
+ if propagate_http_errors and last_transient_exc is not None:
1518
+ raise last_transient_exc
1519
+ event = response_failed_event(
1520
+ "account_response_create_cap",
1521
+ (deferred_error.message if deferred_error else None)
1522
+ or "Account response-create concurrency limit reached",
1523
+ error_type=(deferred_error.type if deferred_error else None) or "server_error",
1524
+ response_id=request_id,
1525
+ )
1526
+ yield format_sse_event(event)
1527
+ return
1528
+ capacity_account = deferred_capacity_account
1529
+ capacity_account_id = capacity_account.id
1530
+ excluded_account_ids.discard(capacity_account_id)
1531
+ async for wait_event in _iter_account_capacity_recovery_wait(
1532
+ request_id=request_id,
1533
+ model=payload.model,
1534
+ account_id=capacity_account_id,
1535
+ error_message=deferred_error.message if deferred_error else None,
1536
+ recovery_sleep_seconds=recovery_sleep_seconds,
1537
+ remaining_budget_seconds=proxy._remaining_budget_seconds(deadline),
1538
+ emit_keepalives=not propagate_http_errors or not enforce_openai_sdk_contract,
1539
+ stage="response_create_no_alternate",
1540
+ scheduler=scheduler,
1541
+ clock=clock,
1542
+ ):
1543
+ yield wait_event
1544
+ if proxy._remaining_budget_seconds(deadline) <= 0:
1545
+ if propagate_http_errors and last_transient_exc is not None:
1546
+ raise last_transient_exc
1547
+ event = response_failed_event(
1548
+ "account_response_create_cap",
1549
+ (deferred_error.message if deferred_error else None)
1550
+ or "Account response-create concurrency limit reached",
1551
+ error_type=(deferred_error.type if deferred_error else None) or "server_error",
1552
+ response_id=request_id,
1553
+ )
1554
+ yield format_sse_event(event)
1555
+ return
1556
+ account = capacity_account
1557
+ current_account_lease = deferred_capacity_lease
1558
+ deferred_capacity_account = None
1559
+ deferred_capacity_lease = None
1560
+ if account is not None and deferred_capacity_account is not None:
1561
+ await _release_tracked_stream_lease(deferred_capacity_lease)
1562
+ deferred_capacity_account = None
1563
+ deferred_capacity_lease = None
1564
+ if (
1565
+ not account
1566
+ and selection.error_code == "hard_affinity_saturated"
1567
+ and transient_failed_account_id is not None
1568
+ and transient_failed_account_id in excluded_account_ids
1569
+ and not hard_affinity_same_owner_retry_attempted
1570
+ ):
1571
+ # A hard CODEX_SESSION row can resolve only to its owner.
1572
+ # If the transport just excluded that same account after
1573
+ # a pre-visible transient failure, alternate selection is
1574
+ # impossible by construction. Retry the known owner once
1575
+ # without weakening or rebinding the durable affinity.
1576
+ excluded_account_ids.discard(transient_failed_account_id)
1577
+ hard_affinity_same_owner_retry_attempted = True
1578
+ _facade().logger.info(
1579
+ "Retrying transient stream failure on hard affinity owner request_id=%s account_id=%s",
1580
+ request_id,
1581
+ transient_failed_account_id,
1582
+ )
1583
+ continue
1584
+ if (
1585
+ not account
1586
+ and selection.error_code != USAGE_LIMIT_REACHED
1587
+ and (
1588
+ selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES
1589
+ or not (propagate_http_errors and last_transient_exc is not None)
1590
+ )
1591
+ and (
1592
+ selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES
1593
+ # A preserved confirmed pre-dispatch failure is
1594
+ # terminal for this request: waiting for capacity
1595
+ # recovery cannot resurrect the dead proxy route.
1596
+ or last_pre_dispatch_transport_error is None
1597
+ )
1598
+ and (
1599
+ selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES
1600
+ or (last_retryable_stream_error is None and last_security_work_retry_error is None)
1601
+ )
1602
+ ):
1603
+ recovery_sleep_seconds = _account_selection_recovery_sleep_seconds(selection)
1604
+ if recovery_sleep_seconds is not None:
1605
+ remaining_budget_seconds = proxy._remaining_budget_seconds(deadline)
1606
+ if remaining_budget_seconds <= 0:
1607
+ break
1608
+ async for wait_event in _iter_account_capacity_recovery_wait(
1609
+ request_id=request_id,
1610
+ model=payload.model,
1611
+ account_id=None,
1612
+ error_message=selection.error_message,
1613
+ recovery_sleep_seconds=recovery_sleep_seconds,
1614
+ remaining_budget_seconds=remaining_budget_seconds,
1615
+ emit_keepalives=not propagate_http_errors or not enforce_openai_sdk_contract,
1616
+ stage="selection",
1617
+ scheduler=scheduler,
1618
+ clock=clock,
1619
+ ):
1620
+ yield wait_event
1621
+ if proxy._remaining_budget_seconds(deadline) <= 0:
1622
+ break
1623
+ continue
1624
+ break
1625
+ if not account:
1626
+ if last_account_model_rejection is not None:
1627
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
1628
+ if propagate_http_errors:
1629
+ raise last_account_model_rejection
1630
+ yield await _render_account_model_rejection(
1631
+ last_account_model_rejection,
1632
+ account_id=last_account_model_rejection_account_id,
1633
+ )
1634
+ return
1635
+ if last_pre_dispatch_transport_error is not None:
1636
+ # No eligible replacement exists: preserve the original
1637
+ # sanitized upstream-unavailable failure instead of
1638
+ # generating a misleading ``no_accounts`` response.
1639
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
1640
+ if propagate_http_errors:
1641
+ raise last_pre_dispatch_transport_error
1642
+ yield _render_dispatch_transport_error(last_pre_dispatch_transport_error)
1643
+ return
1644
+ if selection.error_code == USAGE_LIMIT_REACHED:
1645
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
1646
+ no_accounts_msg = selection.error_message or "Usage limit reached"
1647
+ status_code, error_payload = selection_failure_response(selection)
1648
+ await proxy._write_request_log(
1649
+ affinity_observation=affinity_observation,
1650
+ account_id=None,
1651
+ api_key=api_key,
1652
+ request_id=request_id,
1653
+ model=payload.model,
1654
+ latency_ms=int((clock.monotonic() - start) * 1000),
1655
+ status="error",
1656
+ error_code=USAGE_LIMIT_REACHED,
1657
+ error_message=no_accounts_msg,
1658
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1659
+ transport=request_transport,
1660
+ upstream_transport=upstream_stream_transport,
1661
+ service_tier=payload.service_tier,
1662
+ requested_service_tier=payload.service_tier,
1663
+ useragent=useragent,
1664
+ useragent_group=useragent_group,
1665
+ client_ip=client_ip,
1666
+ )
1667
+ if propagate_http_errors:
1668
+ raise ProxyResponseError(status_code, error_payload)
1669
+ yield format_sse_event(
1670
+ response_failed_event(
1671
+ USAGE_LIMIT_REACHED,
1672
+ no_accounts_msg,
1673
+ error_type=USAGE_LIMIT_REACHED,
1674
+ response_id=request_id,
1675
+ resets_at=selection.resets_at,
1676
+ )
1677
+ )
1678
+ return
1679
+ if selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES:
1680
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
1681
+ no_accounts_msg = selection.error_message or "Local account capacity is exhausted"
1682
+ error_code = selection.error_code
1683
+ event = response_failed_event(
1684
+ error_code,
1685
+ no_accounts_msg,
1686
+ error_type="rate_limit_error",
1687
+ response_id=request_id,
1688
+ )
1689
+ await proxy._write_request_log(
1690
+ affinity_observation=affinity_observation,
1691
+ account_id=None,
1692
+ api_key=api_key,
1693
+ request_id=request_id,
1694
+ model=payload.model,
1695
+ latency_ms=int((clock.monotonic() - start) * 1000),
1696
+ status="error",
1697
+ error_code=error_code,
1698
+ error_message=no_accounts_msg,
1699
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1700
+ transport=request_transport,
1701
+ upstream_transport=upstream_stream_transport,
1702
+ service_tier=payload.service_tier,
1703
+ requested_service_tier=payload.service_tier,
1704
+ useragent=useragent,
1705
+ useragent_group=useragent_group,
1706
+ conversation_id=conversation_id,
1707
+ client_ip=client_ip,
1708
+ )
1709
+ if propagate_http_errors:
1710
+ raise ProxyResponseError(
1711
+ 429,
1712
+ openai_error(
1713
+ error_code,
1714
+ no_accounts_msg,
1715
+ error_type="rate_limit_error",
1716
+ ),
1717
+ )
1718
+ yield format_sse_event(event)
1719
+ return
1720
+ if (
1721
+ require_preferred_account
1722
+ and preferred_account_id is not None
1723
+ and verified_fresh_replay_payload is not None
1724
+ ):
1725
+ excluded_account_ids.add(preferred_account_id)
1726
+ payload = verified_fresh_replay_payload
1727
+ verified_fresh_replay_payload = None
1728
+ preferred_account_id = None
1729
+ require_preferred_account = False
1730
+ affinity = replace(affinity, reallocate_sticky=True)
1731
+ logger.info(
1732
+ "cross_transport_verified_fresh_replay request_id=%s outcome=owner_unavailable",
1733
+ request_id,
1734
+ )
1735
+ continue
1736
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
1737
+ if propagate_http_errors and last_transient_exc is not None:
1738
+ raise last_transient_exc
1739
+ if last_retryable_stream_error is not None:
1740
+ error_message = str(last_retryable_stream_error.error.get("message") or "Upstream error")
1741
+ event = response_failed_event(
1742
+ last_retryable_stream_error.code,
1743
+ error_message,
1744
+ response_id=request_id,
1745
+ )
1746
+ yield format_sse_event(event)
1747
+ await proxy._write_request_log(
1748
+ affinity_observation=affinity_observation,
1749
+ account_id=None,
1750
+ api_key=api_key,
1751
+ request_id=request_id,
1752
+ model=payload.model,
1753
+ latency_ms=int((clock.monotonic() - start) * 1000),
1754
+ status="error",
1755
+ error_code=last_retryable_stream_error.code,
1756
+ error_message=error_message,
1757
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1758
+ transport=request_transport,
1759
+ upstream_transport=upstream_stream_transport,
1760
+ service_tier=payload.service_tier,
1761
+ requested_service_tier=payload.service_tier,
1762
+ useragent=useragent,
1763
+ useragent_group=useragent_group,
1764
+ conversation_id=conversation_id,
1765
+ client_ip=client_ip,
1766
+ )
1767
+ return
1768
+ if require_preferred_account and preferred_account_id is not None:
1769
+ error_code = "previous_response_owner_unavailable"
1770
+ message = "Previous response owner account is unavailable; retry later."
1771
+ reason = "owner_account_unavailable"
1772
+ upstream_error_code = "no_accounts"
1773
+ if selection.error_code == "continuity_owner_conflict":
1774
+ error_code = "continuity_owner_conflict"
1775
+ message = (
1776
+ selection.error_message
1777
+ or "Account-owned continuity sources conflict; retry the logical turn"
1778
+ )
1779
+ reason = "owner_conflict"
1780
+ upstream_error_code = selection.error_code
1781
+ _record_continuity_fail_closed(
1782
+ surface="http_stream",
1783
+ reason=reason,
1784
+ previous_response_id=payload.previous_response_id,
1785
+ session_id=headers.get("x-codex-turn-state") or headers.get("session_id"),
1786
+ upstream_error_code=upstream_error_code,
1787
+ )
1788
+ event = response_failed_event(
1789
+ error_code,
1790
+ message,
1791
+ response_id=request_id,
1792
+ )
1793
+ yield format_sse_event(event)
1794
+ await proxy._write_request_log(
1795
+ affinity_observation=affinity_observation,
1796
+ account_id=preferred_account_id,
1797
+ api_key=api_key,
1798
+ request_id=request_id,
1799
+ model=payload.model,
1800
+ latency_ms=int((clock.monotonic() - start) * 1000),
1801
+ status="error",
1802
+ error_code=error_code,
1803
+ error_message=message,
1804
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1805
+ transport=request_transport,
1806
+ upstream_transport=upstream_stream_transport,
1807
+ service_tier=payload.service_tier,
1808
+ requested_service_tier=payload.service_tier,
1809
+ useragent=useragent,
1810
+ useragent_group=useragent_group,
1811
+ conversation_id=conversation_id,
1812
+ client_ip=client_ip,
1813
+ )
1814
+ return
1815
+ # If a prior attempt stored a transient 500 and the caller
1816
+ # expects HTTP error propagation, re-raise the original error
1817
+ # instead of returning a generic no_accounts event.
1818
+ if propagate_http_errors and last_transient_exc is not None:
1819
+ raise last_transient_exc
1820
+ if last_security_work_retry_error is not None:
1821
+ message = (
1822
+ last_security_work_retry_error.error.get("message")
1823
+ or "Security work authorization is required"
1824
+ )
1825
+ event = response_failed_event(
1826
+ last_security_work_retry_error.code,
1827
+ message,
1828
+ response_id=request_id,
1829
+ )
1830
+ yield format_sse_event(event)
1831
+ await proxy._write_request_log(
1832
+ affinity_observation=affinity_observation,
1833
+ account_id=None,
1834
+ api_key=api_key,
1835
+ request_id=request_id,
1836
+ model=payload.model,
1837
+ latency_ms=int((clock.monotonic() - start) * 1000),
1838
+ status="error",
1839
+ error_code=last_security_work_retry_error.code,
1840
+ error_message=message,
1841
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1842
+ transport=request_transport,
1843
+ upstream_transport=upstream_stream_transport,
1844
+ service_tier=payload.service_tier,
1845
+ requested_service_tier=payload.service_tier,
1846
+ useragent=useragent,
1847
+ useragent_group=useragent_group,
1848
+ conversation_id=conversation_id,
1849
+ client_ip=client_ip,
1850
+ )
1851
+ return
1852
+ no_accounts_msg = selection.error_message or "No active accounts available"
1853
+ error_code = selection.error_code or "no_accounts"
1854
+ event = response_failed_event(
1855
+ error_code,
1856
+ no_accounts_msg,
1857
+ error_type="rate_limit_error"
1858
+ if error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES
1859
+ else "server_error",
1860
+ response_id=request_id,
1861
+ )
1862
+ yield format_sse_event(event)
1863
+ await proxy._write_request_log(
1864
+ affinity_observation=affinity_observation,
1865
+ account_id=None,
1866
+ api_key=api_key,
1867
+ request_id=request_id,
1868
+ model=payload.model,
1869
+ latency_ms=int((clock.monotonic() - start) * 1000),
1870
+ status="error",
1871
+ error_code=error_code,
1872
+ error_message=no_accounts_msg,
1873
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1874
+ transport=request_transport,
1875
+ upstream_transport=upstream_stream_transport,
1876
+ service_tier=payload.service_tier,
1877
+ requested_service_tier=payload.service_tier,
1878
+ useragent=useragent,
1879
+ useragent_group=useragent_group,
1880
+ conversation_id=conversation_id,
1881
+ client_ip=client_ip,
1882
+ )
1883
+ return
1884
+
1885
+ if pending_post_refresh_transient_penalties:
1886
+ post_refresh_transient_replacement_selected = True
1887
+
1888
+ account_id_value = account.id
1889
+ if last_pre_dispatch_transport_error is not None:
1890
+ # The preserved connect failure is only authoritative when
1891
+ # replacement selection is empty. Once another account is
1892
+ # actually attempted, its terminal outcome takes precedence.
1893
+ if last_transient_exc is last_pre_dispatch_transport_error:
1894
+ last_transient_exc = None
1895
+ last_pre_dispatch_transport_error = None
1896
+ if last_account_model_rejection is not None and account.id != last_account_model_rejection_account_id:
1897
+ # The original 400 is only the fallback when account
1898
+ # selection cannot produce a replacement. Once this
1899
+ # replacement attempt starts, its own failure is the one
1900
+ # that must reach the client. Keep the separate replay
1901
+ # budget so another account/model rejection cannot trigger
1902
+ # a second transparent replay.
1903
+ account_model_replacement_account_id = account.id
1904
+ last_account_model_rejection = None
1905
+ last_account_model_rejection_account_id = None
1906
+ if (
1907
+ require_preferred_account
1908
+ and preferred_account_id is not None
1909
+ and account.id != preferred_account_id
1910
+ ):
1911
+ if verified_fresh_replay_payload is not None:
1912
+ payload = verified_fresh_replay_payload
1913
+ verified_fresh_replay_payload = None
1914
+ excluded_account_ids.add(preferred_account_id)
1915
+ preferred_account_id = None
1916
+ require_preferred_account = False
1917
+ affinity = replace(affinity, reallocate_sticky=True)
1918
+ logger.info(
1919
+ "cross_transport_verified_fresh_replay request_id=%s outcome=alternate_selected",
1920
+ request_id,
1921
+ )
1922
+ else:
1923
+ error_code = "previous_response_owner_unavailable"
1924
+ message = "Previous response owner account is unavailable; retry later."
1925
+ reason = "owner_account_unavailable"
1926
+ upstream_error_code = "upstream_unavailable"
1927
+ if selection.error_code == "continuity_owner_conflict":
1928
+ error_code = "continuity_owner_conflict"
1929
+ message = (
1930
+ selection.error_message
1931
+ or "Account-owned continuity sources conflict; retry the logical turn"
1932
+ )
1933
+ reason = "owner_conflict"
1934
+ upstream_error_code = selection.error_code
1935
+ _record_continuity_fail_closed(
1936
+ surface="http_stream",
1937
+ reason=reason,
1938
+ previous_response_id=payload.previous_response_id,
1939
+ session_id=headers.get("x-codex-turn-state") or headers.get("session_id"),
1940
+ upstream_error_code=upstream_error_code,
1941
+ )
1942
+ event = response_failed_event(
1943
+ error_code,
1944
+ message,
1945
+ response_id=request_id,
1946
+ )
1947
+ yield format_sse_event(event)
1948
+ await proxy._write_request_log(
1949
+ affinity_observation=affinity_observation,
1950
+ account_id=preferred_account_id,
1951
+ api_key=api_key,
1952
+ request_id=request_id,
1953
+ model=payload.model,
1954
+ latency_ms=int((clock.monotonic() - start) * 1000),
1955
+ status="error",
1956
+ error_code=error_code,
1957
+ error_message=message,
1958
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1959
+ transport=request_transport,
1960
+ upstream_transport=upstream_stream_transport,
1961
+ service_tier=payload.service_tier,
1962
+ requested_service_tier=payload.service_tier,
1963
+ useragent=useragent,
1964
+ useragent_group=useragent_group,
1965
+ conversation_id=conversation_id,
1966
+ client_ip=client_ip,
1967
+ )
1968
+ return
1969
+ try:
1970
+ remaining_budget = proxy._remaining_budget_seconds(deadline)
1971
+ if remaining_budget <= 0:
1972
+ _facade().logger.warning(
1973
+ "Proxy request budget exhausted before freshness check "
1974
+ "request_id=%s attempt=%s account_id=%s",
1975
+ request_id,
1976
+ attempt + 1,
1977
+ account.id,
1978
+ )
1979
+ await proxy._write_stream_preflight_error(
1980
+ affinity_observation=affinity_observation,
1981
+ account_id=account.id,
1982
+ api_key=api_key,
1983
+ request_id=request_id,
1984
+ model=payload.model,
1985
+ start=start,
1986
+ error_code="upstream_request_timeout",
1987
+ error_message="Proxy request budget exhausted",
1988
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
1989
+ service_tier=payload.service_tier,
1990
+ transport=request_transport,
1991
+ upstream_transport=upstream_stream_transport,
1992
+ useragent=useragent,
1993
+ useragent_group=useragent_group,
1994
+ conversation_id=conversation_id,
1995
+ client_ip=client_ip,
1996
+ )
1997
+ if preserve_native_failure_lifecycle:
1998
+ return
1999
+ yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
2000
+ return
2001
+ try:
2002
+ account = await proxy._ensure_fresh_with_budget(account, timeout_seconds=remaining_budget)
2003
+ except UpstreamProxyRouteError as exc:
2004
+ message = f"Upstream proxy route unavailable: {exc.reason}"
2005
+ await proxy._write_stream_preflight_error(
2006
+ affinity_observation=affinity_observation,
2007
+ account_id=account.id,
2008
+ api_key=api_key,
2009
+ request_id=request_id,
2010
+ model=payload.model,
2011
+ start=start,
2012
+ error_code="upstream_proxy_unavailable",
2013
+ error_message=message,
2014
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
2015
+ service_tier=payload.service_tier,
2016
+ transport=request_transport,
2017
+ upstream_transport=upstream_stream_transport,
2018
+ upstream_proxy_fail_closed_reason=exc.reason,
2019
+ useragent=useragent,
2020
+ useragent_group=useragent_group,
2021
+ conversation_id=conversation_id,
2022
+ client_ip=client_ip,
2023
+ )
2024
+ event = response_failed_event(
2025
+ "upstream_proxy_unavailable",
2026
+ message,
2027
+ response_id=request_id,
2028
+ )
2029
+ yield format_sse_event(event)
2030
+ return
2031
+ except (RefreshError, aiohttp.ClientError, asyncio.TimeoutError) as exc:
2032
+ selected_account_model_replacement = account.id == account_model_replacement_account_id
2033
+ if isinstance(exc, RefreshError):
2034
+ if exc.is_permanent:
2035
+ await proxy._load_balancer.mark_permanent_failure(account, exc.code)
2036
+ # Keep the warning account routable for later
2037
+ # requests, but do not immediately reselect it in
2038
+ # this request after its refresh material failed.
2039
+ # Release its stream slot before failover too.
2040
+ await _release_tracked_stream_lease(current_account_lease)
2041
+ current_account_lease = None
2042
+ if not selected_account_model_replacement:
2043
+ excluded_account_ids.add(account.id)
2044
+ continue
2045
+ if is_transient_refresh_contention(exc):
2046
+ # Transient CROSS-REPLICA refresh contention: benign
2047
+ # claim contention (the account's refresh claim is
2048
+ # held by another replica) OR a post-exchange
2049
+ # persist/status CAS conflict. This is NOT a genuine
2050
+ # ``transport_error`` OAuth failure — the account's
2051
+ # credentials are healthy — so fail over WITHOUT an
2052
+ # account-health penalty (no ``_handle_stream_error``);
2053
+ # the genuine transport failure handled below keeps
2054
+ # its penalty. A post-exchange persist conflict is
2055
+ # logged distinctly (rarer, more-serious race).
2056
+ if refresh_contention_kind(exc) == "persist_conflict":
2057
+ logger.warning(
2058
+ "Stream freshness-check refresh post-exchange persist conflict "
2059
+ "code=%s account_id=%s",
2060
+ exc.code,
2061
+ account.id,
2062
+ )
2063
+ if (
2064
+ not selected_account_model_replacement
2065
+ and not require_preferred_account
2066
+ and preferred_account_id is None
2067
+ ):
2068
+ # Movable request: release the stream lease and
2069
+ # fail over to a different account instead of
2070
+ # reselecting the same one until attempts are
2071
+ # exhausted. Record a retryable
2072
+ # upstream_unavailable so that if EVERY candidate
2073
+ # hits the held-claim condition, exhaustion
2074
+ # surfaces upstream_unavailable instead of a
2075
+ # misleading generic no_accounts response.
2076
+ await _release_tracked_stream_lease(current_account_lease)
2077
+ current_account_lease = None
2078
+ excluded_account_ids.add(account.id)
2079
+ last_retryable_stream_error = _RetryableStreamError(
2080
+ "upstream_unavailable",
2081
+ {
2082
+ "message": (
2083
+ exc.message
2084
+ or "Account refresh is temporarily unavailable; "
2085
+ "no healthy account could be reached."
2086
+ )
2087
+ },
2088
+ exclude_account=True,
2089
+ )
2090
+ continue
2091
+ # PINNED request (``previous_response_id`` /
2092
+ # ``input_file.file_id``): must not cross accounts.
2093
+ # Reselecting the same pinned account until attempts
2094
+ # are exhausted is pointless (it would leak the held
2095
+ # stream lease each iteration and then surface a
2096
+ # misleading ``no_accounts`` result). Stay on the
2097
+ # owner account, release the lease, and surface a
2098
+ # retryable ``upstream_unavailable`` promptly so the
2099
+ # client can retry once the claim clears.
2100
+ await _release_tracked_stream_lease(current_account_lease)
2101
+ current_account_lease = None
2102
+ message = exc.message or "Account refresh is temporarily unavailable; retry later."
2103
+ last_retryable_stream_error = _RetryableStreamError(
2104
+ "upstream_unavailable",
2105
+ {"message": message},
2106
+ )
2107
+ await proxy._write_stream_preflight_error(
2108
+ affinity_observation=affinity_observation,
2109
+ account_id=account.id,
2110
+ api_key=api_key,
2111
+ request_id=request_id,
2112
+ model=payload.model,
2113
+ start=start,
2114
+ error_code="upstream_unavailable",
2115
+ error_message=message,
2116
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
2117
+ service_tier=payload.service_tier,
2118
+ transport=request_transport,
2119
+ upstream_transport=upstream_stream_transport,
2120
+ useragent=useragent,
2121
+ useragent_group=useragent_group,
2122
+ conversation_id=conversation_id,
2123
+ client_ip=client_ip,
2124
+ )
2125
+ event = response_failed_event(
2126
+ "upstream_unavailable",
2127
+ message,
2128
+ response_id=request_id,
2129
+ )
2130
+ yield format_sse_event(event)
2131
+ return
2132
+ if not exc.transport_error and not selected_account_model_replacement:
2133
+ # Non-transport, non-permanent RefreshError: release
2134
+ # the stream lease and reselect (its prior behavior).
2135
+ await _release_tracked_stream_lease(current_account_lease)
2136
+ current_account_lease = None
2137
+ continue
2138
+ # A GENUINE OAuth transport failure
2139
+ # (``code == "transport_error"``): the account/route is
2140
+ # at fault, so it falls through to the shared
2141
+ # transport-failure handling below — identical to a raw
2142
+ # aiohttp/connect failure — which records the
2143
+ # account-health penalty (``_handle_stream_error``) so
2144
+ # the broken account backs off instead of being kept
2145
+ # healthy and reselected on the next request.
2146
+ _facade().logger.warning(
2147
+ "Stream refresh/connect failed request_id=%s attempt=%s account_id=%s",
2148
+ request_id,
2149
+ attempt + 1,
2150
+ account.id,
2151
+ exc_info=True,
2152
+ )
2153
+ message = getattr(exc, "message", None) or str(exc) or "Request to upstream timed out"
2154
+ if (
2155
+ not selected_account_model_replacement
2156
+ and _facade()._should_retry_transient_stream_error("upstream_unavailable", message)
2157
+ and attempt + 1 < max_attempts
2158
+ and _move_verified_fresh_replay_from_owner(
2159
+ account_id=account.id,
2160
+ outcome="owner_refresh_connect_failure",
2161
+ )
2162
+ ):
2163
+ await _handle_or_defer_keyed_stream_health(
2164
+ account,
2165
+ {"message": message},
2166
+ "upstream_unavailable",
2167
+ )
2168
+ await _release_tracked_stream_lease(current_account_lease)
2169
+ current_account_lease = None
2170
+ last_retryable_stream_error = _RetryableStreamError(
2171
+ "upstream_unavailable",
2172
+ {"message": message},
2173
+ exclude_account=True,
2174
+ )
2175
+ continue
2176
+ if (
2177
+ not selected_account_model_replacement
2178
+ and not require_preferred_account
2179
+ and preferred_account_id is None
2180
+ and _facade()._should_retry_transient_stream_error("upstream_unavailable", message)
2181
+ and attempt + 1 < max_attempts
2182
+ ):
2183
+ await _handle_or_defer_keyed_stream_health(
2184
+ account,
2185
+ {"message": message},
2186
+ "upstream_unavailable",
2187
+ )
2188
+ last_retryable_stream_error = _RetryableStreamError(
2189
+ "upstream_unavailable",
2190
+ {"message": message},
2191
+ exclude_account=True,
2192
+ )
2193
+ # The account keeps its health penalty above and is now
2194
+ # excluded from reselection, but its stream-concurrency
2195
+ # slot is still occupied by the lease appended at
2196
+ # selection. Release it before the failover ``continue``
2197
+ # (matching the claim-contention and permanent branches)
2198
+ # so the dead account's slot is freed immediately instead
2199
+ # of being held for the entire duration of the
2200
+ # replacement stream.
2201
+ await _release_tracked_stream_lease(current_account_lease)
2202
+ current_account_lease = None
2203
+ excluded_account_ids.add(account.id)
2204
+ continue
2205
+ await proxy._write_stream_preflight_error(
2206
+ affinity_observation=affinity_observation,
2207
+ account_id=account.id,
2208
+ api_key=api_key,
2209
+ request_id=request_id,
2210
+ model=payload.model,
2211
+ start=start,
2212
+ error_code="upstream_unavailable",
2213
+ error_message=message,
2214
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
2215
+ service_tier=payload.service_tier,
2216
+ transport=request_transport,
2217
+ upstream_transport=upstream_stream_transport,
2218
+ useragent=useragent,
2219
+ useragent_group=useragent_group,
2220
+ conversation_id=conversation_id,
2221
+ client_ip=client_ip,
2222
+ )
2223
+ event = response_failed_event(
2224
+ "upstream_unavailable",
2225
+ message,
2226
+ response_id=request_id,
2227
+ )
2228
+ yield format_sse_event(event)
2229
+ return
2230
+ any_attempt_logged = True
2231
+ settlement = _StreamSettlement()
2232
+ tool_call_dedupe = _WebSocketUpstreamControl()
2233
+ effective_attempt_timeout = proxy._remaining_budget_seconds(deadline)
2234
+ if effective_attempt_timeout <= 0:
2235
+ _facade().logger.warning(
2236
+ "Proxy request budget exhausted before stream attempt "
2237
+ "request_id=%s attempt=%s account_id=%s",
2238
+ request_id,
2239
+ attempt + 1,
2240
+ account.id,
2241
+ )
2242
+ await proxy._write_stream_preflight_error(
2243
+ affinity_observation=affinity_observation,
2244
+ account_id=account.id,
2245
+ api_key=api_key,
2246
+ request_id=request_id,
2247
+ model=payload.model,
2248
+ start=start,
2249
+ error_code="upstream_request_timeout",
2250
+ error_message="Proxy request budget exhausted",
2251
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
2252
+ service_tier=payload.service_tier,
2253
+ transport=request_transport,
2254
+ upstream_transport=upstream_stream_transport,
2255
+ useragent=useragent,
2256
+ useragent_group=useragent_group,
2257
+ conversation_id=conversation_id,
2258
+ client_ip=client_ip,
2259
+ )
2260
+ if preserve_native_failure_lifecycle:
2261
+ return
2262
+ yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
2263
+ return
2264
+ transient_retries = 0
2265
+ allow_retry_flag = attempt < max_attempts - 1
2266
+ while True:
2267
+ set_resilience_toggles(resilience) # rebind per attempt (task handoff)
2268
+ stream_timeout_tokens = _facade()._push_stream_attempt_timeout_overrides(
2269
+ proxy._remaining_budget_seconds(deadline),
2270
+ )
2271
+ try:
2272
+ settlement = _StreamSettlement()
2273
+ register_payload_owner = _authorize_payload_dispatch(account)
2274
+ inner_stream = proxy._stream_once(
2275
+ account,
2276
+ payload,
2277
+ headers,
2278
+ request_id,
2279
+ allow_retry_flag,
2280
+ request_started_at=start,
2281
+ affinity_observation=affinity_observation,
2282
+ allow_transient_retry=(
2283
+ transient_retries < _facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES - 1
2284
+ or allow_retry_flag
2285
+ ),
2286
+ api_key=api_key,
2287
+ api_key_reservation=api_key_reservation,
2288
+ settlement=settlement,
2289
+ suppress_text_done_events=suppress_text_done_events,
2290
+ upstream_stream_transport=upstream_stream_transport,
2291
+ request_transport=request_transport,
2292
+ concurrency_caps=concurrency_caps,
2293
+ useragent=useragent,
2294
+ useragent_group=useragent_group,
2295
+ conversation_id=conversation_id,
2296
+ client_ip=client_ip,
2297
+ # Let the retry path observe a pre-visible
2298
+ # account-recovery error only for the one case
2299
+ # where this owner anchor has a locally
2300
+ # verified, unanchored replacement body. All
2301
+ # other continuations retain the normal
2302
+ # fail-closed owner-error rewrite.
2303
+ preferred_account_id=(
2304
+ None
2305
+ if (
2306
+ require_preferred_account
2307
+ and preferred_account_id == account.id
2308
+ and verified_fresh_replay_payload is not None
2309
+ )
2310
+ else preferred_account_id
2311
+ ),
2312
+ tool_call_dedupe=tool_call_dedupe,
2313
+ enforce_openai_sdk_contract=enforce_openai_sdk_contract,
2314
+ thread_cache_identity=ThreadCacheIdentity(thread_cache_identity_mode, account.id),
2315
+ )
2316
+ try:
2317
+ try:
2318
+ async for line in inner_stream:
2319
+ if register_payload_owner:
2320
+ payload_replay_required_account_id = account.id
2321
+ register_payload_owner = False
2322
+ yield line
2323
+ if register_payload_owner:
2324
+ payload_replay_required_account_id = account.id
2325
+ except BaseException as exc:
2326
+ if register_payload_owner and not (
2327
+ isinstance(exc, ProxyResponseError)
2328
+ and is_confirmed_pre_dispatch_transport_error(exc)
2329
+ ):
2330
+ payload_replay_required_account_id = account.id
2331
+ raise
2332
+ finally:
2333
+ close_task = scheduler.create_task(
2334
+ inner_stream.aclose(),
2335
+ name=f"stream-inner-close-{request_id}",
2336
+ )
2337
+ _, close_cancellation = await _await_task_deferring_cancellation(close_task)
2338
+ if close_cancellation is not None:
2339
+ raise close_cancellation
2340
+ except (asyncio.CancelledError, GeneratorExit):
2341
+ # A terminal frame may already have been yielded when
2342
+ # downstream cancellation is delivered on the next
2343
+ # generator resume. Finalize that terminal usage and
2344
+ # health result before propagating cancellation so the
2345
+ # reservation is not released as abandoned.
2346
+ if settlement.status in {"success", "error"} and not settled:
2347
+ await _finalize_terminal_settlement_after_downstream_close(settlement, account)
2348
+ raise
2349
+ except (_TransientStreamError, ProxyResponseError) as tex:
2350
+ if isinstance(tex, ProxyResponseError) and _must_abort_native_transport_failure(
2351
+ tex,
2352
+ downstream_visible=settlement.downstream_visible,
2353
+ ):
2354
+ raise
2355
+ if account.id == account_model_replacement_account_id:
2356
+ # Account/model routing gets exactly one selected
2357
+ # replacement. Its own pre-visible 5xx/transport
2358
+ # failure is terminal; allowing the normal
2359
+ # transient path here would silently select a third
2360
+ # account. Re-raise into the outer terminal
2361
+ # renderer to retain the replacement details.
2362
+ if isinstance(tex, ProxyResponseError):
2363
+ raise
2364
+ raise ProxyResponseError(
2365
+ 502,
2366
+ openai_error(
2367
+ tex.code,
2368
+ str(tex.error.get("message") or "Upstream error"),
2369
+ error_type="server_error",
2370
+ ),
2371
+ ) from tex
2372
+ if settlement.downstream_visible:
2373
+ failed_response_id = settlement.response_id or request_id
2374
+ if isinstance(tex, ProxyResponseError):
2375
+ error = _parse_openai_error(tex.payload)
2376
+ error_code = _normalize_error_code(
2377
+ error.code if error else None,
2378
+ error.type if error else None,
2379
+ )
2380
+ error_message = error.message if error else "Upstream error"
2381
+ error_type = error.type if error else None
2382
+ error_param = error.param_state if error else None
2383
+ event = response_failed_event(
2384
+ error_code or "upstream_error",
2385
+ error_message or "Upstream error",
2386
+ error_type=error_type or "server_error",
2387
+ response_id=failed_response_id,
2388
+ error_param=error_param,
2389
+ )
2390
+ _apply_error_metadata(event["response"]["error"], error)
2391
+ else:
2392
+ error_code = tex.code
2393
+ error_message = str(tex.error.get("message") or "Upstream error")
2394
+ event = response_failed_event(
2395
+ error_code or "upstream_error",
2396
+ error_message,
2397
+ response_id=failed_response_id,
2398
+ )
2399
+ if isinstance(tex, ProxyResponseError):
2400
+ # Downstream visibility forbids replay, but a
2401
+ # concrete shared HTTP/WebSocket generation
2402
+ # carrying a process-network failure must still
2403
+ # be retired before later callers lease it.
2404
+ await _wait_for_process_network_recovery(
2405
+ account,
2406
+ error_code=error_code,
2407
+ retryable_same_contract=False,
2408
+ failed_session=tex.failed_session,
2409
+ )
2410
+ _facade().logger.warning(
2411
+ "Surfacing mid-stream upstream failure without replay "
2412
+ "request_id=%s account_id=%s code=%s",
2413
+ request_id,
2414
+ account.id,
2415
+ error_code,
2416
+ )
2417
+ settlement.record_success = False
2418
+ settlement.error_code = error_code
2419
+ settlement.error_message = error_message
2420
+ if isinstance(tex, ProxyResponseError):
2421
+ settlement.error = _upstream_error_from_openai(error)
2422
+ else:
2423
+ settlement.error = tex.error
2424
+ settlement.account_health_error = _facade()._should_penalize_stream_error(
2425
+ error_code, error_message
2426
+ )
2427
+ transient_upstream_http_status = (
2428
+ tex.status_code if isinstance(tex, ProxyResponseError) else None
2429
+ )
2430
+ if not (
2431
+ preserve_native_failure_lifecycle
2432
+ and error_code in SYNTHETIC_TRANSPORT_FAILURE_CODES
2433
+ ):
2434
+ try:
2435
+ yield format_sse_event(event)
2436
+ except (asyncio.CancelledError, GeneratorExit):
2437
+ await _finalize_terminal_settlement_after_downstream_close(
2438
+ settlement,
2439
+ account,
2440
+ upstream_http_status=transient_upstream_http_status,
2441
+ )
2442
+ raise
2443
+ settled = await _settle_stream_usage_before_pending_penalty(settlement)
2444
+ if settled and settlement.account_health_error:
2445
+ await proxy._handle_stream_error(
2446
+ account,
2447
+ _stream_settlement_error_payload(settlement),
2448
+ settlement.error_code or "upstream_error",
2449
+ # Evidence only. ``_TransientStreamError``
2450
+ # carries no status, which is correct:
2451
+ # those are the genuinely status-less
2452
+ # transients.
2453
+ upstream_http_status=transient_upstream_http_status,
2454
+ )
2455
+ return
2456
+ if isinstance(tex, ProxyResponseError) and tex.status_code != 500:
2457
+ error = _parse_openai_error(tex.payload)
2458
+ code = _normalize_error_code(
2459
+ error.code if error else None,
2460
+ error.type if error else None,
2461
+ )
2462
+ error_message = error.message if error else None
2463
+ account_model_retry = await _retry_account_model_rejection(
2464
+ tex,
2465
+ account,
2466
+ outcome="previsible",
2467
+ )
2468
+ if account_model_retry is not None:
2469
+ if not account_model_retry:
2470
+ raise
2471
+ # Leaving the same-account loop reaches
2472
+ # the outer account-selection ``continue``
2473
+ # below, where the rejected account is
2474
+ # already excluded by the helper.
2475
+ break
2476
+ if _facade()._is_security_work_authorization_required_error(code, error_message):
2477
+ if (
2478
+ account.security_work_authorized
2479
+ or account.id == file_preferred_account_id
2480
+ or require_preferred_account
2481
+ or attempt >= max_attempts - 1
2482
+ ):
2483
+ raise
2484
+ _facade().logger.info(
2485
+ "Retrying on security-work-authorized account request_id=%s account_id=%s",
2486
+ request_id,
2487
+ account.id,
2488
+ )
2489
+ yield format_sse_event(
2490
+ _facade()._security_work_advisory_event(
2491
+ code=_facade()._SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
2492
+ message=_facade()._SECURITY_WORK_RETRY_MESSAGE,
2493
+ request_id=request_id,
2494
+ action="retry_security_work_authorized",
2495
+ account_id=account.id,
2496
+ )
2497
+ )
2498
+ await _release_tracked_stream_lease(current_account_lease)
2499
+ current_account_lease = None
2500
+ excluded_account_ids.add(account.id)
2501
+ require_security_work_authorized = True
2502
+ last_security_work_retry_error = _RetryableStreamError(
2503
+ _facade()._SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
2504
+ _upstream_error_from_openai(error),
2505
+ )
2506
+ last_transient_exc = tex
2507
+ break
2508
+ if code == "account_response_create_cap":
2509
+ last_transient_exc = tex
2510
+ recovery_sleep_seconds = _account_selection_recovery_sleep_seconds(
2511
+ AccountSelection(
2512
+ account=None,
2513
+ error_message=error_message,
2514
+ error_code=code,
2515
+ )
2516
+ )
2517
+ if recovery_sleep_seconds is not None:
2518
+ can_try_other_account = (
2519
+ not require_preferred_account
2520
+ and account.id != file_preferred_account_id
2521
+ and attempt < max_attempts - 1
2522
+ )
2523
+ if can_try_other_account:
2524
+ deferred_capacity_account = account
2525
+ deferred_capacity_lease = current_account_lease
2526
+ excluded_account_ids.add(account.id)
2527
+ break
2528
+ remaining_budget_seconds = proxy._remaining_budget_seconds(deadline)
2529
+ if remaining_budget_seconds <= 0:
2530
+ raise
2531
+ async for wait_event in _iter_account_capacity_recovery_wait(
2532
+ request_id=request_id,
2533
+ model=payload.model,
2534
+ account_id=account.id,
2535
+ error_message=error_message,
2536
+ recovery_sleep_seconds=recovery_sleep_seconds,
2537
+ remaining_budget_seconds=remaining_budget_seconds,
2538
+ emit_keepalives=not propagate_http_errors
2539
+ or not enforce_openai_sdk_contract,
2540
+ stage="response_create",
2541
+ scheduler=scheduler,
2542
+ clock=clock,
2543
+ ):
2544
+ yield wait_event
2545
+ if proxy._remaining_budget_seconds(deadline) <= 0:
2546
+ raise
2547
+ continue
2548
+ last_transient_exc = tex
2549
+ await _release_tracked_stream_lease(current_account_lease)
2550
+ current_account_lease = None
2551
+ excluded_account_ids.add(account.id)
2552
+ break
2553
+ recovery_decision = await _wait_for_process_network_recovery(
2554
+ account,
2555
+ error_code=code,
2556
+ retryable_same_contract=tex.retryable_same_contract,
2557
+ failed_session=tex.failed_session,
2558
+ )
2559
+ if recovery_decision == "retry":
2560
+ continue
2561
+ if recovery_decision == "exhausted":
2562
+ _facade()._raise_proxy_budget_exhausted()
2563
+ if _facade()._is_account_neutral_error_code(code):
2564
+ raise
2565
+ if is_confirmed_pre_dispatch_transport_error(tex):
2566
+ # The transport proved the request never
2567
+ # dispatched: this account's proxy route is
2568
+ # dead. Release the account's stream lease
2569
+ # before recording health so its slot never
2570
+ # outlives the failed route, then jump
2571
+ # straight to the bounded transient backoff
2572
+ # floor so independent requests stop
2573
+ # rediscovering the dead route one generic
2574
+ # error at a time.
2575
+ await _release_tracked_stream_lease(current_account_lease)
2576
+ current_account_lease = None
2577
+ await _record_or_defer_confirmed_route_backoff(account)
2578
+ # A confirmed pre-dispatch connect failure
2579
+ # guarantees zero upstream bytes, so a
2580
+ # locally verified full-resend replay is
2581
+ # safe to move off the dead owner before
2582
+ # the hard-ownership raise (same file/
2583
+ # turn-state/single-account guards as the
2584
+ # post-visible failover path below).
2585
+ verified_owner_replay_moved = False
2586
+ if (
2587
+ attempt < max_attempts - 1
2588
+ and routing_strategy != "single_account"
2589
+ and file_preferred_account_id is None
2590
+ and turn_state_owner_account_id is None
2591
+ ):
2592
+ verified_owner_replay_moved = _move_verified_fresh_replay_from_owner(
2593
+ account_id=account.id,
2594
+ outcome="owner_pre_dispatch_proxy_connect_failure",
2595
+ )
2596
+ can_try_other_account = verified_owner_replay_moved or (
2597
+ not require_preferred_account
2598
+ and account.id != file_preferred_account_id
2599
+ and attempt < max_attempts - 1
2600
+ )
2601
+ if not can_try_other_account:
2602
+ # Hard account ownership or exhausted
2603
+ # attempts: fail closed on the original
2604
+ # sanitized failure without crossing
2605
+ # accounts.
2606
+ raise
2607
+ last_transient_exc = tex
2608
+ last_pre_dispatch_transport_error = tex
2609
+ transient_failed_account_id = account.id
2610
+ excluded_account_ids.add(account.id)
2611
+ if not verified_owner_replay_moved:
2612
+ affinity = replace(affinity, reallocate_sticky=True)
2613
+ _facade().logger.info(
2614
+ "Retrying stream after confirmed pre-dispatch proxy connect failure "
2615
+ "request_id=%s account_id=%s attempt=%d",
2616
+ request_id,
2617
+ account.id,
2618
+ attempt + 1,
2619
+ )
2620
+ break
2621
+ classified = classify_upstream_failure(
2622
+ error_code=code,
2623
+ error=_upstream_error_from_openai(error),
2624
+ http_status=tex.status_code,
2625
+ phase="first_event",
2626
+ )
2627
+ burst = is_upstream_burst_rejection(
2628
+ failure_class=classified["failure_class"],
2629
+ http_status=tex.status_code,
2630
+ )
2631
+ if resilience.deterministic_failover_enabled:
2632
+ action = failover_decision(
2633
+ failure_class=classified["failure_class"],
2634
+ downstream_visible=settlement.downstream_visible,
2635
+ candidates_remaining=max_attempts - attempt - 1,
2636
+ owner_bound=_stream_owner_bound_to(account),
2637
+ same_account_retry_available=_burst_same_account_retry_available(
2638
+ account, burst=burst
2639
+ ),
2640
+ )
2641
+ else:
2642
+ action = "surface"
2643
+ _facade().logger.info(
2644
+ "Failover decision request_id=%s transport=stream account_id=%s "
2645
+ "attempt=%d failure_class=%s action=%s",
2646
+ request_id,
2647
+ account.id,
2648
+ attempt + 1,
2649
+ classified["failure_class"],
2650
+ action,
2651
+ )
2652
+ if action == "retry_same_account":
2653
+ # Owner-bound burst 429: the request cannot
2654
+ # move, so back off briefly on the owner and
2655
+ # redispatch. Only the replica-local burst
2656
+ # cooldown is engaged per retry (steering
2657
+ # *other* unbound requests away at rejection
2658
+ # time, keyed or not); the transient error
2659
+ # penalty is written once, when the failure
2660
+ # is finally surfaced, so a bursting owner
2661
+ # is not pushed into selection error backoff
2662
+ # and a success is never followed by a
2663
+ # deferred penalty. The account is neither
2664
+ # excluded nor released from its dispatch
2665
+ # anchor, and the stream lease is kept so the
2666
+ # redispatch stays inside the admitted slot.
2667
+ await record_upstream_burst_rejection(
2668
+ proxy._load_balancer,
2669
+ account,
2670
+ retry_after_seconds=tex.retry_after_seconds,
2671
+ )
2672
+ retry_index = _next_burst_same_account_retry(account)
2673
+ delay = burst_same_account_backoff_seconds(
2674
+ retry_index,
2675
+ retry_after_seconds=tex.retry_after_seconds,
2676
+ )
2677
+ _facade().logger.info(
2678
+ "Burst 429 on owner-bound stream, retrying same account "
2679
+ "request_id=%s account_id=%s retry=%s/%s delay=%.2fs retry_after_seconds=%s",
2680
+ request_id,
2681
+ account.id,
2682
+ retry_index,
2683
+ BURST_SAME_ACCOUNT_MAX_RETRIES,
2684
+ delay,
2685
+ tex.retry_after_seconds,
2686
+ )
2687
+ async for wait_event in _iter_account_capacity_recovery_wait(
2688
+ request_id=request_id,
2689
+ model=payload.model,
2690
+ account_id=account.id,
2691
+ error_message=error_message,
2692
+ recovery_sleep_seconds=delay,
2693
+ remaining_budget_seconds=proxy._remaining_budget_seconds(deadline),
2694
+ # The wait is bounded (<= 10 s) so no
2695
+ # keepalive is needed; on the propagating
2696
+ # route a keepalive frame would commit
2697
+ # 200/SSE and make the surfaced 429
2698
+ # unreachable as an HTTP status.
2699
+ emit_keepalives=not propagate_http_errors,
2700
+ stage="burst_backoff",
2701
+ scheduler=scheduler,
2702
+ clock=clock,
2703
+ ):
2704
+ yield wait_event
2705
+ if proxy._remaining_budget_seconds(deadline) > 0:
2706
+ continue
2707
+ # Budget spent during the wait: surface the
2708
+ # original rejection below (one health write).
2709
+ if action == "failover_next":
2710
+ await _handle_or_defer_keyed_stream_health(
2711
+ account,
2712
+ _upstream_error_from_openai(error),
2713
+ code,
2714
+ http_status=tex.status_code,
2715
+ retry_after_seconds=tex.retry_after_seconds,
2716
+ )
2717
+ last_transient_exc = tex
2718
+ transient_failed_account_id = account.id
2719
+ await _release_tracked_stream_lease(current_account_lease)
2720
+ current_account_lease = None
2721
+ excluded_account_ids.add(account.id)
2722
+ _move_verified_fresh_replay_from_owner(
2723
+ account_id=account.id,
2724
+ outcome="owner_previsible_failure",
2725
+ )
2726
+ break
2727
+ # Workspace failures are terminal for this
2728
+ # owner; defer keyed health until settlement.
2729
+ # Preserve the existing policy for other codes.
2730
+ if classified["failure_class"] == "account_unavailable":
2731
+ await _handle_or_defer_keyed_stream_health(
2732
+ account,
2733
+ _upstream_error_from_openai(error),
2734
+ code,
2735
+ http_status=tex.status_code,
2736
+ retry_after_seconds=tex.retry_after_seconds,
2737
+ )
2738
+ else:
2739
+ await proxy._handle_stream_error(
2740
+ account,
2741
+ _upstream_error_from_openai(error),
2742
+ code,
2743
+ http_status=tex.status_code,
2744
+ **_retry_after_kwargs(tex.retry_after_seconds),
2745
+ )
2746
+ setattr(tex, _STREAM_HEALTH_RECORDED_ATTR, True)
2747
+ if burst:
2748
+ _stamp_surfaced_burst_retry_after(tex)
2749
+ raise
2750
+ error_payload: UpstreamError = (
2751
+ tex.error
2752
+ if isinstance(tex, _TransientStreamError)
2753
+ else _upstream_error_from_openai(_parse_openai_error(tex.payload))
2754
+ )
2755
+ error_code = _transient_retry_error_code(tex)
2756
+ error_message = str(error_payload.get("message") or "")
2757
+ recovery_decision = await _wait_for_process_network_recovery(
2758
+ account,
2759
+ error_code=error_code,
2760
+ retryable_same_contract=(
2761
+ isinstance(tex, ProxyResponseError) and tex.retryable_same_contract
2762
+ ),
2763
+ failed_session=tex.failed_session if isinstance(tex, ProxyResponseError) else None,
2764
+ )
2765
+ if recovery_decision == "retry":
2766
+ continue
2767
+ if recovery_decision == "exhausted":
2768
+ await _settle_process_network_budget_exhaustion(account, settlement)
2769
+ if preserve_native_failure_lifecycle:
2770
+ return
2771
+ yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
2772
+ return
2773
+ transient_retries += 1
2774
+ if (
2775
+ transient_retries < _facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES
2776
+ and proxy._remaining_budget_seconds(deadline) > 0
2777
+ and not settlement.downstream_visible
2778
+ ):
2779
+ delay = backoff_seconds(transient_retries)
2780
+ _facade().logger.info(
2781
+ "Transient stream error, retrying same account "
2782
+ "request_id=%s account_id=%s retry=%s/%s delay=%.2fs code=%s",
2783
+ request_id,
2784
+ account.id,
2785
+ transient_retries,
2786
+ _facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES,
2787
+ delay,
2788
+ error_code,
2789
+ )
2790
+ await scheduler.sleep(delay)
2791
+ continue # inner loop: retry same account
2792
+ # Exhausted same-account retries — penalize and failover
2793
+ _facade().logger.warning(
2794
+ "Transient retries exhausted for account "
2795
+ "request_id=%s account_id=%s retries=%s code=%s",
2796
+ request_id,
2797
+ account.id,
2798
+ transient_retries,
2799
+ error_code,
2800
+ )
2801
+ await _handle_or_defer_keyed_stream_health(
2802
+ account,
2803
+ error_payload,
2804
+ error_code,
2805
+ http_status=(tex.status_code if isinstance(tex, ProxyResponseError) else None),
2806
+ transient_retry_count=transient_retries,
2807
+ )
2808
+ # Preserve last ProxyResponseError for propagate_http_errors path.
2809
+ if isinstance(tex, ProxyResponseError):
2810
+ last_transient_exc = tex
2811
+ if isinstance(tex, _TransientStreamError) and (
2812
+ tex.preserve_on_selection_exhausted or error_code == "stream_incomplete"
2813
+ ):
2814
+ last_retryable_stream_error = _RetryableStreamError(
2815
+ error_code,
2816
+ error_payload,
2817
+ exclude_account=True,
2818
+ )
2819
+ await _release_tracked_stream_lease(current_account_lease)
2820
+ current_account_lease = None
2821
+ excluded_account_ids.add(account.id)
2822
+ break # outer loop: select different account
2823
+ finally:
2824
+ pop_stream_timeout_overrides(stream_timeout_tokens)
2825
+ if settlement.settlement_order_required:
2826
+ await _finalize_terminal_settlement_after_downstream_close(
2827
+ settlement,
2828
+ account,
2829
+ settlement_order_required=True,
2830
+ )
2831
+ else:
2832
+ settled = await _settle_stream_usage_before_pending_penalty(settlement)
2833
+ if settled and settlement.account_health_error:
2834
+ await proxy._handle_stream_error(
2835
+ account,
2836
+ _stream_settlement_error_payload(settlement),
2837
+ settlement.error_code or "upstream_error",
2838
+ )
2839
+ elif settled and settlement.record_success:
2840
+ await proxy._load_balancer.record_success(account)
2841
+ network_recovery.log_recovered()
2842
+ upstream_transport_metric_status = settlement.status
2843
+ _record_upstream_transport_metric_once(settlement.status)
2844
+ return
2845
+ continue # outer loop: account failover after transient exhaustion
2846
+ except _RetryableStreamError as exc:
2847
+ if _facade()._is_security_work_authorization_required_error(exc.code, exc.error.get("message")):
2848
+ if (
2849
+ account.security_work_authorized
2850
+ or account.id == file_preferred_account_id
2851
+ or require_preferred_account
2852
+ or attempt >= max_attempts - 1
2853
+ ):
2854
+ event = response_failed_event(
2855
+ exc.code,
2856
+ exc.error.get("message") or "Security work authorization is required",
2857
+ response_id=request_id,
2858
+ )
2859
+ yield format_sse_event(event)
2860
+ return
2861
+ _facade().logger.info(
2862
+ "Retrying on security-work-authorized account request_id=%s account_id=%s",
2863
+ request_id,
2864
+ account.id,
2865
+ )
2866
+ yield format_sse_event(
2867
+ _facade()._security_work_advisory_event(
2868
+ code=_facade()._SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
2869
+ message=_facade()._SECURITY_WORK_RETRY_MESSAGE,
2870
+ request_id=request_id,
2871
+ action="retry_security_work_authorized",
2872
+ account_id=account.id,
2873
+ )
2874
+ )
2875
+ await _release_tracked_stream_lease(current_account_lease)
2876
+ current_account_lease = None
2877
+ excluded_account_ids.add(account.id)
2878
+ require_security_work_authorized = True
2879
+ last_security_work_retry_error = exc
2880
+ continue
2881
+ await _handle_or_defer_keyed_stream_health(account, exc.error, exc.code)
2882
+ last_retryable_stream_error = exc
2883
+ if exc.exclude_account:
2884
+ await _release_tracked_stream_lease(current_account_lease)
2885
+ current_account_lease = None
2886
+ excluded_account_ids.add(account.id)
2887
+ _move_verified_fresh_replay_from_owner(
2888
+ account_id=account.id,
2889
+ outcome="owner_previsible_retryable_failure",
2890
+ )
2891
+ continue
2892
+ except _TerminalStreamError:
2893
+ if settlement.settlement_order_required:
2894
+ await _finalize_terminal_settlement_after_downstream_close(
2895
+ settlement,
2896
+ account,
2897
+ settlement_order_required=True,
2898
+ )
2899
+ else:
2900
+ health_write_allowed = await _settle_stream_usage_before_pending_penalty(settlement)
2901
+ if health_write_allowed and settlement.account_health_error:
2902
+ await proxy._handle_stream_error(
2903
+ account,
2904
+ _stream_settlement_error_payload(settlement),
2905
+ settlement.error_code or "upstream_error",
2906
+ )
2907
+ return
2908
+ except ProxyResponseError as exc:
2909
+ if _must_abort_native_transport_failure(
2910
+ exc,
2911
+ downstream_visible=settlement.downstream_visible,
2912
+ ):
2913
+ # A native Codex transport failure is ambiguous once the
2914
+ # helper may have dispatched the request. Re-raising the
2915
+ # original exception aborts the downstream stream without
2916
+ # replaying side effects on this or another account.
2917
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
2918
+ raise
2919
+ if _facade()._is_proxy_budget_exhausted_error(exc):
2920
+ await _settle_process_network_budget_exhaustion(account, settlement)
2921
+ if preserve_native_failure_lifecycle:
2922
+ return
2923
+ yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
2924
+ return
2925
+ account_model_retry = await _retry_account_model_rejection(
2926
+ exc,
2927
+ account,
2928
+ outcome="outer_proxy_error",
2929
+ )
2930
+ if account_model_retry:
2931
+ continue
2932
+ if account_model_retry is False:
2933
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
2934
+ if propagate_http_errors:
2935
+ raise
2936
+ yield await _render_account_model_rejection(exc, account_id=account.id)
2937
+ return
2938
+ if exc.status_code == 401:
2939
+ remaining_budget = proxy._remaining_budget_seconds(deadline)
2940
+ if remaining_budget <= 0:
2941
+ _facade().logger.warning(
2942
+ "Proxy request budget exhausted before forced refresh retry "
2943
+ "request_id=%s attempt=%s account_id=%s",
2944
+ request_id,
2945
+ attempt + 1,
2946
+ account.id,
2947
+ )
2948
+ await proxy._write_stream_preflight_error(
2949
+ affinity_observation=affinity_observation,
2950
+ account_id=account.id,
2951
+ api_key=api_key,
2952
+ request_id=request_id,
2953
+ model=payload.model,
2954
+ start=start,
2955
+ error_code="upstream_request_timeout",
2956
+ error_message="Proxy request budget exhausted",
2957
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
2958
+ service_tier=payload.service_tier,
2959
+ transport=request_transport,
2960
+ upstream_transport=upstream_stream_transport,
2961
+ useragent=useragent,
2962
+ useragent_group=useragent_group,
2963
+ conversation_id=conversation_id,
2964
+ client_ip=client_ip,
2965
+ )
2966
+ if preserve_native_failure_lifecycle:
2967
+ return
2968
+ yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
2969
+ return
2970
+ try:
2971
+ account = await proxy._ensure_fresh_with_budget(
2972
+ account,
2973
+ force=True,
2974
+ timeout_seconds=remaining_budget,
2975
+ )
2976
+ except (RefreshError, aiohttp.ClientError, asyncio.TimeoutError) as refresh_exc:
2977
+ if isinstance(refresh_exc, RefreshError):
2978
+ if refresh_exc.is_permanent:
2979
+ await proxy._load_balancer.mark_permanent_failure(account, refresh_exc.code)
2980
+ # Keep the warning account routable for later
2981
+ # requests, but exclude it from this request's
2982
+ # retry pool after upstream rejected its token.
2983
+ await _release_tracked_stream_lease(current_account_lease)
2984
+ current_account_lease = None
2985
+ excluded_account_ids.add(account.id)
2986
+ continue
2987
+ if is_transient_refresh_contention(refresh_exc):
2988
+ # Transient CROSS-REPLICA refresh contention on
2989
+ # the post-401 forced refresh: benign claim
2990
+ # contention OR a post-exchange persist/status
2991
+ # CAS conflict. This is NOT a genuine
2992
+ # ``transport_error`` OAuth failure — the
2993
+ # account's credentials are healthy — so fail
2994
+ # over WITHOUT an account-health penalty (no
2995
+ # ``_handle_stream_error``); the genuine
2996
+ # transport failure handled below keeps its
2997
+ # penalty. A post-exchange persist conflict is
2998
+ # logged distinctly (rarer, more-serious race).
2999
+ if refresh_contention_kind(refresh_exc) == "persist_conflict":
3000
+ logger.warning(
3001
+ "Stream post-401 forced-refresh post-exchange persist conflict "
3002
+ "code=%s account_id=%s",
3003
+ refresh_exc.code,
3004
+ account.id,
3005
+ )
3006
+ if not require_preferred_account and preferred_account_id is None:
3007
+ # Movable request: release the skipped
3008
+ # account's stream lease and fail over to a
3009
+ # different account. Record a retryable
3010
+ # upstream_unavailable so exhaustion surfaces
3011
+ # upstream_unavailable instead of a
3012
+ # misleading generic no_accounts response.
3013
+ await _release_tracked_stream_lease(current_account_lease)
3014
+ current_account_lease = None
3015
+ excluded_account_ids.add(account.id)
3016
+ last_retryable_stream_error = _RetryableStreamError(
3017
+ "upstream_unavailable",
3018
+ {
3019
+ "message": (
3020
+ refresh_exc.message
3021
+ or "Account refresh is temporarily unavailable; "
3022
+ "no healthy account could be reached."
3023
+ )
3024
+ },
3025
+ exclude_account=True,
3026
+ )
3027
+ continue
3028
+ # PINNED request: must not cross accounts. Stay
3029
+ # on the owner account, release the lease, and
3030
+ # surface a retryable ``upstream_unavailable``
3031
+ # promptly so the client can retry once the claim
3032
+ # clears.
3033
+ await _release_tracked_stream_lease(current_account_lease)
3034
+ current_account_lease = None
3035
+ message = (
3036
+ refresh_exc.message
3037
+ or "Account refresh is temporarily unavailable; retry later."
3038
+ )
3039
+ last_retryable_stream_error = _RetryableStreamError(
3040
+ "upstream_unavailable",
3041
+ {"message": message},
3042
+ )
3043
+ await proxy._write_stream_preflight_error(
3044
+ affinity_observation=affinity_observation,
3045
+ account_id=account.id,
3046
+ api_key=api_key,
3047
+ request_id=request_id,
3048
+ model=payload.model,
3049
+ start=start,
3050
+ error_code="upstream_unavailable",
3051
+ error_message=message,
3052
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
3053
+ service_tier=payload.service_tier,
3054
+ transport=request_transport,
3055
+ upstream_transport=upstream_stream_transport,
3056
+ useragent=useragent,
3057
+ useragent_group=useragent_group,
3058
+ conversation_id=conversation_id,
3059
+ client_ip=client_ip,
3060
+ )
3061
+ event = response_failed_event(
3062
+ "upstream_unavailable",
3063
+ message,
3064
+ response_id=request_id,
3065
+ )
3066
+ yield format_sse_event(event)
3067
+ return
3068
+ if not refresh_exc.transport_error:
3069
+ # Non-transport, non-permanent RefreshError:
3070
+ # release the stream lease and reselect.
3071
+ await _release_tracked_stream_lease(current_account_lease)
3072
+ current_account_lease = None
3073
+ continue
3074
+ # A GENUINE OAuth transport failure
3075
+ # (``code == "transport_error"``): the account/route
3076
+ # is at fault, so it falls through to the shared
3077
+ # transport-failure handling below — identical to a
3078
+ # raw aiohttp/connect failure — which records the
3079
+ # account-health penalty (``_handle_stream_error``)
3080
+ # so the broken account backs off instead of being
3081
+ # kept healthy and reselected on the next request.
3082
+ _facade().logger.warning(
3083
+ "Stream forced refresh/connect failed request_id=%s attempt=%s account_id=%s",
3084
+ request_id,
3085
+ attempt + 1,
3086
+ account.id,
3087
+ exc_info=True,
3088
+ )
3089
+ message = getattr(refresh_exc, "message", None) or str(refresh_exc)
3090
+ message = message or "Request to upstream timed out"
3091
+ if (
3092
+ not require_preferred_account
3093
+ and preferred_account_id is None
3094
+ and _facade()._should_retry_transient_stream_error("upstream_unavailable", message)
3095
+ and attempt + 1 < max_attempts
3096
+ ):
3097
+ await _handle_or_defer_keyed_stream_health(
3098
+ account,
3099
+ {"message": message},
3100
+ "upstream_unavailable",
3101
+ )
3102
+ last_retryable_stream_error = _RetryableStreamError(
3103
+ "upstream_unavailable",
3104
+ {"message": message},
3105
+ exclude_account=True,
3106
+ )
3107
+ # Release the excluded account's stream lease before
3108
+ # the failover ``continue`` so its stream-concurrency
3109
+ # slot is not held for the duration of the replacement
3110
+ # stream (matching the pre-refresh transport branch and
3111
+ # the claim-contention/permanent branches above).
3112
+ await _release_tracked_stream_lease(current_account_lease)
3113
+ current_account_lease = None
3114
+ excluded_account_ids.add(account.id)
3115
+ continue
3116
+ await proxy._write_stream_preflight_error(
3117
+ affinity_observation=affinity_observation,
3118
+ account_id=account.id,
3119
+ api_key=api_key,
3120
+ request_id=request_id,
3121
+ model=payload.model,
3122
+ start=start,
3123
+ error_code="upstream_unavailable",
3124
+ error_message=message,
3125
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
3126
+ service_tier=payload.service_tier,
3127
+ transport=request_transport,
3128
+ upstream_transport=upstream_stream_transport,
3129
+ useragent=useragent,
3130
+ useragent_group=useragent_group,
3131
+ conversation_id=conversation_id,
3132
+ client_ip=client_ip,
3133
+ )
3134
+ event = response_failed_event(
3135
+ "upstream_unavailable",
3136
+ message,
3137
+ response_id=request_id,
3138
+ )
3139
+ yield format_sse_event(event)
3140
+ return
3141
+ settlement = _StreamSettlement()
3142
+ effective_attempt_timeout = proxy._remaining_budget_seconds(deadline)
3143
+ if effective_attempt_timeout <= 0:
3144
+ _facade().logger.warning(
3145
+ "Proxy request budget exhausted before post-refresh stream attempt "
3146
+ "request_id=%s attempt=%s account_id=%s",
3147
+ request_id,
3148
+ attempt + 1,
3149
+ account.id,
3150
+ )
3151
+ await proxy._write_stream_preflight_error(
3152
+ affinity_observation=affinity_observation,
3153
+ account_id=account.id,
3154
+ api_key=api_key,
3155
+ request_id=request_id,
3156
+ model=payload.model,
3157
+ start=start,
3158
+ error_code="upstream_request_timeout",
3159
+ error_message="Proxy request budget exhausted",
3160
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
3161
+ service_tier=payload.service_tier,
3162
+ transport=request_transport,
3163
+ upstream_transport=upstream_stream_transport,
3164
+ useragent=useragent,
3165
+ useragent_group=useragent_group,
3166
+ conversation_id=conversation_id,
3167
+ client_ip=client_ip,
3168
+ )
3169
+ if preserve_native_failure_lifecycle:
3170
+ return
3171
+ yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
3172
+ return
3173
+ try:
3174
+ can_try_other_account = (
3175
+ not require_preferred_account
3176
+ and account.id != file_preferred_account_id
3177
+ and attempt < max_attempts - 1
3178
+ )
3179
+ post_refresh_stream = _stream_post_refresh_with_capacity_recovery(
3180
+ account,
3181
+ settlement=settlement,
3182
+ can_try_other_account=can_try_other_account,
3183
+ tool_call_dedupe=tool_call_dedupe,
3184
+ )
3185
+ try:
3186
+ async for line in post_refresh_stream:
3187
+ yield line
3188
+ finally:
3189
+ # Closing this generator runs its internal
3190
+ # cancellation-safe close/terminal finalization;
3191
+ # without an owned aclose() the child would stay
3192
+ # suspended after a downstream disconnect.
3193
+ close_task = scheduler.create_task(
3194
+ post_refresh_stream.aclose(),
3195
+ name=f"stream-post-refresh-outer-close-{request_id}",
3196
+ )
3197
+ _, close_cancellation = await _await_task_deferring_cancellation(close_task)
3198
+ if close_cancellation is not None:
3199
+ raise close_cancellation
3200
+ except ProxyResponseError as retry_exc:
3201
+ if _must_abort_native_transport_failure(
3202
+ retry_exc,
3203
+ downstream_visible=settlement.downstream_visible,
3204
+ ):
3205
+ raise
3206
+ if _facade()._is_proxy_budget_exhausted_error(retry_exc):
3207
+ await _settle_process_network_budget_exhaustion(account, settlement)
3208
+ if preserve_native_failure_lifecycle:
3209
+ return
3210
+ yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
3211
+ return
3212
+ if settlement.downstream_visible:
3213
+ failed_response_id = settlement.response_id or request_id
3214
+ error = _parse_openai_error(retry_exc.payload)
3215
+ error_code = _normalize_error_code(
3216
+ error.code if error else None,
3217
+ error.type if error else None,
3218
+ )
3219
+ error_message = error.message if error else "Upstream error"
3220
+ event = response_failed_event(
3221
+ error_code or "upstream_error",
3222
+ error_message or "Upstream error",
3223
+ error_type=(error.type if error else None) or "server_error",
3224
+ response_id=failed_response_id,
3225
+ error_param=error.param_state if error else None,
3226
+ )
3227
+ _apply_error_metadata(event["response"]["error"], error)
3228
+ _facade().logger.warning(
3229
+ "Surfacing post-refresh stream failure without replay "
3230
+ "request_id=%s account_id=%s code=%s",
3231
+ request_id,
3232
+ account.id,
3233
+ error_code,
3234
+ )
3235
+ yield format_sse_event(event)
3236
+ settlement.record_success = False
3237
+ settlement.error_code = error_code
3238
+ settlement.error_message = error_message
3239
+ settlement.error = _upstream_error_from_openai(error)
3240
+ settlement.account_health_error = _facade()._should_penalize_stream_error(
3241
+ error_code, error_message
3242
+ )
3243
+ settled = await _settle_stream_usage_before_pending_penalty(settlement)
3244
+ if settled and settlement.account_health_error:
3245
+ await proxy._handle_stream_error(
3246
+ account,
3247
+ _stream_settlement_error_payload(settlement),
3248
+ settlement.error_code or "upstream_error",
3249
+ http_status=retry_exc.status_code,
3250
+ )
3251
+ return
3252
+ error = _parse_openai_error(retry_exc.payload)
3253
+ error_code = _normalize_error_code(
3254
+ error.code if error else None,
3255
+ error.type if error else None,
3256
+ )
3257
+ if getattr(retry_exc, _POST_REFRESH_TRANSIENT_EXHAUSTED_ATTR, False):
3258
+ transient_error_payload = _stream_settlement_error_payload(settlement)
3259
+ last_transient_exc = retry_exc
3260
+ last_retryable_stream_error = _RetryableStreamError(
3261
+ settlement.error_code or error_code or "upstream_error",
3262
+ transient_error_payload,
3263
+ )
3264
+ await _release_tracked_stream_lease(current_account_lease)
3265
+ current_account_lease = None
3266
+ excluded_account_ids.add(account.id)
3267
+ continue
3268
+ account_model_retry = await _retry_account_model_rejection(
3269
+ retry_exc,
3270
+ account,
3271
+ outcome="post_refresh",
3272
+ )
3273
+ if account_model_retry:
3274
+ continue
3275
+ if account_model_retry is False:
3276
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
3277
+ if propagate_http_errors:
3278
+ raise
3279
+ yield await _render_account_model_rejection(
3280
+ retry_exc,
3281
+ account_id=account.id,
3282
+ )
3283
+ return
3284
+ if error_code == "account_response_create_cap":
3285
+ last_transient_exc = retry_exc
3286
+ if can_try_other_account:
3287
+ deferred_capacity_account = account
3288
+ deferred_capacity_lease = current_account_lease
3289
+ excluded_account_ids.add(account.id)
3290
+ continue
3291
+ # The same-account helper only re-raises this
3292
+ # neutral cap when recovery cannot continue
3293
+ # within the original budget. Exit the account
3294
+ # loop so the preserved cap is propagated or
3295
+ # rendered below, instead of replacing it with
3296
+ # a next-attempt timeout.
3297
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
3298
+ break
3299
+ if _facade()._is_account_neutral_error_code(error_code):
3300
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
3301
+ raise
3302
+ if is_confirmed_pre_dispatch_transport_error(retry_exc):
3303
+ # This retry still failed before dispatch. Handle
3304
+ # the proven dead route before the generic
3305
+ # failover policy: that policy does not know
3306
+ # about hard account ownership and may otherwise
3307
+ # cross a previous-response, turn-state, file, or
3308
+ # single-account boundary.
3309
+ await _release_tracked_stream_lease(current_account_lease)
3310
+ current_account_lease = None
3311
+ await _record_or_defer_confirmed_route_backoff(account)
3312
+ last_transient_exc = retry_exc
3313
+ last_pre_dispatch_transport_error = retry_exc
3314
+
3315
+ verified_owner_replay_moved = False
3316
+ if (
3317
+ attempt < max_attempts - 1
3318
+ and routing_strategy != "single_account"
3319
+ and file_preferred_account_id is None
3320
+ and turn_state_owner_account_id is None
3321
+ ):
3322
+ verified_owner_replay_moved = _move_verified_fresh_replay_from_owner(
3323
+ account_id=account.id,
3324
+ outcome="owner_post_refresh_proxy_connect_failure",
3325
+ )
3326
+
3327
+ can_try_other_account = bool(
3328
+ attempt < max_attempts - 1
3329
+ and routing_strategy != "single_account"
3330
+ and file_preferred_account_id is None
3331
+ and turn_state_owner_account_id is None
3332
+ and not require_preferred_account
3333
+ )
3334
+ if can_try_other_account:
3335
+ excluded_account_ids.add(account.id)
3336
+ if not verified_owner_replay_moved:
3337
+ affinity = replace(affinity, reallocate_sticky=True)
3338
+ _facade().logger.info(
3339
+ "Retrying post-refresh stream after confirmed pre-dispatch proxy "
3340
+ "connect failure request_id=%s account_id=%s attempt=%d",
3341
+ request_id,
3342
+ account.id,
3343
+ attempt + 1,
3344
+ )
3345
+ continue
3346
+
3347
+ # Hard ownership or an exhausted attempt budget:
3348
+ # stop here. The shared terminal path settles the
3349
+ # reservation, drains the deferred backoff floor,
3350
+ # and preserves this sanitized failure.
3351
+ break
3352
+ current_error_payload = _upstream_error_from_openai(error)
3353
+ current_error_code = error_code or "upstream_error"
3354
+ classified = classify_upstream_failure(
3355
+ error_code=current_error_code,
3356
+ error=current_error_payload,
3357
+ http_status=retry_exc.status_code,
3358
+ phase="first_event",
3359
+ )
3360
+ candidates_remaining = max_attempts - attempt - 1
3361
+ burst = is_upstream_burst_rejection(
3362
+ failure_class=classified["failure_class"],
3363
+ http_status=retry_exc.status_code,
3364
+ )
3365
+ if retry_exc.status_code == 401 and candidates_remaining > 0:
3366
+ action = "failover_next"
3367
+ elif resilience.deterministic_failover_enabled:
3368
+ action = failover_decision(
3369
+ failure_class=classified["failure_class"],
3370
+ downstream_visible=False,
3371
+ candidates_remaining=candidates_remaining,
3372
+ owner_bound=_stream_owner_bound_to(account),
3373
+ # The post-refresh redispatch re-enters the
3374
+ # attempt loop, so it also needs an attempt.
3375
+ same_account_retry_available=(
3376
+ candidates_remaining > 0
3377
+ and _burst_same_account_retry_available(account, burst=burst)
3378
+ ),
3379
+ )
3380
+ else:
3381
+ action = "surface"
3382
+ _facade().logger.info(
3383
+ "Failover decision request_id=%s transport=stream account_id=%s "
3384
+ "attempt=%d phase=post_refresh failure_class=%s action=%s",
3385
+ request_id,
3386
+ account.id,
3387
+ attempt + 1,
3388
+ classified["failure_class"],
3389
+ action,
3390
+ )
3391
+ if action == "retry_same_account":
3392
+ # Owner-bound burst 429 after the forced refresh:
3393
+ # keep the dispatch anchor and let the next
3394
+ # attempt re-select the same owner (it is not
3395
+ # excluded) after a short backoff. As on the
3396
+ # pre-visible path only the burst cooldown is
3397
+ # engaged here; the transient penalty is written
3398
+ # once if the failure is finally surfaced.
3399
+ await record_upstream_burst_rejection(
3400
+ proxy._load_balancer,
3401
+ account,
3402
+ retry_after_seconds=retry_exc.retry_after_seconds,
3403
+ )
3404
+ await _release_tracked_stream_lease(current_account_lease)
3405
+ current_account_lease = None
3406
+ retry_index = _next_burst_same_account_retry(account)
3407
+ delay = burst_same_account_backoff_seconds(
3408
+ retry_index,
3409
+ retry_after_seconds=retry_exc.retry_after_seconds,
3410
+ )
3411
+ # A failed re-selection re-raises this exception:
3412
+ # stamp Retry-After now (after the delay was
3413
+ # derived from the upstream value).
3414
+ _stamp_surfaced_burst_retry_after(retry_exc)
3415
+ last_transient_exc = retry_exc
3416
+ _facade().logger.info(
3417
+ "Burst 429 on owner-bound stream, retrying same account "
3418
+ "request_id=%s account_id=%s phase=post_refresh retry=%s/%s delay=%.2fs "
3419
+ "retry_after_seconds=%s",
3420
+ request_id,
3421
+ account.id,
3422
+ retry_index,
3423
+ BURST_SAME_ACCOUNT_MAX_RETRIES,
3424
+ delay,
3425
+ retry_exc.retry_after_seconds,
3426
+ )
3427
+ async for wait_event in _iter_account_capacity_recovery_wait(
3428
+ request_id=request_id,
3429
+ model=payload.model,
3430
+ account_id=account.id,
3431
+ error_message=error.message if error else None,
3432
+ recovery_sleep_seconds=delay,
3433
+ remaining_budget_seconds=proxy._remaining_budget_seconds(deadline),
3434
+ emit_keepalives=not propagate_http_errors,
3435
+ stage="burst_backoff",
3436
+ scheduler=scheduler,
3437
+ clock=clock,
3438
+ ):
3439
+ yield wait_event
3440
+ if proxy._remaining_budget_seconds(deadline) > 0:
3441
+ continue
3442
+ # Budget spent during the wait: surface the
3443
+ # original rejection below (one health write).
3444
+ if action == "failover_next":
3445
+ await _handle_or_defer_keyed_stream_health(
3446
+ account,
3447
+ current_error_payload,
3448
+ current_error_code,
3449
+ http_status=retry_exc.status_code,
3450
+ retry_after_seconds=retry_exc.retry_after_seconds,
3451
+ )
3452
+ last_transient_exc = retry_exc
3453
+ await _release_tracked_stream_lease(current_account_lease)
3454
+ current_account_lease = None
3455
+ _move_verified_fresh_replay_from_owner(
3456
+ account_id=account.id,
3457
+ outcome="owner_post_refresh_failure",
3458
+ )
3459
+ excluded_account_ids.add(account.id)
3460
+ continue
3461
+ health_write_allowed = await _drain_pending_post_refresh_penalty_on_terminal(settlement)
3462
+ if health_write_allowed:
3463
+ await proxy._handle_stream_error(
3464
+ account,
3465
+ current_error_payload,
3466
+ current_error_code,
3467
+ http_status=retry_exc.status_code,
3468
+ **_retry_after_kwargs(retry_exc.retry_after_seconds),
3469
+ )
3470
+ setattr(retry_exc, _STREAM_HEALTH_RECORDED_ATTR, True)
3471
+ if burst:
3472
+ _stamp_surfaced_burst_retry_after(retry_exc)
3473
+ if propagate_http_errors:
3474
+ raise
3475
+ error_message = error.message if error else None
3476
+ event = response_failed_event(
3477
+ error_code or "upstream_error",
3478
+ error_message or "Upstream error",
3479
+ error_type=(error.type if error else None) or "server_error",
3480
+ response_id=request_id,
3481
+ error_param=error.param_state if error else None,
3482
+ )
3483
+ _apply_error_metadata(event["response"]["error"], error)
3484
+ yield format_sse_event(event)
3485
+ return
3486
+ current_account_penalty_queued = any(
3487
+ failed_account is account
3488
+ for failed_account, *_rest in pending_post_refresh_transient_penalties
3489
+ )
3490
+ ordered_settlement_required = bool(
3491
+ pending_post_refresh_transient_penalties or deferred_account_error_backoffs
3492
+ )
3493
+ if ordered_settlement_required:
3494
+ health_write_allowed = await _drain_pending_post_refresh_penalty_on_terminal(settlement)
3495
+ if (
3496
+ health_write_allowed
3497
+ and settlement.account_health_error
3498
+ and not current_account_penalty_queued
3499
+ ):
3500
+ await proxy._handle_stream_error(
3501
+ account,
3502
+ _stream_settlement_error_payload(settlement),
3503
+ settlement.error_code or "upstream_error",
3504
+ )
3505
+ elif health_write_allowed and settlement.record_success:
3506
+ await proxy._load_balancer.record_success(account)
3507
+ else:
3508
+ await _finalize_terminal_settlement_after_downstream_close(
3509
+ settlement,
3510
+ account,
3511
+ settlement_order_required=True,
3512
+ )
3513
+ upstream_transport_metric_status = settlement.status
3514
+ _record_upstream_transport_metric_once(settlement.status)
3515
+ return
3516
+ error = _parse_openai_error(exc.payload)
3517
+ error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
3518
+ error_message = error.message if error else None
3519
+ error_type = error.type if error else None
3520
+ error_param = error.param_state if error else None
3521
+ if _facade()._is_security_work_authorization_required_error(error_code, error_message):
3522
+ if (
3523
+ not account.security_work_authorized
3524
+ and account.id != file_preferred_account_id
3525
+ and not require_preferred_account
3526
+ and attempt < max_attempts - 1
3527
+ ):
3528
+ _facade().logger.info(
3529
+ "Retrying on security-work-authorized account request_id=%s account_id=%s",
3530
+ request_id,
3531
+ account.id,
3532
+ )
3533
+ yield format_sse_event(
3534
+ _facade()._security_work_advisory_event(
3535
+ code=_facade()._SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
3536
+ message=_facade()._SECURITY_WORK_RETRY_MESSAGE,
3537
+ request_id=request_id,
3538
+ action="retry_security_work_authorized",
3539
+ account_id=account.id,
3540
+ )
3541
+ )
3542
+ await _release_tracked_stream_lease(current_account_lease)
3543
+ current_account_lease = None
3544
+ excluded_account_ids.add(account.id)
3545
+ require_security_work_authorized = True
3546
+ continue
3547
+ health_write_allowed = await _drain_pending_post_refresh_penalty_on_terminal(settlement)
3548
+ if (
3549
+ health_write_allowed
3550
+ and not getattr(exc, _STREAM_HEALTH_RECORDED_ATTR, False)
3551
+ and _facade()._should_penalize_stream_error(error_code, error_message)
3552
+ ):
3553
+ await proxy._handle_stream_error(
3554
+ account,
3555
+ _upstream_error_from_openai(error),
3556
+ error_code,
3557
+ # Evidence only: this failure carries an upstream
3558
+ # HTTP status, so it is not a status-less terminal
3559
+ # and must stay out of the soft-overload window.
3560
+ # Passing it positionally would change the neutral
3561
+ # rejection predicates and the 429 branch too.
3562
+ upstream_http_status=exc.status_code,
3563
+ )
3564
+ if propagate_http_errors:
3565
+ raise
3566
+ event = response_failed_event(
3567
+ error_code,
3568
+ error_message or "Upstream error",
3569
+ error_type=error_type or "server_error",
3570
+ response_id=request_id,
3571
+ error_param=error_param,
3572
+ )
3573
+ _apply_error_metadata(event["response"]["error"], error)
3574
+ yield format_sse_event(event)
3575
+ return
3576
+ except RefreshError as exc:
3577
+ if exc.is_permanent:
3578
+ await proxy._load_balancer.mark_permanent_failure(account, exc.code)
3579
+ await _release_tracked_stream_lease(current_account_lease)
3580
+ current_account_lease = None
3581
+ excluded_account_ids.add(account.id)
3582
+ continue
3583
+ except Exception:
3584
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
3585
+ _facade().logger.warning(
3586
+ "Proxy streaming failed without retry account_id=%s request_id=%s",
3587
+ account_id_value,
3588
+ request_id,
3589
+ exc_info=True,
3590
+ )
3591
+ event = response_failed_event(
3592
+ "upstream_error",
3593
+ "Proxy streaming failed",
3594
+ response_id=request_id,
3595
+ )
3596
+ yield format_sse_event(event)
3597
+ return
3598
+ await _drain_pending_post_refresh_penalty_on_terminal(settlement)
3599
+ # When HTTP error propagation is enabled and the last failure was
3600
+ # a transient 500, re-raise to preserve the upstream status/payload.
3601
+ if last_account_model_rejection is not None:
3602
+ if propagate_http_errors:
3603
+ raise last_account_model_rejection
3604
+ yield await _render_account_model_rejection(
3605
+ last_account_model_rejection,
3606
+ account_id=last_account_model_rejection_account_id,
3607
+ )
3608
+ return
3609
+ if propagate_http_errors and last_transient_exc is not None:
3610
+ raise last_transient_exc
3611
+ if last_pre_dispatch_transport_error is not None:
3612
+ # Attempt budget exhausted after confirmed pre-dispatch route
3613
+ # failures: surface the original sanitized failure rather than
3614
+ # a generated ``no_accounts`` response.
3615
+ yield _render_dispatch_transport_error(last_pre_dispatch_transport_error)
3616
+ return
3617
+ if last_retryable_stream_error is not None:
3618
+ retries_exhausted_msg = str(last_retryable_stream_error.error.get("message") or "Upstream error")
3619
+ event = response_failed_event(
3620
+ last_retryable_stream_error.code,
3621
+ retries_exhausted_msg,
3622
+ response_id=request_id,
3623
+ )
3624
+ if last_retryable_stream_error.code in SYNTHETIC_TRANSPORT_FAILURE_CODES:
3625
+ event = synthetic_transport_failure_event(event)
3626
+ if not (
3627
+ preserve_native_failure_lifecycle
3628
+ and last_retryable_stream_error.code in SYNTHETIC_TRANSPORT_FAILURE_CODES
3629
+ ):
3630
+ yield format_sse_event(event)
3631
+ if not any_attempt_logged:
3632
+ await proxy._write_request_log(
3633
+ affinity_observation=affinity_observation,
3634
+ account_id=None,
3635
+ api_key=api_key,
3636
+ request_id=request_id,
3637
+ model=payload.model,
3638
+ latency_ms=int((clock.monotonic() - start) * 1000),
3639
+ status="error",
3640
+ error_code=last_retryable_stream_error.code,
3641
+ error_message=retries_exhausted_msg,
3642
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
3643
+ transport=request_transport,
3644
+ upstream_transport=upstream_stream_transport,
3645
+ service_tier=payload.service_tier,
3646
+ requested_service_tier=payload.service_tier,
3647
+ useragent=useragent,
3648
+ useragent_group=useragent_group,
3649
+ conversation_id=conversation_id,
3650
+ client_ip=client_ip,
3651
+ )
3652
+ return
3653
+ if last_transient_exc is not None:
3654
+ error = _parse_openai_error(last_transient_exc.payload)
3655
+ error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
3656
+ if error_code == "account_response_create_cap":
3657
+ error_message = error.message if error else None
3658
+ event = response_failed_event(
3659
+ error_code,
3660
+ error_message or "Account response-create concurrency limit reached",
3661
+ error_type=(error.type if error else None) or "server_error",
3662
+ response_id=request_id,
3663
+ error_param=error.param_state if error else None,
3664
+ )
3665
+ _apply_error_metadata(event["response"]["error"], error)
3666
+ yield format_sse_event(event)
3667
+ return
3668
+
3669
+ retries_exhausted_msg = "No available accounts after retries"
3670
+ _facade().logger.warning(
3671
+ "Proxy streaming exhausted accounts request_id=%s model=%s transport=%s attempts=%s "
3672
+ "excluded_count=%s preferred_account_id=%s api_key_present=%s",
3673
+ request_id,
3674
+ payload.model,
3675
+ request_transport,
3676
+ attempt,
3677
+ len(excluded_account_ids),
3678
+ preferred_account_id,
3679
+ api_key is not None,
3680
+ )
3681
+ event = response_failed_event(
3682
+ "no_accounts",
3683
+ retries_exhausted_msg,
3684
+ response_id=request_id,
3685
+ )
3686
+ yield format_sse_event(event)
3687
+ if not any_attempt_logged:
3688
+ await proxy._write_request_log(
3689
+ affinity_observation=affinity_observation,
3690
+ account_id=None,
3691
+ api_key=api_key,
3692
+ request_id=request_id,
3693
+ model=payload.model,
3694
+ latency_ms=int((clock.monotonic() - start) * 1000),
3695
+ status="error",
3696
+ error_code="no_accounts",
3697
+ error_message=retries_exhausted_msg,
3698
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
3699
+ transport=request_transport,
3700
+ upstream_transport=upstream_stream_transport,
3701
+ service_tier=payload.service_tier,
3702
+ requested_service_tier=payload.service_tier,
3703
+ useragent=useragent,
3704
+ useragent_group=useragent_group,
3705
+ conversation_id=conversation_id,
3706
+ client_ip=client_ip,
3707
+ )
3708
+ finally:
3709
+ if not upstream_transport_metric_recorded:
3710
+ _record_upstream_transport_metric_once(upstream_transport_metric_status or "error")
3711
+ for account_lease in account_leases:
3712
+ await proxy._load_balancer.release_account_lease(account_lease)
3713
+ if (
3714
+ not settled
3715
+ and not settlement.usage_settlement_transferred
3716
+ and api_key is not None
3717
+ and api_key_reservation is not None
3718
+ ):
3719
+
3720
+ async def _release_reservation_then_drain_backoffs() -> None:
3721
+ nonlocal settled
3722
+ if pending_post_refresh_transient_penalties:
3723
+ # Mid-loop keyed health may already be queued. Prefer
3724
+ # settle-then-flush so cancel cleanup still applies the
3725
+ # deferred account penalty after the reservation closes.
3726
+ settled = await _drain_pending_post_refresh_penalty_on_terminal(settlement)
3727
+ if settled:
3728
+ return
3729
+ released = await proxy._release_unsettled_stream_api_key_usage(
3730
+ api_key=api_key,
3731
+ api_key_reservation=api_key_reservation,
3732
+ request_id=request_id,
3733
+ )
3734
+ if released:
3735
+ try:
3736
+ await proxy._drain_deferred_account_error_backoffs(deferred_account_error_backoffs)
3737
+ finally:
3738
+ # Confirmed release owns the same independent
3739
+ # post-settlement health lanes as confirmed settle.
3740
+ await _flush_or_schedule_pending_post_refresh_penalties()
3741
+
3742
+ release_coro = _release_reservation_then_drain_backoffs()
3743
+ current_task = asyncio.current_task()
3744
+ if current_task is not None and current_task.cancelling():
3745
+ proxy._schedule_cancel_safe_cleanup(
3746
+ release_coro,
3747
+ action="release_stream_api_key_reservation",
3748
+ request_id=request_id,
3749
+ )
3750
+ else:
3751
+ await release_coro
3752
+ elif settled and deferred_account_error_backoffs:
3753
+ try:
3754
+ await proxy._drain_deferred_account_error_backoffs(deferred_account_error_backoffs)
3755
+ finally:
3756
+ # A retained backoff failure must not block ownership of
3757
+ # later penalties left by a cancelled flush.
3758
+ await _flush_or_schedule_pending_post_refresh_penalties()
3759
+ elif settled:
3760
+ # Finish penalties left behind if a prior flush was cancelled
3761
+ # mid-loop after settlement already closed the reservation.
3762
+ await _flush_or_schedule_pending_post_refresh_penalties()