codexLB-up 1.25.0b10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (827) hide show
  1. app/__init__.py +10 -0
  2. app/admin_cli.py +494 -0
  3. app/cli.py +339 -0
  4. app/codex_sessions_retag.py +487 -0
  5. app/core/__init__.py +0 -0
  6. app/core/audit/__init__.py +0 -0
  7. app/core/audit/service.py +216 -0
  8. app/core/audit/types.py +94 -0
  9. app/core/auth/__init__.py +236 -0
  10. app/core/auth/api_key_cache.py +65 -0
  11. app/core/auth/dashboard_access.py +429 -0
  12. app/core/auth/dashboard_mode.py +171 -0
  13. app/core/auth/dashboard_session_ttl.py +74 -0
  14. app/core/auth/dashboard_users_cache.py +120 -0
  15. app/core/auth/dependencies.py +710 -0
  16. app/core/auth/external_identity.py +78 -0
  17. app/core/auth/guardian.py +374 -0
  18. app/core/auth/models.py +49 -0
  19. app/core/auth/providers/__init__.py +129 -0
  20. app/core/auth/providers/oidc.py +838 -0
  21. app/core/auth/providers/registry.py +118 -0
  22. app/core/auth/refresh.py +394 -0
  23. app/core/auth/step_up.py +117 -0
  24. app/core/auth/totp.py +81 -0
  25. app/core/balancer/__init__.py +71 -0
  26. app/core/balancer/logic.py +1612 -0
  27. app/core/balancer/rendezvous_hash.py +21 -0
  28. app/core/balancer/types.py +21 -0
  29. app/core/bootstrap.py +172 -0
  30. app/core/cache/invalidation.py +426 -0
  31. app/core/clients/__init__.py +0 -0
  32. app/core/clients/codex.py +996 -0
  33. app/core/clients/codex_version.py +187 -0
  34. app/core/clients/codex_version_snapshot.py +2 -0
  35. app/core/clients/files.py +534 -0
  36. app/core/clients/headers.py +29 -0
  37. app/core/clients/http.py +511 -0
  38. app/core/clients/model_fetcher.py +276 -0
  39. app/core/clients/native_egress.py +1304 -0
  40. app/core/clients/oauth.py +477 -0
  41. app/core/clients/proxy.py +6089 -0
  42. app/core/clients/proxy_websocket.py +1411 -0
  43. app/core/clients/rate_limit_reset_credits.py +412 -0
  44. app/core/clients/stream_errors.py +12 -0
  45. app/core/clients/thread_cache_identity.py +347 -0
  46. app/core/clients/usage.py +467 -0
  47. app/core/clock.py +170 -0
  48. app/core/config/__init__.py +0 -0
  49. app/core/config/background_jobs.py +73 -0
  50. app/core/config/context_window_overrides.py +166 -0
  51. app/core/config/dashboard_overrides.py +145 -0
  52. app/core/config/inheritable.py +47 -0
  53. app/core/config/key_fingerprint.py +115 -0
  54. app/core/config/settings.py +872 -0
  55. app/core/config/settings_cache.py +100 -0
  56. app/core/config/spool_retention.py +217 -0
  57. app/core/config/tiers.py +174 -0
  58. app/core/conversation_archive.py +568 -0
  59. app/core/crypto.py +37 -0
  60. app/core/draining.py +11 -0
  61. app/core/errors.py +365 -0
  62. app/core/exceptions.py +155 -0
  63. app/core/handlers/__init__.py +3 -0
  64. app/core/handlers/exceptions.py +478 -0
  65. app/core/http_protocol.py +176 -0
  66. app/core/http_protocol_httptools.py +204 -0
  67. app/core/ingress_limits.py +17 -0
  68. app/core/metadata_store.py +18 -0
  69. app/core/metrics/__init__.py +0 -0
  70. app/core/metrics/middleware.py +74 -0
  71. app/core/metrics/prometheus.py +604 -0
  72. app/core/middleware/__init__.py +25 -0
  73. app/core/middleware/api_firewall.py +111 -0
  74. app/core/middleware/app_version.py +37 -0
  75. app/core/middleware/dashboard_auth_proxy.py +55 -0
  76. app/core/middleware/dashboard_csrf.py +177 -0
  77. app/core/middleware/dashboard_gzip.py +41 -0
  78. app/core/middleware/dashboard_overrides.py +59 -0
  79. app/core/middleware/firewall_cache.py +91 -0
  80. app/core/middleware/inflight.py +126 -0
  81. app/core/middleware/multipart_content_encoding.py +129 -0
  82. app/core/middleware/path_rewrite.py +148 -0
  83. app/core/middleware/request_body_limit.py +196 -0
  84. app/core/middleware/request_decompression.py +209 -0
  85. app/core/middleware/request_id.py +72 -0
  86. app/core/middleware/required_capability_http.py +116 -0
  87. app/core/middleware/trusted_proxy_headers.py +36 -0
  88. app/core/multipart.py +237 -0
  89. app/core/multipart_fields.py +91 -0
  90. app/core/openai/__init__.py +0 -0
  91. app/core/openai/chat_requests.py +493 -0
  92. app/core/openai/chat_responses.py +931 -0
  93. app/core/openai/contracts.py +46 -0
  94. app/core/openai/exceptions.py +24 -0
  95. app/core/openai/host_models.py +14 -0
  96. app/core/openai/images.py +413 -0
  97. app/core/openai/message_coercion.py +376 -0
  98. app/core/openai/model_refresh_scheduler.py +423 -0
  99. app/core/openai/model_registry.py +997 -0
  100. app/core/openai/model_registry_store.py +548 -0
  101. app/core/openai/models.py +172 -0
  102. app/core/openai/parsing.py +97 -0
  103. app/core/openai/requests.py +1980 -0
  104. app/core/openai/strict_schema.py +199 -0
  105. app/core/openai/tool_call_safety.py +99 -0
  106. app/core/openai/v1_requests.py +187 -0
  107. app/core/plan_types.py +90 -0
  108. app/core/prestop.py +202 -0
  109. app/core/rate_limiter/__init__.py +0 -0
  110. app/core/rate_limiter/db_rate_limiter.py +215 -0
  111. app/core/request_locality.py +473 -0
  112. app/core/resilience/__init__.py +0 -0
  113. app/core/resilience/backpressure.py +57 -0
  114. app/core/resilience/bulkhead.py +202 -0
  115. app/core/resilience/circuit_breaker.py +150 -0
  116. app/core/resilience/degradation.py +55 -0
  117. app/core/resilience/loop_lag_monitor.py +60 -0
  118. app/core/resilience/memory_monitor.py +137 -0
  119. app/core/resilience/network_recovery.py +256 -0
  120. app/core/resilience/overload.py +98 -0
  121. app/core/resilience/toggles.py +114 -0
  122. app/core/retention/__init__.py +0 -0
  123. app/core/retention/job.py +240 -0
  124. app/core/retention/scheduler.py +69 -0
  125. app/core/runtime_logging.py +601 -0
  126. app/core/scheduling/__init__.py +0 -0
  127. app/core/scheduling/leader_election.py +993 -0
  128. app/core/scheduling/leader_election_handle.py +25 -0
  129. app/core/server.py +140 -0
  130. app/core/shutdown.py +326 -0
  131. app/core/socket_peer.py +22 -0
  132. app/core/startup.py +41 -0
  133. app/core/timeout_invariants.py +347 -0
  134. app/core/tracing/__init__.py +0 -0
  135. app/core/tracing/otel.py +207 -0
  136. app/core/types.py +6 -0
  137. app/core/upstream_proxy/__init__.py +13 -0
  138. app/core/upstream_proxy/cache.py +130 -0
  139. app/core/upstream_proxy/resolver.py +189 -0
  140. app/core/upstream_proxy/types.py +104 -0
  141. app/core/usage/__init__.py +407 -0
  142. app/core/usage/depletion.py +109 -0
  143. app/core/usage/live_hub.py +41 -0
  144. app/core/usage/live_snapshots.py +174 -0
  145. app/core/usage/logs.py +212 -0
  146. app/core/usage/metadata_scheduler.py +118 -0
  147. app/core/usage/models.py +54 -0
  148. app/core/usage/pricing.py +576 -0
  149. app/core/usage/pricing_catalog.py +241 -0
  150. app/core/usage/pricing_snapshot.json +821 -0
  151. app/core/usage/quota.py +133 -0
  152. app/core/usage/refresh_policy.py +21 -0
  153. app/core/usage/refresh_scheduler.py +713 -0
  154. app/core/usage/reset_credits_refresh_scheduler.py +435 -0
  155. app/core/usage/types.py +168 -0
  156. app/core/utils/__init__.py +0 -0
  157. app/core/utils/json_guards.py +18 -0
  158. app/core/utils/locks.py +33 -0
  159. app/core/utils/masking.py +20 -0
  160. app/core/utils/proxy_env.py +104 -0
  161. app/core/utils/request_id.py +65 -0
  162. app/core/utils/retry.py +77 -0
  163. app/core/utils/shared_future.py +172 -0
  164. app/core/utils/sse.py +286 -0
  165. app/core/utils/time.py +24 -0
  166. app/db/__init__.py +0 -0
  167. app/db/account_identity_lock.py +56 -0
  168. app/db/alembic/__init__.py +1 -0
  169. app/db/alembic/env.py +70 -0
  170. app/db/alembic/revision_ids.py +43 -0
  171. app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
  172. app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
  173. app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
  174. app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
  175. app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
  176. app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
  177. app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
  178. app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
  179. app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
  180. app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
  181. app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
  182. app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
  183. app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
  184. app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
  185. app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
  186. app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
  187. app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
  188. app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
  189. app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
  190. app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
  191. app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
  192. app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
  193. app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
  194. app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
  195. app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
  196. app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
  197. app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
  198. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
  199. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
  200. app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
  201. app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
  202. app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
  203. app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
  204. app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
  205. app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
  206. app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
  207. app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
  208. app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
  209. app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
  210. app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
  211. app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
  212. app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
  213. app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
  214. app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
  215. app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
  216. app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
  217. app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
  218. app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
  219. app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
  220. app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
  221. app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
  222. app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
  223. app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
  224. app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
  225. app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
  226. app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
  227. app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
  228. app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
  229. app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
  230. app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
  231. app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
  232. app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
  233. app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
  234. app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
  235. app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
  236. app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
  237. app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
  238. app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
  239. app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
  240. app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
  241. app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
  242. app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
  243. app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
  244. app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
  245. app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
  246. app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
  247. app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
  248. app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
  249. app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
  250. app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
  251. app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
  252. app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
  253. app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
  254. app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
  255. app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
  256. app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
  257. app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
  258. app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
  259. app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
  260. app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
  261. app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
  262. app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
  263. app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
  264. app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
  265. app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
  266. app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
  267. app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
  268. app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
  269. app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
  270. app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
  271. app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
  272. app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
  273. app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
  274. app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
  275. app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
  276. app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
  277. app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
  278. app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
  279. app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
  280. app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
  281. app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
  282. app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
  283. app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
  284. app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
  285. app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
  286. app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
  287. app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
  288. app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
  289. app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
  290. app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
  291. app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
  292. app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
  293. app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
  294. app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
  295. app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
  296. app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
  297. app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
  298. app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
  299. app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
  300. app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
  301. app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
  302. app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
  303. app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
  304. app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
  305. app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
  306. app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
  307. app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
  308. app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
  309. app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
  310. app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
  311. app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
  312. app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
  313. app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
  314. app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
  315. app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
  316. app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
  317. app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
  318. app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
  319. app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
  320. app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
  321. app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
  322. app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
  323. app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
  324. app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
  325. app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
  326. app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
  327. app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
  328. app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
  329. app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
  330. app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
  331. app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
  332. app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
  333. app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
  334. app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
  335. app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
  336. app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
  337. app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
  338. app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
  339. app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
  340. app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
  341. app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
  342. app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
  343. app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
  344. app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
  345. app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
  346. app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
  347. app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
  348. app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
  349. app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
  350. app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
  351. app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
  352. app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
  353. app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
  354. app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
  355. app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
  356. app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
  357. app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
  358. app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
  359. app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
  360. app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
  361. app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
  362. app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
  363. app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
  364. app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
  365. app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
  366. app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
  367. app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
  368. app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
  369. app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
  370. app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
  371. app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
  372. app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
  373. app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
  374. app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
  375. app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
  376. app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
  377. app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
  378. app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
  379. app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
  380. app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
  381. app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
  382. app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
  383. app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
  384. app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
  385. app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
  386. app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
  387. app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
  388. app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
  389. app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
  390. app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
  391. app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
  392. app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
  393. app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
  394. app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
  395. app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
  396. app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
  397. app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
  398. app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
  399. app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
  400. app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
  401. app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
  402. app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
  403. app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
  404. app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
  405. app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
  406. app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
  407. app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
  408. app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
  409. app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
  410. app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
  411. app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
  412. app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
  413. app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
  414. app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
  415. app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
  416. app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
  417. app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
  418. app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
  419. app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
  420. app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
  421. app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
  422. app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
  423. app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
  424. app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
  425. app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
  426. app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
  427. app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
  428. app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
  429. app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
  430. app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
  431. app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
  432. app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
  433. app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
  434. app/db/alembic/versions/__init__.py +1 -0
  435. app/db/backup.py +64 -0
  436. app/db/migrate.py +1249 -0
  437. app/db/migration_lock.py +160 -0
  438. app/db/migration_url.py +18 -0
  439. app/db/models.py +3100 -0
  440. app/db/recover.py +244 -0
  441. app/db/session.py +1295 -0
  442. app/db/snapshot.py +46 -0
  443. app/db/sqlite_lock_retry.py +190 -0
  444. app/db/sqlite_utils.py +549 -0
  445. app/dependencies.py +462 -0
  446. app/main.py +1240 -0
  447. app/modules/__init__.py +0 -0
  448. app/modules/accounts/__init__.py +0 -0
  449. app/modules/accounts/api.py +426 -0
  450. app/modules/accounts/auth_manager.py +1439 -0
  451. app/modules/accounts/background_repository.py +160 -0
  452. app/modules/accounts/deletion.py +540 -0
  453. app/modules/accounts/mappers.py +654 -0
  454. app/modules/accounts/refresh_claims.py +352 -0
  455. app/modules/accounts/repository.py +1679 -0
  456. app/modules/accounts/schemas.py +308 -0
  457. app/modules/accounts/service.py +809 -0
  458. app/modules/accounts/usage_rollup.py +486 -0
  459. app/modules/accounts/usage_rollup_scheduler.py +75 -0
  460. app/modules/accounts/usage_time_rollup.py +1037 -0
  461. app/modules/accounts/usage_time_rollup_read.py +522 -0
  462. app/modules/api_keys/__init__.py +1 -0
  463. app/modules/api_keys/api.py +344 -0
  464. app/modules/api_keys/last_used_coalescer.py +216 -0
  465. app/modules/api_keys/limit_windows.py +41 -0
  466. app/modules/api_keys/repository.py +1200 -0
  467. app/modules/api_keys/reset_scheduler.py +85 -0
  468. app/modules/api_keys/schemas.py +135 -0
  469. app/modules/api_keys/service.py +2123 -0
  470. app/modules/audit/__init__.py +0 -0
  471. app/modules/audit/api.py +88 -0
  472. app/modules/audit/repository.py +74 -0
  473. app/modules/audit/schemas.py +33 -0
  474. app/modules/audit/service.py +71 -0
  475. app/modules/auth_providers/__init__.py +0 -0
  476. app/modules/auth_providers/api.py +127 -0
  477. app/modules/auth_providers/config.py +343 -0
  478. app/modules/auth_providers/repository.py +101 -0
  479. app/modules/auth_providers/schemas.py +88 -0
  480. app/modules/auth_providers/seed.py +105 -0
  481. app/modules/auth_providers/service.py +289 -0
  482. app/modules/automations/__init__.py +1 -0
  483. app/modules/automations/api.py +369 -0
  484. app/modules/automations/repository.py +1896 -0
  485. app/modules/automations/scheduler.py +107 -0
  486. app/modules/automations/schemas.py +158 -0
  487. app/modules/automations/service.py +2548 -0
  488. app/modules/cache_isolation_probe/__init__.py +0 -0
  489. app/modules/cache_isolation_probe/api.py +188 -0
  490. app/modules/cache_isolation_probe/prefix.py +144 -0
  491. app/modules/cache_isolation_probe/schemas.py +94 -0
  492. app/modules/cache_isolation_probe/sender.py +229 -0
  493. app/modules/cache_isolation_probe/service.py +525 -0
  494. app/modules/conversation_archive/__init__.py +1 -0
  495. app/modules/conversation_archive/api.py +101 -0
  496. app/modules/conversation_archive/schemas.py +28 -0
  497. app/modules/conversation_archive/service.py +165 -0
  498. app/modules/dashboard/__init__.py +1 -0
  499. app/modules/dashboard/api.py +106 -0
  500. app/modules/dashboard/builders.py +107 -0
  501. app/modules/dashboard/repository.py +304 -0
  502. app/modules/dashboard/schemas.py +148 -0
  503. app/modules/dashboard/service.py +534 -0
  504. app/modules/dashboard/timeframes.py +19 -0
  505. app/modules/dashboard/weekly_pace.py +617 -0
  506. app/modules/dashboard_auth/__init__.py +1 -0
  507. app/modules/dashboard_auth/api.py +1344 -0
  508. app/modules/dashboard_auth/oidc_api.py +753 -0
  509. app/modules/dashboard_auth/oidc_flows.py +330 -0
  510. app/modules/dashboard_auth/repository.py +452 -0
  511. app/modules/dashboard_auth/schemas.py +232 -0
  512. app/modules/dashboard_auth/service.py +1384 -0
  513. app/modules/dashboard_roles/__init__.py +0 -0
  514. app/modules/dashboard_roles/api.py +64 -0
  515. app/modules/dashboard_roles/repository.py +54 -0
  516. app/modules/dashboard_roles/schemas.py +29 -0
  517. app/modules/dashboard_roles/seed.py +80 -0
  518. app/modules/dashboard_roles/service.py +142 -0
  519. app/modules/dashboard_users/__init__.py +0 -0
  520. app/modules/dashboard_users/api.py +278 -0
  521. app/modules/dashboard_users/break_glass.py +169 -0
  522. app/modules/dashboard_users/credentials.py +18 -0
  523. app/modules/dashboard_users/identity_resolver.py +665 -0
  524. app/modules/dashboard_users/repository.py +725 -0
  525. app/modules/dashboard_users/schemas.py +117 -0
  526. app/modules/dashboard_users/service.py +1054 -0
  527. app/modules/firewall/__init__.py +1 -0
  528. app/modules/firewall/api.py +78 -0
  529. app/modules/firewall/repository.py +51 -0
  530. app/modules/firewall/schemas.py +28 -0
  531. app/modules/firewall/service.py +93 -0
  532. app/modules/fleet/__init__.py +1 -0
  533. app/modules/fleet/api.py +177 -0
  534. app/modules/fleet/mappers.py +56 -0
  535. app/modules/fleet/observability.py +372 -0
  536. app/modules/fleet/schemas.py +119 -0
  537. app/modules/health/__init__.py +0 -0
  538. app/modules/health/api.py +246 -0
  539. app/modules/health/schemas.py +27 -0
  540. app/modules/limit_warmup/__init__.py +1 -0
  541. app/modules/limit_warmup/repository.py +159 -0
  542. app/modules/limit_warmup/service.py +1001 -0
  543. app/modules/model_sources/__init__.py +1 -0
  544. app/modules/model_sources/api.py +99 -0
  545. app/modules/model_sources/catalog.py +307 -0
  546. app/modules/model_sources/forwarding.py +1435 -0
  547. app/modules/model_sources/projection.py +55 -0
  548. app/modules/model_sources/repository.py +201 -0
  549. app/modules/model_sources/schemas.py +79 -0
  550. app/modules/model_sources/selection.py +166 -0
  551. app/modules/model_sources/service.py +236 -0
  552. app/modules/oauth/__init__.py +0 -0
  553. app/modules/oauth/api.py +101 -0
  554. app/modules/oauth/repository.py +238 -0
  555. app/modules/oauth/schemas.py +49 -0
  556. app/modules/oauth/service.py +1196 -0
  557. app/modules/oauth/templates/oauth_success.html +122 -0
  558. app/modules/proxy/__init__.py +0 -0
  559. app/modules/proxy/_load_balancer/__init__.py +1 -0
  560. app/modules/proxy/_load_balancer/error_rate.py +116 -0
  561. app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
  562. app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
  563. app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
  564. app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
  565. app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
  566. app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
  567. app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
  568. app/modules/proxy/_load_balancer/tunables.py +97 -0
  569. app/modules/proxy/_load_balancer/types.py +96 -0
  570. app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
  571. app/modules/proxy/_service/__init__.py +6 -0
  572. app/modules/proxy/_service/api_key_usage.py +745 -0
  573. app/modules/proxy/_service/clock_budget.py +38 -0
  574. app/modules/proxy/_service/codex_control.py +611 -0
  575. app/modules/proxy/_service/compact.py +2211 -0
  576. app/modules/proxy/_service/file_ops.py +749 -0
  577. app/modules/proxy/_service/http_bridge/__init__.py +3 -0
  578. app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
  579. app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
  580. app/modules/proxy/_service/http_bridge/activity.py +178 -0
  581. app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
  582. app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
  583. app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
  584. app/modules/proxy/_service/http_bridge/protocol.py +167 -0
  585. app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
  586. app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
  587. app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
  588. app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
  589. app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
  590. app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
  591. app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
  592. app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
  593. app/modules/proxy/_service/observability.py +410 -0
  594. app/modules/proxy/_service/rate_limit.py +361 -0
  595. app/modules/proxy/_service/realtime_live.py +655 -0
  596. app/modules/proxy/_service/refresh.py +154 -0
  597. app/modules/proxy/_service/request_log.py +585 -0
  598. app/modules/proxy/_service/response_create.py +930 -0
  599. app/modules/proxy/_service/streaming/__init__.py +3 -0
  600. app/modules/proxy/_service/streaming/helpers.py +1317 -0
  601. app/modules/proxy/_service/streaming/mixin.py +1100 -0
  602. app/modules/proxy/_service/streaming/protocol.py +32 -0
  603. app/modules/proxy/_service/streaming/retry.py +3762 -0
  604. app/modules/proxy/_service/support.py +2122 -0
  605. app/modules/proxy/_service/transcribe.py +434 -0
  606. app/modules/proxy/_service/warmup.py +483 -0
  607. app/modules/proxy/_service/websocket/__init__.py +3 -0
  608. app/modules/proxy/_service/websocket/helpers.py +2360 -0
  609. app/modules/proxy/_service/websocket/mixin.py +7285 -0
  610. app/modules/proxy/_service/websocket/protocol.py +74 -0
  611. app/modules/proxy/account_cache.py +229 -0
  612. app/modules/proxy/account_eligibility.py +87 -0
  613. app/modules/proxy/additional_model_limits.py +40 -0
  614. app/modules/proxy/affinity.py +830 -0
  615. app/modules/proxy/affinity_observation.py +101 -0
  616. app/modules/proxy/api.py +10254 -0
  617. app/modules/proxy/api_key_usage.py +113 -0
  618. app/modules/proxy/cap_partitioning.py +280 -0
  619. app/modules/proxy/capability_lineage.py +65 -0
  620. app/modules/proxy/capability_lineage_repository.py +82 -0
  621. app/modules/proxy/capability_routing.py +253 -0
  622. app/modules/proxy/continuity.py +112 -0
  623. app/modules/proxy/downstream_delivery.py +277 -0
  624. app/modules/proxy/durable_bridge_coordinator.py +1217 -0
  625. app/modules/proxy/durable_bridge_repository.py +4516 -0
  626. app/modules/proxy/durable_bridge_runtime.py +9 -0
  627. app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
  628. app/modules/proxy/fair_share.py +131 -0
  629. app/modules/proxy/file_pin_repository.py +210 -0
  630. app/modules/proxy/helpers.py +439 -0
  631. app/modules/proxy/http_bridge_event_batcher.py +613 -0
  632. app/modules/proxy/http_bridge_forwarding.py +797 -0
  633. app/modules/proxy/http_continuation.py +76 -0
  634. app/modules/proxy/images_observability.py +62 -0
  635. app/modules/proxy/images_service.py +1054 -0
  636. app/modules/proxy/load_balancer.py +3016 -0
  637. app/modules/proxy/rate_limit_cache.py +46 -0
  638. app/modules/proxy/replay_safety.py +1063 -0
  639. app/modules/proxy/repo_bundle.py +31 -0
  640. app/modules/proxy/request_policy.py +1054 -0
  641. app/modules/proxy/ring_membership.py +251 -0
  642. app/modules/proxy/schemas.py +345 -0
  643. app/modules/proxy/selection_errors.py +41 -0
  644. app/modules/proxy/service.py +2600 -0
  645. app/modules/proxy/source_admission.py +125 -0
  646. app/modules/proxy/source_dispatch.py +999 -0
  647. app/modules/proxy/sticky_repository.py +831 -0
  648. app/modules/proxy/thread_anchors.py +491 -0
  649. app/modules/proxy/tool_call_dedupe.py +802 -0
  650. app/modules/proxy/types.py +54 -0
  651. app/modules/proxy/work_admission.py +121 -0
  652. app/modules/quota_planner/__init__.py +1 -0
  653. app/modules/quota_planner/api.py +276 -0
  654. app/modules/quota_planner/logic.py +743 -0
  655. app/modules/quota_planner/repository.py +722 -0
  656. app/modules/quota_planner/scheduler.py +212 -0
  657. app/modules/quota_planner/schemas.py +102 -0
  658. app/modules/quota_planner/warmup.py +693 -0
  659. app/modules/rate_limit_reset_credits/__init__.py +0 -0
  660. app/modules/rate_limit_reset_credits/api.py +601 -0
  661. app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
  662. app/modules/rate_limit_reset_credits/store.py +114 -0
  663. app/modules/reports/__init__.py +1 -0
  664. app/modules/reports/api.py +134 -0
  665. app/modules/reports/cache.py +74 -0
  666. app/modules/reports/filters.py +27 -0
  667. app/modules/reports/repository.py +525 -0
  668. app/modules/reports/rollup.py +150 -0
  669. app/modules/reports/rollup_read.py +148 -0
  670. app/modules/reports/schemas.py +123 -0
  671. app/modules/reports/service.py +287 -0
  672. app/modules/reports/thread_identity.py +256 -0
  673. app/modules/request_logs/__init__.py +0 -0
  674. app/modules/request_logs/api.py +224 -0
  675. app/modules/request_logs/cost_backfill.py +154 -0
  676. app/modules/request_logs/mappers.py +97 -0
  677. app/modules/request_logs/repository.py +1765 -0
  678. app/modules/request_logs/schemas.py +145 -0
  679. app/modules/request_logs/service.py +348 -0
  680. app/modules/role_mappings/__init__.py +0 -0
  681. app/modules/role_mappings/api.py +206 -0
  682. app/modules/role_mappings/matching.py +55 -0
  683. app/modules/role_mappings/repository.py +106 -0
  684. app/modules/role_mappings/schemas.py +67 -0
  685. app/modules/role_mappings/service.py +334 -0
  686. app/modules/runtime/__init__.py +1 -0
  687. app/modules/runtime/api.py +18 -0
  688. app/modules/runtime/schemas.py +14 -0
  689. app/modules/runtime/service.py +218 -0
  690. app/modules/scim/__init__.py +1 -0
  691. app/modules/scim/api.py +194 -0
  692. app/modules/scim/dependencies.py +112 -0
  693. app/modules/scim/errors.py +77 -0
  694. app/modules/scim/management_api.py +156 -0
  695. app/modules/scim/repository.py +212 -0
  696. app/modules/scim/schemas.py +145 -0
  697. app/modules/scim/service.py +786 -0
  698. app/modules/scim/tokens.py +43 -0
  699. app/modules/settings/__init__.py +1 -0
  700. app/modules/settings/api.py +1677 -0
  701. app/modules/settings/repository.py +606 -0
  702. app/modules/settings/schemas.py +496 -0
  703. app/modules/settings/service.py +769 -0
  704. app/modules/shared/__init__.py +0 -0
  705. app/modules/shared/schemas.py +22 -0
  706. app/modules/sticky_sessions/__init__.py +1 -0
  707. app/modules/sticky_sessions/api.py +118 -0
  708. app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
  709. app/modules/sticky_sessions/schemas.py +78 -0
  710. app/modules/sticky_sessions/service.py +196 -0
  711. app/modules/telemetry/__init__.py +5 -0
  712. app/modules/telemetry/api.py +115 -0
  713. app/modules/telemetry/clients.py +62 -0
  714. app/modules/telemetry/consent.py +115 -0
  715. app/modules/telemetry/scheduler.py +85 -0
  716. app/modules/telemetry/schemas.py +163 -0
  717. app/modules/telemetry/sender.py +221 -0
  718. app/modules/telemetry/snapshot.py +490 -0
  719. app/modules/usage/__init__.py +0 -0
  720. app/modules/usage/additional_quota_keys.py +379 -0
  721. app/modules/usage/api.py +45 -0
  722. app/modules/usage/background_repository.py +147 -0
  723. app/modules/usage/builders.py +559 -0
  724. app/modules/usage/depletion_service.py +277 -0
  725. app/modules/usage/live_ingest.py +356 -0
  726. app/modules/usage/mappers.py +29 -0
  727. app/modules/usage/plan_downgrade_observations.py +488 -0
  728. app/modules/usage/repository.py +1890 -0
  729. app/modules/usage/schemas.py +68 -0
  730. app/modules/usage/service.py +102 -0
  731. app/modules/usage/updater.py +1488 -0
  732. app/static/assets/Trans-Cj-J9mVA.js +1 -0
  733. app/static/assets/access-page-DHzZiXZi.js +1 -0
  734. app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
  735. app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
  736. app/static/assets/account-status-DIsU5sUS.js +1 -0
  737. app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
  738. app/static/assets/accounts-page-Bbai8ift.js +3 -0
  739. app/static/assets/api-Dwa7XxPe.js +1 -0
  740. app/static/assets/api-client-dugX1Pti.js +55 -0
  741. app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
  742. app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
  743. app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
  744. app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
  745. app/static/assets/apis-page-BvXz9c6i.js +2 -0
  746. app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
  747. app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
  748. app/static/assets/automations-page-DQgNAJOR.js +3 -0
  749. app/static/assets/badge-DqMb9es0.js +1 -0
  750. app/static/assets/blur-email-HEvKagG2.js +1 -0
  751. app/static/assets/button-BjVmmiGW.js +1 -0
  752. app/static/assets/checkbox-apsl9vzk.js +1 -0
  753. app/static/assets/chevron-left-1d_pHVa5.js +1 -0
  754. app/static/assets/circle-alert-BS0xd_o1.js +1 -0
  755. app/static/assets/clsx-DB0hHKMi.js +1 -0
  756. app/static/assets/colors-BBy8GEtr.js +1 -0
  757. app/static/assets/copy-button-D7nAvOKR.js +1 -0
  758. app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
  759. app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
  760. app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
  761. app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
  762. app/static/assets/donut-chart-Bxc69cra.js +1 -0
  763. app/static/assets/download-kQteIcsB.js +1 -0
  764. app/static/assets/ellipsis-B_5OgIre.js +1 -0
  765. app/static/assets/empty-state-JcA7tc8m.js +1 -0
  766. app/static/assets/es2015-bMH8YKZT.js +41 -0
  767. app/static/assets/es6-B264ni6W.js +69 -0
  768. app/static/assets/external-link-UVAOgMQ0.js +1 -0
  769. app/static/assets/i18n-BWqZ9Ua2.js +1 -0
  770. app/static/assets/inbox-B_RM2s1J.js +1 -0
  771. app/static/assets/index-DIpd264Z.js +23 -0
  772. app/static/assets/index-DMeLKRJx.css +2 -0
  773. app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
  774. app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
  775. app/static/assets/iso-Bpe5Ihbb.js +1 -0
  776. app/static/assets/label-CGaLmLNf.js +1 -0
  777. app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
  778. app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
  779. app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
  780. app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
  781. app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
  782. app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
  783. app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
  784. app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
  785. app/static/assets/payload-CIYnGZz9.js +1 -0
  786. app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
  787. app/static/assets/report-chart-card-BXDliwcc.js +1 -0
  788. app/static/assets/reports-page-CGCJeMHG.js +3 -0
  789. app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
  790. app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
  791. app/static/assets/search-HfhqwERt.js +1 -0
  792. app/static/assets/select-Cl33bz-g.js +1 -0
  793. app/static/assets/settings-page-mZj01HYl.js +2 -0
  794. app/static/assets/shim-CZwyM4Q7.js +1 -0
  795. app/static/assets/skeleton-C_EUc4Rp.js +1 -0
  796. app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
  797. app/static/assets/status-badge-CHHA5X6c.js +1 -0
  798. app/static/assets/switch-CzxgokcM.js +1 -0
  799. app/static/assets/table-DXaNSQ0G.js +1 -0
  800. app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
  801. app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
  802. app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
  803. app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
  804. app/static/assets/trash-D3KyWr1h.js +1 -0
  805. app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
  806. app/static/assets/use-accounts-PWlVGw3z.js +1 -0
  807. app/static/assets/use-auth-CuDlW3iC.js +1 -0
  808. app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
  809. app/static/assets/use-model-sources-Bul24JBu.js +1 -0
  810. app/static/assets/use-models-DUCPUuZg.js +1 -0
  811. app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
  812. app/static/assets/useTranslation--56nHhlH.js +1 -0
  813. app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
  814. app/static/assets/users-uIC3TIlE.js +1 -0
  815. app/static/assets/vendor-query-_bHZuxEx.js +1 -0
  816. app/static/assets/vendor-react-BVL0YZPW.js +9 -0
  817. app/static/favicon.svg +10 -0
  818. app/static/fonts/GeistSans-Variable.woff2 +0 -0
  819. app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
  820. app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
  821. app/static/index.html +29 -0
  822. codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
  823. codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
  824. codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
  825. codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
  826. codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
  827. config/additional_quota_registry.json +11 -0
@@ -0,0 +1,4582 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import json
5
+ import logging
6
+ import math
7
+ import random
8
+ from collections import deque
9
+ from collections.abc import Callable
10
+ from dataclasses import dataclass, replace
11
+ from typing import Any, Literal, Mapping, cast
12
+ from uuid import uuid4
13
+
14
+ from app.core.clients.files import create_file as core_create_file # noqa: F401
15
+ from app.core.clients.files import finalize_file as core_finalize_file # noqa: F401
16
+ from app.core.clients.proxy import ( # noqa: F401
17
+ CODEX_INSTALLATION_ID_HEADER,
18
+ CODEX_TURN_METADATA_HEADER,
19
+ ImageFetchSession,
20
+ ProxyResponseError,
21
+ UpstreamProxyRouteTrace,
22
+ _as_image_fetch_session,
23
+ _finalize_responses_lite_reasoning_context,
24
+ _historical_agent_control_output_occurrences,
25
+ _inline_content_images,
26
+ _inline_input_image_urls,
27
+ _payload_has_responses_lite_websocket_marker,
28
+ _payload_uses_responses_lite,
29
+ _ws_transport_payload_budget_bytes,
30
+ apply_codex_installation_metadata,
31
+ filter_inbound_headers,
32
+ pop_compact_timeout_overrides,
33
+ pop_stream_timeout_overrides,
34
+ pop_transcribe_timeout_overrides,
35
+ push_compact_timeout_overrides,
36
+ push_stream_timeout_overrides,
37
+ push_transcribe_timeout_overrides,
38
+ )
39
+ from app.core.clients.proxy import CodexControlResponse as CodexControlResponse
40
+ from app.core.clients.proxy import codex_control_request as core_codex_control_request # noqa: F401
41
+ from app.core.clients.proxy import compact_responses as core_compact_responses # noqa: F401
42
+ from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401
43
+ from app.core.clients.proxy_websocket import (
44
+ UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE,
45
+ UpstreamWebSocketTransportError,
46
+ is_account_neutral_websocket_error_code,
47
+ )
48
+ from app.core.clients.thread_cache_identity import (
49
+ THREAD_CACHE_IDENTITY_MODE_SHARED,
50
+ ThreadCacheIdentity,
51
+ apply_thread_cache_identity,
52
+ thread_cache_identity_from,
53
+ )
54
+ from app.core.clock import REAL_CLOCK, Clock, clock_for, scheduler_for
55
+ from app.core.config.dashboard_overrides import with_dashboard_overrides
56
+ from app.core.errors import openai_error
57
+ from app.core.openai.parsing import parse_sse_event
58
+ from app.core.openai.requests import (
59
+ ResponsesRequest,
60
+ )
61
+ from app.core.resilience.overload import is_local_overload_error_code
62
+ from app.core.types import JsonObject, JsonValue
63
+ from app.core.utils.locks import fast_lock
64
+ from app.core.utils.request_id import (
65
+ ensure_request_id,
66
+ ensure_request_scope_id,
67
+ get_request_id,
68
+ reset_request_id,
69
+ set_request_id,
70
+ )
71
+ from app.core.utils.sse import format_sse_event, parse_sse_data_json
72
+ from app.db.models import DashboardSettings, StickySessionKind
73
+ from app.modules.api_keys.service import (
74
+ ApiKeyData,
75
+ ApiKeyUsageReservationData,
76
+ )
77
+ from app.modules.proxy._load_balancer.tunables import RoutingTunables
78
+ from app.modules.proxy._service.api_key_usage import (
79
+ _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS,
80
+ )
81
+ from app.modules.proxy._service.compact import (
82
+ _sticky_key_for_compact_request as _sticky_key_for_compact_request,
83
+ )
84
+ from app.modules.proxy._service.compact import (
85
+ _sticky_key_from_compact_payload as _sticky_key_from_compact_payload,
86
+ )
87
+ from app.modules.proxy._service.http_bridge import helpers as _http_bridge_helpers
88
+ from app.modules.proxy._service.http_bridge.accepted_replay import (
89
+ _claim_websocket_replay_create_gate,
90
+ _http_bridge_accepted_replay_may_exclude_account,
91
+ )
92
+ from app.modules.proxy._service.http_bridge.helpers import (
93
+ _HTTP_BRIDGE_COOLDOWN_SUPPRESSION_ATTR,
94
+ _HTTP_BRIDGE_PRE_SUBMIT_FAILURE_ATTR,
95
+ _await_task_deferring_cancellation,
96
+ _bind_http_bridge_proxy_injected_anchor,
97
+ _build_http_bridge_prewarm_text,
98
+ _http_bridge_abandonment_may_settle_circuit,
99
+ _http_bridge_client_full_history_recovery_error,
100
+ _http_bridge_denied_anchor_fence_advanced,
101
+ _http_bridge_durable_lease_ttl_seconds,
102
+ _http_bridge_is_previous_response_owner_unavailable,
103
+ _http_bridge_key_strength,
104
+ _http_bridge_payload_looks_like_full_resend,
105
+ _http_bridge_precreated_retry_failure_error,
106
+ _http_bridge_prewarm_enabled,
107
+ _http_bridge_request_budget_seconds,
108
+ _http_bridge_request_counts_against_queue,
109
+ _http_bridge_request_state_holds_safe_replay,
110
+ _http_bridge_retry_circuit_attempt_selection_for_pending_requests,
111
+ _log_http_bridge_event,
112
+ _record_continuity_fail_closed,
113
+ _record_http_bridge_prewarm_outcome,
114
+ _register_http_bridge_turn_state_aliases_locked,
115
+ _release_http_bridge_unanchored_handoff,
116
+ )
117
+ from app.modules.proxy._service.http_bridge.quarantine import (
118
+ _http_bridge_session_key_poison_quarantined,
119
+ _record_http_bridge_quarantine_wedged_pending,
120
+ )
121
+ from app.modules.proxy._service.http_bridge.retry_circuit import (
122
+ _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
123
+ _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD,
124
+ _POISON_ANCHOR_CAPTURE_UNAVAILABLE,
125
+ _http_bridge_retry_circuit_suppression_message,
126
+ )
127
+ from app.modules.proxy._service.http_bridge.service_stubs import (
128
+ _call_with_supported_optional_kwargs,
129
+ _classify_upstream_close,
130
+ _count_external_image_urls,
131
+ _enforce_response_create_size_limit,
132
+ _estimated_lease_tokens_from_request_usage_budget,
133
+ _fingerprint_input_items,
134
+ _inline_top_level_input_image_urls,
135
+ _normalize_service_tier_value,
136
+ _normalize_session_id,
137
+ _prepare_websocket_request_state_for_account_switch,
138
+ _prepare_websocket_request_state_for_auth_replay,
139
+ _prepare_websocket_request_state_for_visible_output_replay,
140
+ _prewarm_response_timeout_seconds,
141
+ _release_websocket_response_create_gate,
142
+ _response_create_client_metadata,
143
+ _security_work_advisory_event,
144
+ _service_as_image_fetch_session,
145
+ _service_get_settings,
146
+ _service_get_settings_cache,
147
+ _service_inline_input_image_urls,
148
+ _service_lease_http_session,
149
+ _slim_response_create_payload_for_upstream,
150
+ _upstream_response_create_max_bytes,
151
+ _websocket_auth_failure_permanent_code,
152
+ _websocket_auth_failure_requires_reauth,
153
+ _websocket_request_text_is_account_neutral_fresh_replay,
154
+ )
155
+ from app.modules.proxy._service.http_bridge.upstream_events import (
156
+ _abandon_durable_http_bridge_continuity,
157
+ )
158
+ from app.modules.proxy._service.observability import (
159
+ _hash_identifier as _hash_identifier,
160
+ )
161
+ from app.modules.proxy._service.observability import (
162
+ _hash_identifier_or_none as _hash_identifier_or_none,
163
+ )
164
+ from app.modules.proxy._service.observability import (
165
+ _interesting_header_keys as _interesting_header_keys,
166
+ )
167
+ from app.modules.proxy._service.observability import (
168
+ _tools_hash as _tools_hash,
169
+ )
170
+ from app.modules.proxy._service.observability import (
171
+ _truncate_identifier as _truncate_identifier,
172
+ )
173
+ from app.modules.proxy._service.support import (
174
+ _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE,
175
+ _HARD_HTTP_BRIDGE_AFFINITY_KINDS, # noqa: F401
176
+ _REQUEST_TRANSPORT_WEBSOCKET,
177
+ _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401
178
+ _api_key_fair_share_threshold_pct_from_settings,
179
+ _clear_websocket_request_error_overrides,
180
+ _copy_websocket_route_metadata_from_session,
181
+ _event_type_from_payload,
182
+ _HTTPBridgeResponseCreateAttempt,
183
+ _HTTPBridgeRetryCircuitAttemptSelection,
184
+ _HTTPBridgeSession,
185
+ _request_log_client_fields,
186
+ _websocket_request_can_replay_before_visible_output,
187
+ _WebSocketRequestState,
188
+ )
189
+ from app.modules.proxy._service.support import (
190
+ _websocket_route_log_kwargs as _websocket_route_log_kwargs,
191
+ )
192
+ from app.modules.proxy._service.warmup import (
193
+ WarmupExecutionData as WarmupExecutionData,
194
+ )
195
+ from app.modules.proxy._service.warmup import (
196
+ WarmupFailedAccountData as WarmupFailedAccountData,
197
+ )
198
+ from app.modules.proxy._service.warmup import (
199
+ WarmupSkippedAccountData as WarmupSkippedAccountData,
200
+ )
201
+ from app.modules.proxy._service.warmup import (
202
+ WarmupSubmittedAccountData as WarmupSubmittedAccountData,
203
+ )
204
+ from app.modules.proxy._service.warmup import (
205
+ _is_warmup_usage_eligible as _is_warmup_usage_eligible,
206
+ )
207
+ from app.modules.proxy._service.warmup import (
208
+ _materialize_warmup_account as _materialize_warmup_account,
209
+ )
210
+ from app.modules.proxy._service.warmup import (
211
+ _snapshot_warmup_account as _snapshot_warmup_account,
212
+ )
213
+ from app.modules.proxy._service.warmup import (
214
+ _WarmupAccountSnapshot as _WarmupAccountSnapshot,
215
+ )
216
+ from app.modules.proxy._service.warmup import (
217
+ _WarmupSubmitResult as _WarmupSubmitResult,
218
+ )
219
+ from app.modules.proxy._service.warmup import (
220
+ _WarmupUsageSnapshot as _WarmupUsageSnapshot,
221
+ )
222
+ from app.modules.proxy.affinity import (
223
+ _AffinityPolicy,
224
+ _extract_model_class,
225
+ _owner_lookup_session_id_from_headers,
226
+ _sticky_key_from_turn_state_header,
227
+ )
228
+ from app.modules.proxy.affinity_observation import AffinityObservation
229
+ from app.modules.proxy.api_key_usage import estimate_api_key_request_usage
230
+ from app.modules.proxy.continuity import is_http_bridge_account_neutral_replay
231
+ from app.modules.proxy.durable_bridge_repository import (
232
+ DurableBridgeAliasRegistration,
233
+ DurableBridgeAliasRegistrationReceipt,
234
+ durable_bridge_api_key_scope,
235
+ durable_bridge_hash,
236
+ durable_bridge_operation_fingerprint,
237
+ durable_bridge_operation_id,
238
+ )
239
+ from app.modules.proxy.fair_share import (
240
+ API_KEY_STREAM_FAIR_SHARE_ERROR_CODE,
241
+ ApiKeyFairShareDenialError,
242
+ )
243
+ from app.modules.proxy.helpers import (
244
+ _normalize_error_code,
245
+ _parse_openai_error,
246
+ )
247
+ from app.modules.proxy.load_balancer import effective_account_concurrency_caps, effective_routing_tunables
248
+ from app.modules.proxy.tool_call_dedupe import (
249
+ dedupe_replayed_side_effect_input_items,
250
+ )
251
+
252
+ logger = logging.getLogger("app.modules.proxy.service")
253
+
254
+ _HTTP_BRIDGE_CLEAN_CLOSE_RETRY_MAX_COUNT = 1
255
+ _HTTP_BRIDGE_CLEAN_CLOSE_RETRY_JITTER_MAX_SECONDS = 2.0
256
+
257
+ _REQUEST_TRANSPORT_HTTP = "http"
258
+ _WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE = "account_auth_invalidated"
259
+ _NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE = "no_security_work_authorized_accounts"
260
+ _SECURITY_WORK_NO_AUTHORIZED_ACCOUNTS_MESSAGE = (
261
+ "Upstream flagged this request as possible cybersecurity work, but no account is marked as authorized for "
262
+ "security work. codex-lb is continuing with normal account selection; the upstream request may still fail until "
263
+ "an account with Trusted Access for Cyber is marked as security-work-authorized."
264
+ )
265
+
266
+
267
+ @dataclass(frozen=True, slots=True)
268
+ class _HTTPBridgeStaleGateSnapshot:
269
+ pending_states: list[_WebSocketRequestState]
270
+ queued_count: int
271
+ threshold_seconds: float
272
+ stale_request_states: list[_WebSocketRequestState]
273
+ should_retire: bool
274
+ retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection
275
+
276
+
277
+ def _http_bridge_cooldown_suppression_is_replay_safe(request_state: _WebSocketRequestState) -> bool:
278
+ """Return whether a cooldown-suppressed submission is provably undispatched.
279
+
280
+ The retry-circuit gate runs before any reconnect or send for the current
281
+ submission, so a request that carries no continuation identity (client or
282
+ proxy-injected), no file account pin, and has seen zero send attempts,
283
+ zero upstream response events and zero replays cannot have executed
284
+ upstream. Only those requests may degrade to the raw-HTTP fallback;
285
+ ambiguous continuations keep the bounded 503 because a replay could
286
+ execute the turn twice.
287
+
288
+ A non-empty payload ``conversation`` counts as continuation identity even
289
+ though it is neither an anchor nor a turn-state key: it binds the turn to
290
+ the bridge session's account, and unlike a response anchor it has no owner
291
+ index, so a raw-HTTP replay cannot prove that owner in a multi-account
292
+ pool and would fail the turn closed instead of letting the cooldown expire.
293
+
294
+ ``response_create_attempt_count`` is the send marker rather than
295
+ ``awaiting_response_created``: the latter is set optimistically when the
296
+ request state is built and is only cleared after a submission is torn
297
+ down, so it is true for every fresh turn and would make this predicate —
298
+ and the fallback it gates — unreachable. The attempt count increments at
299
+ the ``response.create`` send itself and is never reset, so zero proves no
300
+ frame left for upstream.
301
+ """
302
+ return (
303
+ request_state.previous_response_id is None
304
+ and not request_state.hard_continuity_anchor
305
+ and not request_state.proxy_injected_previous_response_id
306
+ and not request_state.file_required_preferred_account
307
+ and request_state.response_id is None
308
+ and request_state.response_event_count == 0
309
+ and request_state.replay_count == 0
310
+ and request_state.last_downstream_sequence_number is None
311
+ and not request_state.downstream_visible
312
+ and not request_state.payload_conversation_bound
313
+ and request_state.response_create_attempt_count == 0
314
+ )
315
+
316
+
317
+ def _http_bridge_operation_fingerprint(
318
+ *,
319
+ session_id: str,
320
+ api_key_scope: str,
321
+ request_state: _WebSocketRequestState,
322
+ text_data: str,
323
+ ) -> str:
324
+ fingerprint_text = _text_without_account_installation_id(text_data)
325
+ return durable_bridge_operation_fingerprint(
326
+ api_key_scope=api_key_scope,
327
+ request_text=fingerprint_text,
328
+ )
329
+
330
+
331
+ def _http_bridge_terminal_hard_turn_response_id(
332
+ request_state: _WebSocketRequestState,
333
+ operation: Any,
334
+ *,
335
+ allow_anchored_continuation: bool = False,
336
+ ) -> str | None:
337
+ """Return a completed hard-turn anchor when this is a new client turn.
338
+
339
+ Hard turn-state requests can omit ``previous_response_id``. Their durable
340
+ operation fingerprint is therefore otherwise identical for repeated
341
+ prompts. A terminal operation with a response id represents the prior turn,
342
+ not an in-flight retry, so the next request must advance from its response
343
+ rather than replaying that transcript. Recovery/rebind states retain the
344
+ operation identity and are intentionally excluded here. Spool completeness
345
+ is required only when replaying the stored transcript.
346
+ """
347
+ if (
348
+ (request_state.previous_response_id is not None and not allow_anchored_continuation)
349
+ or not request_state.hard_continuity_anchor
350
+ or request_state.operation_id is not None
351
+ or request_state.operation_rebind_required
352
+ or request_state.replay_count != 0
353
+ ):
354
+ return None
355
+ operation_state = getattr(operation, "state", None)
356
+ operation_state = getattr(operation_state, "value", operation_state)
357
+ if operation_state != "completed":
358
+ return None
359
+ response_id = getattr(operation, "response_id", None)
360
+ return response_id if isinstance(response_id, str) and response_id else None
361
+
362
+
363
+ async def _rollback_http_bridge_recovery_turn_state_registration(
364
+ service: Any,
365
+ receipt: DurableBridgeAliasRegistrationReceipt,
366
+ ) -> tuple[bool, asyncio.CancelledError | None]:
367
+ rollback_task = scheduler_for(service).create_task(
368
+ service._durable_bridge.rollback_recovery_turn_state_registration(receipt=receipt)
369
+ )
370
+ return await _await_task_deferring_cancellation(rollback_task)
371
+
372
+
373
+ def _bridge_thread_cache_identity(session: Any) -> ThreadCacheIdentity | None:
374
+ """The mode this bridge session sends under, or ``None`` for a pure no-op.
375
+
376
+ The bridge dispatches ``response.create`` itself rather than going through
377
+ ``_stream_responses_with_session``, so without this a thread served partly
378
+ by the bridge and partly by the per-turn bypass would alternate between a
379
+ scoped and an unscoped identity turn by turn — the worst possible outcome,
380
+ because upstream would see two names for one thread on one account.
381
+
382
+ The fleet value is read off the overlaid settings (dashboard over
383
+ environment over default). The per-API-key override is deliberately not
384
+ consulted here: this path holds only the key's id, not the row, and a
385
+ request that reaches the bridge has already had the override applied at
386
+ ingress when it was dispatched through the ordinary transport. Resolving it
387
+ a second time from a different source could disagree with that decision.
388
+ """
389
+
390
+ mode = getattr(with_dashboard_overrides(_service_get_settings()), "thread_cache_identity_mode", None)
391
+ identity = thread_cache_identity_from(mode, getattr(getattr(session, "account", None), "id", None))
392
+ return identity
393
+
394
+
395
+ def _text_with_thread_cache_identity(text_data: str, identity: ThreadCacheIdentity | None) -> str:
396
+ """Scope the frame's thread identity for egress only.
397
+
398
+ Send-only, exactly like ``_text_with_operation_id``: the result is never
399
+ written back to ``request_state.request_text`` or
400
+ ``fresh_upstream_request_text``, because
401
+ ``_http_bridge_operation_fingerprint`` must keep hashing account-neutral
402
+ text. If a scoped value entered the durable fingerprint, every account swap
403
+ would change the operation identity and the spool lookup would miss
404
+ mid-recovery.
405
+ """
406
+
407
+ if identity is None or identity.mode == THREAD_CACHE_IDENTITY_MODE_SHARED:
408
+ return text_data
409
+ try:
410
+ frame = json.loads(text_data)
411
+ except (TypeError, json.JSONDecodeError):
412
+ return text_data
413
+ if not isinstance(frame, dict):
414
+ return text_data
415
+ apply_thread_cache_identity(frame, identity)
416
+ return json.dumps(frame, ensure_ascii=True, separators=(",", ":"))
417
+
418
+
419
+ async def _send_http_bridge_request_text_with_archive_id(
420
+ session: "_HTTPBridgeSession",
421
+ request_state: _WebSocketRequestState,
422
+ text_data: str,
423
+ *,
424
+ on_send_started: Callable[[], None] | None = None,
425
+ clock: Clock = REAL_CLOCK,
426
+ ) -> None:
427
+ text_data = _text_with_operation_id(text_data, request_state.operation_id)
428
+ text_data = _text_with_thread_cache_identity(text_data, _bridge_thread_cache_identity(session))
429
+ # Operation metadata is added after the initial payload sizing pass. Check
430
+ # the exact frame that will cross the websocket so the metadata cannot
431
+ # push an otherwise-valid response.create over the upstream limit.
432
+ _enforce_http_bridge_response_create_text_size(request_state, text_data)
433
+ if on_send_started is not None:
434
+ on_send_started()
435
+ token = set_request_id(request_state.archive_request_id)
436
+ try:
437
+ request_state.response_create_attempt_count += 1
438
+ attempt = _HTTPBridgeResponseCreateAttempt(ordinal=request_state.response_create_attempt_count)
439
+ request_state.response_create_attempt = attempt
440
+ request_state.response_create_sent_at = clock.monotonic()
441
+ session.upstream_reader_wakeup.set()
442
+ try:
443
+ await session.upstream.send_text(text_data)
444
+ except BaseException:
445
+ # A failed or cancelled send is settled by its caller. Disarm the
446
+ # owner watchdog before lifecycle ownership is released so the
447
+ # reader cannot race that cleanup and settle the request twice.
448
+ attempt.disarmed = True
449
+ if request_state.response_create_attempt is attempt:
450
+ request_state.response_create_sent_at = None
451
+ session.upstream_reader_wakeup.set()
452
+ raise
453
+ finally:
454
+ reset_request_id(token)
455
+
456
+
457
+ async def _settle_claimed_http_bridge_liveness_failure(
458
+ service: Any,
459
+ session: "_HTTPBridgeSession",
460
+ *,
461
+ error_message: str,
462
+ ) -> None:
463
+ """Finish the pending-deque settlement claimed beside a failed send."""
464
+
465
+ if session.liveness_settlement_owner != "send":
466
+ raise RuntimeError("HTTP bridge liveness settlement started without the send claim")
467
+ async with session.lifecycle_lock:
468
+ await service._fail_http_bridge_reader_and_maybe_retire(
469
+ session,
470
+ error_code=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE,
471
+ error_message=error_message,
472
+ penalize_account=False,
473
+ force_retire=True,
474
+ )
475
+
476
+
477
+ _CLIENT_METADATA_TAIL_KEY = ',"client_metadata":'
478
+
479
+
480
+ def _dumps_compact(value: JsonValue) -> str:
481
+ return json.dumps(value, ensure_ascii=True, separators=(",", ":"))
482
+
483
+
484
+ def _splice_account_installation_id(text_data: str, codex_installation_id: str | None) -> str | None:
485
+ """Stamp ``client_metadata`` without decoding the rest of the frame.
486
+
487
+ Compact ``json.dumps`` (fixed separators, ``ensure_ascii``, insertion
488
+ order) is compositional: an object encodes as the concatenation of its
489
+ independently encoded members. For a frame produced by that encoder,
490
+ rewriting only the trailing top-level ``client_metadata`` value therefore
491
+ yields exactly the bytes a full decode/rewrite/encode would. This covers the
492
+ two dominant shapes -- ``client_metadata`` as the last key, or absent -- and
493
+ returns ``None`` for anything else so the caller falls back to the full
494
+ round trip.
495
+ """
496
+ if len(text_data) < 3 or text_data[0] != "{" or text_data[-1] != "}":
497
+ return None
498
+ index = text_data.rfind(_CLIENT_METADATA_TAIL_KEY)
499
+ if index < 0:
500
+ if '"client_metadata"' in text_data:
501
+ # Leading or nested key: not provably absent at the top level.
502
+ return None
503
+ inserted: dict[str, JsonValue] = {}
504
+ apply_codex_installation_metadata(inserted, codex_installation_id)
505
+ inserted_metadata = inserted.get("client_metadata")
506
+ if inserted_metadata is None:
507
+ return text_data
508
+ return f"{text_data[:-1]}{_CLIENT_METADATA_TAIL_KEY}{_dumps_compact(inserted_metadata)}}}"
509
+ encoded_metadata = text_data[index + len(_CLIENT_METADATA_TAIL_KEY) : -1]
510
+ if not encoded_metadata.startswith("{"):
511
+ return None
512
+ try:
513
+ raw_metadata = json.loads(encoded_metadata)
514
+ except json.JSONDecodeError:
515
+ # Nested match, or a value followed by further keys: not the trailing top-level key.
516
+ return None
517
+ if not isinstance(raw_metadata, dict):
518
+ return None
519
+ container: dict[str, JsonValue] = {"client_metadata": cast(dict[str, JsonValue], raw_metadata)}
520
+ apply_codex_installation_metadata(container, codex_installation_id)
521
+ metadata = container.get("client_metadata")
522
+ if metadata is None:
523
+ return f"{text_data[:index]}}}"
524
+ updated_metadata = _dumps_compact(metadata)
525
+ if updated_metadata == encoded_metadata:
526
+ return text_data
527
+ return f"{text_data[:index]}{_CLIENT_METADATA_TAIL_KEY}{updated_metadata}}}"
528
+
529
+
530
+ def _text_with_account_installation_id(text_data: str, codex_installation_id: str | None) -> str:
531
+ spliced = _splice_account_installation_id(text_data, codex_installation_id)
532
+ if spliced is not None:
533
+ return spliced
534
+ payload = json.loads(text_data)
535
+ if not isinstance(payload, dict):
536
+ return text_data
537
+ apply_codex_installation_metadata(cast(dict[str, JsonValue], payload), codex_installation_id)
538
+ return _dumps_compact(payload)
539
+
540
+
541
+ def _text_with_operation_id(text_data: str, operation_id: str | None) -> str:
542
+ """Attach a stable operation identity without changing the request contract."""
543
+ if not operation_id:
544
+ return text_data
545
+ try:
546
+ payload = json.loads(text_data)
547
+ except (TypeError, json.JSONDecodeError):
548
+ return text_data
549
+ if not isinstance(payload, dict):
550
+ return text_data
551
+ raw_metadata = payload.get("client_metadata")
552
+ metadata = dict(raw_metadata) if isinstance(raw_metadata, dict) else {}
553
+ # This namespace is reserved by the bridge; never trust a caller-supplied
554
+ # value to stand in for the durable operation identity.
555
+ metadata["codex_lb_operation_id"] = operation_id
556
+ payload["client_metadata"] = metadata
557
+ return json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
558
+
559
+
560
+ def _text_without_operation_id(text_data: str) -> str:
561
+ """Remove caller-supplied bridge identity before durable fingerprinting."""
562
+ try:
563
+ payload = json.loads(text_data)
564
+ except (TypeError, json.JSONDecodeError):
565
+ return text_data
566
+ if not isinstance(payload, dict):
567
+ return text_data
568
+ raw_metadata = payload.get("client_metadata")
569
+ if not isinstance(raw_metadata, dict) or "codex_lb_operation_id" not in raw_metadata:
570
+ return text_data
571
+ metadata = dict(raw_metadata)
572
+ metadata.pop("codex_lb_operation_id", None)
573
+ if metadata:
574
+ payload["client_metadata"] = metadata
575
+ else:
576
+ payload.pop("client_metadata", None)
577
+ return json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
578
+
579
+
580
+ def _text_without_account_installation_id(text_data: str) -> str:
581
+ """Normalize account-specific installation metadata out of a fingerprint."""
582
+ try:
583
+ payload = json.loads(text_data)
584
+ except (TypeError, json.JSONDecodeError):
585
+ return text_data
586
+ if not isinstance(payload, dict):
587
+ return text_data
588
+ raw_metadata = payload.get("client_metadata")
589
+ if not isinstance(raw_metadata, dict):
590
+ return text_data
591
+ metadata: dict[str, JsonValue] = {}
592
+ for key, value in raw_metadata.items():
593
+ if not isinstance(key, str) or key.lower() == CODEX_INSTALLATION_ID_HEADER:
594
+ continue
595
+ if key.lower() == CODEX_TURN_METADATA_HEADER and isinstance(value, str):
596
+ try:
597
+ turn_metadata = json.loads(value)
598
+ except json.JSONDecodeError:
599
+ turn_metadata = None
600
+ if isinstance(turn_metadata, dict) and "installation_id" in turn_metadata:
601
+ turn_metadata.pop("installation_id", None)
602
+ value = json.dumps(turn_metadata, ensure_ascii=True, separators=(",", ":"))
603
+ metadata[key] = value
604
+ if metadata:
605
+ payload["client_metadata"] = metadata
606
+ else:
607
+ payload.pop("client_metadata", None)
608
+ return json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
609
+
610
+
611
+ def _text_with_previous_response_id(text_data: str, response_id: str | None) -> str:
612
+ if not response_id:
613
+ return text_data
614
+ try:
615
+ payload = json.loads(text_data)
616
+ except (TypeError, json.JSONDecodeError):
617
+ return text_data
618
+ if not isinstance(payload, dict) or not response_id:
619
+ return text_data
620
+ payload["previous_response_id"] = response_id
621
+ return json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
622
+
623
+
624
+ def _enforce_http_bridge_response_create_text_size(
625
+ request_state: _WebSocketRequestState,
626
+ text_data: str,
627
+ ) -> None:
628
+ original_request_text = request_state.request_text
629
+ request_state.request_text = text_data
630
+ try:
631
+ _enforce_response_create_size_limit(request_state)
632
+ finally:
633
+ request_state.request_text = original_request_text
634
+
635
+
636
+ def _request_kind_from_headers(headers: Mapping[str, str] | None) -> str:
637
+ if not headers:
638
+ return "normal"
639
+ raw_turn_metadata = headers.get("x-codex-turn-metadata") or headers.get("X-Codex-Turn-Metadata")
640
+ if not isinstance(raw_turn_metadata, str):
641
+ return "normal"
642
+ try:
643
+ turn_metadata = json.loads(raw_turn_metadata)
644
+ except json.JSONDecodeError:
645
+ return "normal"
646
+ if not isinstance(turn_metadata, dict):
647
+ return "normal"
648
+ raw_request_kind = turn_metadata.get("request_kind")
649
+ if not isinstance(raw_request_kind, str):
650
+ return "normal"
651
+ request_kind = raw_request_kind.strip()
652
+ if request_kind in {"normal", "prewarm"}:
653
+ return request_kind
654
+ return "normal"
655
+
656
+
657
+ def _http_bridge_session_unowned_locked(
658
+ session: "_HTTPBridgeSession",
659
+ *,
660
+ ignore_unanchored_reservation: bool = False,
661
+ ) -> bool:
662
+ """Whether no turn owns ``session`` right now; call under ``pending_lock``.
663
+
664
+ A registered admission waiter owns a turn that has not yet been counted
665
+ into the queue (it may be suspended on the pre-lock fair-share resolve,
666
+ issue #1971, or anywhere between the retry-circuit gate and dispatch);
667
+ retiring under it would fail an admitted turn with upstream_unavailable.
668
+ A deferred retirement re-runs on the turn's own drain triggers once it
669
+ proceeds; if the waiter instead fails admission, its cleanup releases the
670
+ registration and the submit finalizer retires the flagged session.
671
+ """
672
+ has_visible_pending = any(
673
+ _http_bridge_request_counts_against_queue(request_state) for request_state in session.pending_requests
674
+ )
675
+ return (
676
+ not has_visible_pending
677
+ and session.queued_request_count == 0
678
+ and session.admission_waiter_count == 0
679
+ and (ignore_unanchored_reservation or session.unanchored_reservation_id is None)
680
+ and not session.upstream_close_attempted
681
+ )
682
+
683
+
684
+ class _HTTPBridgeRequestSubmitMixin:
685
+ @staticmethod
686
+ def _http_bridge_clean_close_retry_max_count() -> int:
687
+ configured = _HTTP_BRIDGE_CLEAN_CLOSE_RETRY_MAX_COUNT
688
+ # Keep this recovery bounded even if an unsafe higher value is supplied.
689
+ return max(0, min(1, configured))
690
+
691
+ @staticmethod
692
+ def _http_bridge_clean_close_retry_jitter_seconds() -> float:
693
+ return random.uniform(0.0, _HTTP_BRIDGE_CLEAN_CLOSE_RETRY_JITTER_MAX_SECONDS)
694
+
695
+ def _prepare_http_bridge_request(
696
+ self: Any,
697
+ payload: ResponsesRequest,
698
+ headers: Mapping[str, str],
699
+ *,
700
+ api_key: ApiKeyData | None,
701
+ api_key_reservation: ApiKeyUsageReservationData | None,
702
+ request_id: str | None = None,
703
+ client_ip: str | None = None,
704
+ enforce_openai_sdk_contract: bool = True,
705
+ preserve_responses_lite_client_metadata: bool = False,
706
+ ) -> tuple[_WebSocketRequestState, str]:
707
+ # One dump feeds client-metadata derivation, the frame and the usage
708
+ # budget; ``to_payload`` is deterministic so sharing it is exact.
709
+ base_payload = payload.to_payload()
710
+ request_state, text_data = self._prepare_response_bridge_request_state(
711
+ payload,
712
+ api_key=api_key,
713
+ api_key_reservation=api_key_reservation,
714
+ include_type_field=True,
715
+ attach_event_queue=True,
716
+ transport=_REQUEST_TRANSPORT_HTTP,
717
+ client_metadata=_response_create_client_metadata(
718
+ base_payload,
719
+ headers=headers,
720
+ preserve_existing_responses_lite=preserve_responses_lite_client_metadata,
721
+ ),
722
+ headers=headers,
723
+ session_id=_owner_lookup_session_id_from_headers(headers),
724
+ request_log_id=request_id or get_request_id() or ensure_request_id(None),
725
+ enforce_openai_sdk_contract=enforce_openai_sdk_contract,
726
+ upstream_payload_base=base_payload,
727
+ )
728
+ (
729
+ request_state.useragent,
730
+ request_state.useragent_group,
731
+ request_state.conversation_id,
732
+ ) = _request_log_client_fields(headers)
733
+ request_state.client_ip = client_ip
734
+ return request_state, text_data
735
+
736
+ def _prepare_response_bridge_request_state(
737
+ self: Any,
738
+ payload: ResponsesRequest,
739
+ *,
740
+ api_key: ApiKeyData | None,
741
+ api_key_reservation: ApiKeyUsageReservationData | None,
742
+ include_type_field: bool,
743
+ attach_event_queue: bool,
744
+ transport: str,
745
+ client_metadata: Mapping[str, JsonValue] | None,
746
+ headers: Mapping[str, str] | None = None,
747
+ session_id: str | None = None,
748
+ request_id: str | None = None,
749
+ request_log_id: str | None = None,
750
+ enforce_openai_sdk_contract: bool = True,
751
+ upstream_payload_base: JsonObject | None = None,
752
+ ) -> tuple[_WebSocketRequestState, str]:
753
+ deduped_replayed_input_count: int | None = None
754
+ deduped_replayed_input_fingerprint: str | None = None
755
+ deduped_replayed_tool_call_count = 0
756
+ if payload.previous_response_id is not None and isinstance(payload.input, list):
757
+ replayed_input_items = cast(list[JsonValue], payload.input)
758
+ deduped_input_items, deduped_replayed_tool_call_count = dedupe_replayed_side_effect_input_items(
759
+ replayed_input_items,
760
+ sanitize_missing_outputs=False,
761
+ )
762
+ if deduped_replayed_tool_call_count > 0:
763
+ deduped_replayed_input_count = len(replayed_input_items)
764
+ deduped_replayed_input_fingerprint = _fingerprint_input_items(replayed_input_items)
765
+ payload = payload.model_copy(update={"input": deduped_input_items})
766
+ # The caller's dump describes the un-deduped input; it must not
767
+ # become the forwarded frame or the budget base.
768
+ upstream_payload_base = None
769
+ protected_agent_control_output_occurrences = (
770
+ _historical_agent_control_output_occurrences(cast(list[JsonValue], payload.input))
771
+ if isinstance(payload.input, list)
772
+ else {}
773
+ )
774
+ if upstream_payload_base is None:
775
+ upstream_payload_base = payload.to_payload()
776
+ # Shallow copy: every mutation below rebinds top-level keys only, so
777
+ # ``upstream_payload_base`` stays the pristine dump for the budget.
778
+ upstream_payload = dict(upstream_payload_base)
779
+ upstream_payload.pop("stream", None)
780
+ upstream_payload.pop("background", None)
781
+ if include_type_field:
782
+ upstream_payload["type"] = "response.create"
783
+ if client_metadata:
784
+ upstream_payload["client_metadata"] = client_metadata
785
+ _finalize_responses_lite_reasoning_context(
786
+ upstream_payload,
787
+ responses_lite=(
788
+ _payload_uses_responses_lite(upstream_payload)
789
+ or _payload_has_responses_lite_websocket_marker(upstream_payload)
790
+ ),
791
+ )
792
+ forwarded_service_tier = _normalize_service_tier_value(upstream_payload.get("service_tier"))
793
+ input_item_count = 0
794
+ input_full_fingerprint: str | None = None
795
+ payload_input = payload.input
796
+ if isinstance(payload_input, list):
797
+ payload_input_list = cast(list[JsonValue], payload_input)
798
+ input_item_count = len(payload_input_list)
799
+ if input_item_count > 0:
800
+ input_full_fingerprint = _fingerprint_input_items(payload_input_list)
801
+
802
+ resolved_request_id = request_id or f"ws_{uuid4().hex}"
803
+ header_request_kind = _request_kind_from_headers(headers)
804
+ generate_false_prewarm = header_request_kind == "prewarm" and upstream_payload.get("generate") is False
805
+ connection_request_kind = header_request_kind if transport == _REQUEST_TRANSPORT_WEBSOCKET else None
806
+ request_kind = (
807
+ "normal" if connection_request_kind == "prewarm" and not generate_false_prewarm else header_request_kind
808
+ )
809
+ request_state = _WebSocketRequestState(
810
+ request_id=resolved_request_id,
811
+ request_log_id=request_log_id,
812
+ archive_request_id=request_log_id or resolved_request_id,
813
+ model=payload.model,
814
+ service_tier=forwarded_service_tier,
815
+ reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
816
+ api_key_reservation=api_key_reservation,
817
+ started_at=clock_for(self).monotonic(),
818
+ thread_affinity_last_touch_at=clock_for(self).monotonic(),
819
+ api_key_reservation_last_touch_at=clock_for(self).monotonic(),
820
+ requested_service_tier=forwarded_service_tier,
821
+ awaiting_response_created=True,
822
+ event_queue=asyncio.Queue() if attach_event_queue else None,
823
+ transport=transport,
824
+ enforce_openai_sdk_contract=enforce_openai_sdk_contract,
825
+ api_key=api_key,
826
+ request_usage_budget=estimate_api_key_request_usage(payload, upstream_payload=upstream_payload_base),
827
+ previous_response_id=payload.previous_response_id,
828
+ session_id=_normalize_session_id(session_id),
829
+ hard_continuity_anchor=(
830
+ payload.previous_response_id is not None
831
+ or _sticky_key_from_turn_state_header(headers or {}) is not None
832
+ ),
833
+ # Hard turn-state operation recovery can inject its anchor later,
834
+ # after the request state is prepared. Preserve the original
835
+ # payload shape so a full resend remains eligible for denied-anchor
836
+ # retirement when that operation path supplies the id.
837
+ proxy_injected_anchor_had_full_resend_payload=_http_bridge_payload_looks_like_full_resend(payload),
838
+ payload_conversation_bound=bool(payload.conversation),
839
+ input_item_count=input_item_count,
840
+ input_full_fingerprint=input_full_fingerprint,
841
+ request_kind=request_kind,
842
+ connection_request_kind=connection_request_kind,
843
+ generate_false_prewarm=generate_false_prewarm,
844
+ )
845
+ if deduped_replayed_input_count is not None:
846
+ request_state.input_item_count = deduped_replayed_input_count
847
+ request_state.input_full_fingerprint = deduped_replayed_input_fingerprint
848
+ logger.warning(
849
+ "%s_replayed_tool_call_input_deduped request_id=%s original_items=%s deduped_to=%s "
850
+ "removed_tool_calls=%s previous_response_id=%s",
851
+ transport,
852
+ request_state.request_id,
853
+ deduped_replayed_input_count,
854
+ input_item_count,
855
+ deduped_replayed_tool_call_count,
856
+ payload.previous_response_id,
857
+ )
858
+ text_data = json.dumps(upstream_payload, ensure_ascii=True, separators=(",", ":"))
859
+ payload_size = len(text_data.encode("utf-8"))
860
+ max_bytes = _upstream_response_create_max_bytes()
861
+ if payload_size > max_bytes:
862
+ slimmed_payload, slim_summary = _slim_response_create_payload_for_upstream(
863
+ upstream_payload,
864
+ max_bytes=max_bytes,
865
+ protected_agent_control_output_occurrences=protected_agent_control_output_occurrences,
866
+ )
867
+ if slim_summary is not None:
868
+ upstream_payload = slimmed_payload
869
+ text_data = json.dumps(upstream_payload, ensure_ascii=True, separators=(",", ":"))
870
+ logger.warning(
871
+ (
872
+ "Slimmed response.create request_id=%s request_log_id=%s transport=%s "
873
+ "original_bytes=%s slimmed_bytes=%s "
874
+ "historical_tool_outputs_slimmed=%s historical_images_slimmed=%s"
875
+ ),
876
+ request_state.request_id,
877
+ request_state.request_log_id,
878
+ transport,
879
+ payload_size,
880
+ len(text_data.encode("utf-8")),
881
+ slim_summary["historical_tool_outputs_slimmed"],
882
+ slim_summary["historical_images_slimmed"],
883
+ )
884
+ request_state.request_text = text_data
885
+ _enforce_response_create_size_limit(request_state)
886
+ return request_state, text_data
887
+
888
+ def _http_bridge_text_with_account_installation_id(
889
+ self: Any,
890
+ session: "_HTTPBridgeSession",
891
+ request_state: _WebSocketRequestState,
892
+ text_data: str,
893
+ ) -> str:
894
+ codex_installation_id = getattr(session.account, "codex_installation_id", None)
895
+ # The memo holds the exact objects the previous call returned for this
896
+ # installation id; a text rewrite yields a new object and an account
897
+ # swap changes the id, so both miss and take the full stamp below.
898
+ memo_valid = request_state.installation_stamp_installation_id == codex_installation_id
899
+ if memo_valid and (
900
+ text_data is request_state.installation_stamp_text
901
+ or text_data is request_state.installation_stamp_fresh_text
902
+ ):
903
+ updated_text = text_data
904
+ else:
905
+ updated_text = _text_with_account_installation_id(text_data, codex_installation_id)
906
+ fresh_text = request_state.fresh_upstream_request_text
907
+ if fresh_text is not None and not (memo_valid and fresh_text is request_state.installation_stamp_fresh_text):
908
+ updated_fresh_text = _text_with_account_installation_id(fresh_text, codex_installation_id)
909
+ _enforce_http_bridge_response_create_text_size(request_state, updated_fresh_text)
910
+ request_state.fresh_upstream_request_text = updated_fresh_text
911
+ if updated_text == text_data:
912
+ updated_text = text_data
913
+ else:
914
+ request_state.request_text = updated_text
915
+ _enforce_response_create_size_limit(request_state)
916
+ request_state.installation_stamp_installation_id = codex_installation_id
917
+ request_state.installation_stamp_text = updated_text
918
+ request_state.installation_stamp_fresh_text = request_state.fresh_upstream_request_text
919
+ return updated_text
920
+
921
+ async def _inline_http_bridge_image_urls(
922
+ self: Any,
923
+ text_data: str,
924
+ request_state: _WebSocketRequestState,
925
+ ) -> str:
926
+ """Inline external ``input_image`` URLs into ``data:`` URLs.
927
+
928
+ The HTTP direct-stream path already does this via
929
+ ``_inline_input_image_urls`` in :mod:`app.core.clients.proxy`, but the
930
+ HTTP bridge (WebSocket pool) path was missing the conversion. The
931
+ upstream ChatGPT WebSocket only accepts ``data:image/…`` payloads; an
932
+ external ``https://`` image URL causes it to silently reject or hang
933
+ the request.
934
+
935
+ This method applies the same transformation to the already-serialised
936
+ ``text_data`` JSON that will be sent on the upstream WebSocket.
937
+ If any external image URLs survive inlining (because the fetch failed),
938
+ the request is rejected immediately with a 400 error rather than
939
+ allowing the upstream to hang.
940
+ """
941
+ settings = _service_get_settings()
942
+ # Quick string-level pre-check: skip the parse/fetch cycle when the
943
+ # payload contains no ``input_image`` items with an ``http`` URL.
944
+ if "input_image" not in text_data:
945
+ return text_data
946
+ try:
947
+ payload_dict: dict[str, JsonValue] = json.loads(text_data)
948
+ except (json.JSONDecodeError, TypeError):
949
+ return text_data
950
+ connect_timeout = getattr(settings, "upstream_connect_timeout_seconds", 5.0)
951
+ async with _service_lease_http_session()() as http_session:
952
+ image_fetch_session = _service_as_image_fetch_session()(http_session)
953
+ inlined = await _service_inline_input_image_urls()(
954
+ payload_dict,
955
+ image_fetch_session,
956
+ connect_timeout,
957
+ )
958
+ inlined = await _inline_top_level_input_image_urls(inlined, image_fetch_session, connect_timeout)
959
+ # After inlining, check if any external URLs survived (i.e. fetch
960
+ # failed). The upstream WS only accepts data: URLs so sending an
961
+ # external URL would just cause a silent hang.
962
+ remaining_external = _count_external_image_urls(inlined)
963
+ if remaining_external > 0:
964
+ raise ProxyResponseError(
965
+ 400,
966
+ openai_error(
967
+ "image_download_failed",
968
+ (
969
+ f"Failed to download {remaining_external} external image(s). "
970
+ "The upstream API only accepts inline data: URLs. "
971
+ "Send images as base64 data URLs (data:image/png;base64,...) "
972
+ "or ensure the image URLs are publicly accessible."
973
+ ),
974
+ ),
975
+ )
976
+ updated_text = json.dumps(inlined, ensure_ascii=True, separators=(",", ":"))
977
+ if updated_text == text_data:
978
+ return text_data
979
+ request_state.request_text = updated_text
980
+ _enforce_response_create_size_limit(request_state)
981
+ return updated_text
982
+
983
+ async def _submit_http_bridge_request(
984
+ self: Any,
985
+ session: "_HTTPBridgeSession",
986
+ *,
987
+ request_state: _WebSocketRequestState,
988
+ text_data: str,
989
+ queue_limit: int,
990
+ recovery_turn_state: str | None = None,
991
+ ) -> None:
992
+ request_scope_id = ensure_request_scope_id()
993
+ owned_unanchored_handoff = session.unanchored_reservation_id == request_scope_id
994
+ try:
995
+ await self._submit_http_bridge_request_with_handoff(
996
+ session,
997
+ request_state=request_state,
998
+ text_data=text_data,
999
+ queue_limit=queue_limit,
1000
+ request_scope_id=request_scope_id,
1001
+ owned_unanchored_handoff=owned_unanchored_handoff,
1002
+ recovery_turn_state=recovery_turn_state,
1003
+ )
1004
+ finally:
1005
+ _release_http_bridge_unanchored_handoff(
1006
+ session,
1007
+ request_scope_id=request_scope_id,
1008
+ )
1009
+ # Like the half-open lease below, the admission registration the
1010
+ # submit took at entry is handed back here when no dispatch took
1011
+ # it over.
1012
+ released_admission_waiter = await self._release_http_bridge_admission_preregistration(
1013
+ session, request_state=request_state
1014
+ )
1015
+ if request_state.claimed_half_open_until > 0.0 and request_state.response_create_attempt_count == 0:
1016
+ # This admission claimed the half-open probe but the request
1017
+ # never ATTEMPTED the upstream send — a poisoned-anchor
1018
+ # rejection, a recovery-journal or operation-ledger refusal,
1019
+ # a reconnect failure, or a completed-operation spool return.
1020
+ # The attempt marker, not the sent timestamp, decides this:
1021
+ # an ambiguous send failure clears the timestamp while the
1022
+ # frame may already be running upstream, and releasing that
1023
+ # probe would let a second dispatch run beside it.
1024
+ # Hand exactly that probe back and restore the transition
1025
+ # marker (an expired but positive cooldown) so the next
1026
+ # request re-claims the lease instead of the whole window
1027
+ # suppressing traffic behind a probe that never flew.
1028
+ async with self._http_bridge_retry_circuit_lock:
1029
+ unused_probe_state = self._http_bridge_retry_circuits.get(session.key)
1030
+ if (
1031
+ unused_probe_state is not None
1032
+ and unused_probe_state.half_open_until == request_state.claimed_half_open_until
1033
+ ):
1034
+ unused_probe_state.half_open_until = 0.0
1035
+ unused_probe_state.cooldown_until = clock_for(self).monotonic() - 1.0
1036
+ request_state.claimed_half_open_until = 0.0
1037
+ # Inner pre-submit cleanup may clear the reservation before control
1038
+ # returns here, so ownership must be captured before awaiting it.
1039
+ # Only that request, or one whose pre-dispatch exit just released
1040
+ # the admission registration a retirement was deferring on, can
1041
+ # make detached-session retirement newly ready; an ordinary
1042
+ # send/reader failure already owns terminal settlement, and
1043
+ # closing again would run that funnel twice.
1044
+ if (
1045
+ (owned_unanchored_handoff or released_admission_waiter)
1046
+ and session.upstream_control.retire_after_drain
1047
+ and not session.upstream_close_attempted
1048
+ ):
1049
+ await self._retire_http_bridge_after_drain_if_ready(session)
1050
+
1051
+ async def _submit_http_bridge_request_with_handoff(
1052
+ self: Any,
1053
+ session: "_HTTPBridgeSession",
1054
+ *,
1055
+ request_state: _WebSocketRequestState,
1056
+ text_data: str,
1057
+ queue_limit: int,
1058
+ request_scope_id: str,
1059
+ owned_unanchored_handoff: bool,
1060
+ recovery_turn_state: str | None = None,
1061
+ ) -> None:
1062
+ clock = clock_for(self)
1063
+ # Own admission from submit entry, not from the dispatch registration:
1064
+ # a concurrent cooldown-suppressed sibling retires a session no turn
1065
+ # owns, and between the retry-circuit gate and that registration this
1066
+ # turn would otherwise be invisible to it (issue #1943 follow-up).
1067
+ # The dispatch registration takes this over; the interruption cleanup
1068
+ # and the submit finalizer release it on every earlier exit.
1069
+ async with session.pending_lock:
1070
+ session.admission_waiter_count += 1
1071
+ request_state.admission_waiter_preregistered = True
1072
+ recovery_attempt_consumed = False
1073
+ # Eventless upstream timeouts retire the current socket. A client
1074
+ # reconnect can otherwise create a fresh socket for the same hard key
1075
+ # and submit the identical request repeatedly while the retry circuit
1076
+ # is cooling down. Gate new submissions before any reconnect/send so
1077
+ # the circuit turns this into a bounded 503 instead of another
1078
+ # response.create attempt. A proof-gated full resend remains allowed
1079
+ # because it is the client's own replay-safe request, not an opaque
1080
+ # continuation replay.
1081
+ allow_proof_gated_continuity_replay = bool(
1082
+ request_state.previous_response_id is not None
1083
+ and request_state.fresh_upstream_request_is_retry_safe
1084
+ and request_state.fresh_upstream_request_text
1085
+ and request_state.response_event_count == 0
1086
+ and request_state.replay_count == 0
1087
+ )
1088
+ admission_claimed_leases: list[float] = []
1089
+ retry_allowed = await self._http_bridge_precreated_retry_allowed(
1090
+ session,
1091
+ allow_proof_gated_continuity_replay=allow_proof_gated_continuity_replay,
1092
+ claimed_lease_out=admission_claimed_leases,
1093
+ )
1094
+ # The exact lease this admission claimed, handed out by the claim
1095
+ # itself under its own lock: empty when this request claimed
1096
+ # nothing, so a later fail-closed path can hand back only its own
1097
+ # probe and never a lease another submission installed under any
1098
+ # interleaving.
1099
+ claimed_half_open_until = admission_claimed_leases[-1] if admission_claimed_leases else 0.0
1100
+ request_state.claimed_half_open_until = claimed_half_open_until
1101
+ if not retry_allowed:
1102
+ block_seconds, block_reason = await self._http_bridge_precreated_retry_block(session)
1103
+ retry_after_seconds = max(1, math.ceil(block_seconds))
1104
+ # The session may have been created before this late admission
1105
+ # check (issue #1943). Do not leave an unsubmitted socket
1106
+ # reusable during the cooldown: hand back this request's own
1107
+ # admission registration, then retire the session when no other
1108
+ # turn owns it, letting the bounded drain path handle registry
1109
+ # detach, aliases, leases, and close ownership. An admitted
1110
+ # sibling (the half-open probe this cooldown permits) is a live
1111
+ # owner even before its dispatch registration; flagging the
1112
+ # session under it would fail that one allowed submission.
1113
+ await self._release_http_bridge_admission_preregistration(session, request_state=request_state)
1114
+ await self._retire_idle_http_bridge_session_on_cooldown_suppression(
1115
+ session,
1116
+ owned_unanchored_handoff=owned_unanchored_handoff,
1117
+ )
1118
+ _log_http_bridge_event(
1119
+ "submit_retry_circuit_suppressed",
1120
+ session.key,
1121
+ account_id=session.account.id,
1122
+ model=session.request_model,
1123
+ detail=block_reason,
1124
+ cache_key_family=session.key.affinity_kind,
1125
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1126
+ )
1127
+ cooldown_error = ProxyResponseError(
1128
+ 503,
1129
+ openai_error(
1130
+ "upstream_request_timeout",
1131
+ _http_bridge_retry_circuit_suppression_message(block_reason, retry_after_seconds),
1132
+ ),
1133
+ retry_after_seconds=retry_after_seconds,
1134
+ )
1135
+ if _http_bridge_cooldown_suppression_is_replay_safe(request_state):
1136
+ setattr(cooldown_error, _HTTP_BRIDGE_PRE_SUBMIT_FAILURE_ATTR, True)
1137
+ setattr(cooldown_error, _HTTP_BRIDGE_COOLDOWN_SUPPRESSION_ATTR, True)
1138
+ raise cooldown_error
1139
+ submit_time_anchor_quarantined = _http_bridge_session_key_poison_quarantined(self, session.key)
1140
+ submit_time_anchor_tombstoned = False
1141
+ if request_state.proxy_injected_previous_response_id and not submit_time_anchor_quarantined:
1142
+ # A remotely written abandonment tombstone arms no quarantine by
1143
+ # design; the submit-time load adopted it, and the injected
1144
+ # anchor it fences is just as dead as a quarantined one.
1145
+ async with self._http_bridge_retry_circuit_lock:
1146
+ submit_tombstone_state = self._http_bridge_retry_circuits.get(session.key)
1147
+ submit_time_anchor_tombstoned = bool(
1148
+ submit_tombstone_state is not None
1149
+ and submit_tombstone_state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
1150
+ )
1151
+ if request_state.proxy_injected_previous_response_id and (
1152
+ submit_time_anchor_quarantined or submit_time_anchor_tombstoned
1153
+ ):
1154
+ # The submit-time load above can be the first to arm the poison
1155
+ # quarantine: planning may have served a cached view while a
1156
+ # replica's opening — stamped by an arbitrary wall clock —
1157
+ # already looked expired here. The payload was serialized with
1158
+ # the anchor that quarantine just proved dead, so dispatching it
1159
+ # would spend the admitted probe on a known-poisoned attach.
1160
+ # Fail fast with the same rejection the planning gate surfaces
1161
+ # and let the client resend its full history; the planning
1162
+ # caches stay a performance bound, never a correctness
1163
+ # assumption. Client-supplied anchors are untouched — only the
1164
+ # proxy's own injection is refused.
1165
+ _log_http_bridge_event(
1166
+ "previous_response_poisoned_anchor_fail_fast",
1167
+ session.key,
1168
+ account_id=session.account.id,
1169
+ model=session.request_model,
1170
+ detail=(
1171
+ "outcome=submit_time_quarantine_fail_closed"
1172
+ if submit_time_anchor_quarantined
1173
+ else "outcome=submit_time_tombstone_fail_closed"
1174
+ ),
1175
+ cache_key_family=session.key.affinity_kind,
1176
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1177
+ )
1178
+ raise ProxyResponseError(
1179
+ 404,
1180
+ openai_error(
1181
+ "bridge_previous_response_not_found",
1182
+ "The reattached anchor for this session was proven dead; resend the full conversation.",
1183
+ ),
1184
+ )
1185
+ # Persist the recovery checkpoint only after the retry circuit has
1186
+ # admitted this request. A client reconnect suppressed by the
1187
+ # cooldown must not create or refresh a journal entry for a request
1188
+ # that was never dispatched upstream.
1189
+ if (
1190
+ request_state.fresh_upstream_request_is_retry_safe
1191
+ and request_state.fresh_upstream_request_text
1192
+ and request_state.replay_count == 0
1193
+ and request_state.recovery_attempt_fingerprint is None
1194
+ and session.durable_session_id is not None
1195
+ and session.durable_owner_epoch is not None
1196
+ ):
1197
+ attempt_fingerprint = durable_bridge_hash(request_state.fresh_upstream_request_text)
1198
+ try:
1199
+ attempt = await self._durable_bridge.record_recovery_attempt(
1200
+ session_id=session.durable_session_id,
1201
+ api_key_id=session.key.api_key_id,
1202
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
1203
+ owner_epoch=session.durable_owner_epoch,
1204
+ request_fingerprint=attempt_fingerprint,
1205
+ request_id=request_state.request_id,
1206
+ account_id=session.account.id,
1207
+ model=request_state.model,
1208
+ replay_safe=True,
1209
+ )
1210
+ if attempt is None:
1211
+ # ``None`` is the durable owner fence rejecting this
1212
+ # worker, not an unavailable journal (which raises and
1213
+ # is handled below). Never dispatch from a stale owner.
1214
+ session.closed = True
1215
+ session.upstream_control.reconnect_requested = True
1216
+ session.upstream_control.retire_after_drain = True
1217
+ _record_continuity_fail_closed(
1218
+ surface="http_bridge",
1219
+ reason="recovery_attempt_owner_fence_rejected",
1220
+ previous_response_id=request_state.previous_response_id,
1221
+ session_id=request_state.session_id,
1222
+ upstream_error_code="bridge_continuity_persistence_failed",
1223
+ )
1224
+ raise ProxyResponseError(
1225
+ 502,
1226
+ openai_error(
1227
+ "bridge_continuity_persistence_failed",
1228
+ "HTTP responses session ownership changed; retry the request.",
1229
+ ),
1230
+ )
1231
+ if getattr(attempt.state, "value", attempt.state) != "unknown":
1232
+ if getattr(attempt.state, "value", attempt.state) != "replayed":
1233
+ raise ProxyResponseError(
1234
+ 502,
1235
+ openai_error(
1236
+ "bridge_continuity_persistence_failed",
1237
+ "The recovery checkpoint was already consumed; retry the request.",
1238
+ ),
1239
+ )
1240
+ # A REPLAYED checkpoint may belong to a completed
1241
+ # operation whose finalized transcript is safe to replay.
1242
+ # Defer rejection until the operation ledger lookup below
1243
+ # can decide that terminal-spool case; nonterminal rows
1244
+ # remain fail-closed after that lookup.
1245
+ recovery_attempt_consumed = True
1246
+ if (
1247
+ not recovery_attempt_consumed
1248
+ and getattr(attempt, "request_id", request_state.request_id) != request_state.request_id
1249
+ ):
1250
+ raise ProxyResponseError(
1251
+ 502,
1252
+ openai_error(
1253
+ "bridge_continuity_persistence_failed",
1254
+ "Another recovery request is already in flight; retry the request.",
1255
+ ),
1256
+ )
1257
+ request_state.recovery_attempt_fingerprint = attempt_fingerprint
1258
+ request_state.recovery_attempt_session_id = session.durable_session_id
1259
+ request_state.recovery_attempt_owner_epoch = session.durable_owner_epoch
1260
+ except ProxyResponseError:
1261
+ raise
1262
+ except Exception as exc:
1263
+ # The journal is an additional recovery fence. Without a
1264
+ # durable UNKNOWN row, an ambiguous send cannot be claimed by
1265
+ # another owner, so fail closed instead of dispatching an
1266
+ # unjournaled recovery-safe request.
1267
+ session.closed = True
1268
+ session.upstream_control.reconnect_requested = True
1269
+ session.upstream_control.retire_after_drain = True
1270
+ _record_continuity_fail_closed(
1271
+ surface="http_bridge",
1272
+ reason="recovery_attempt_persistence_failed",
1273
+ previous_response_id=request_state.previous_response_id,
1274
+ session_id=request_state.session_id,
1275
+ upstream_error_code="bridge_continuity_persistence_failed",
1276
+ )
1277
+ raise ProxyResponseError(
1278
+ 502,
1279
+ openai_error(
1280
+ "bridge_continuity_persistence_failed",
1281
+ "Recovered response continuity could not be persisted; retry the request.",
1282
+ ),
1283
+ ) from exc
1284
+ # Account installation metadata is part of the final upstream frame.
1285
+ # Apply and size-check it before recording the operation so a local
1286
+ # payload-too-large rejection cannot leave a submitted retry fence.
1287
+ text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data)
1288
+ record_operation = getattr(self._durable_bridge, "record_operation", None)
1289
+ if (
1290
+ callable(record_operation)
1291
+ and (
1292
+ request_state.previous_response_id is not None
1293
+ or request_state.operation_rebind_required
1294
+ or recovery_attempt_consumed
1295
+ )
1296
+ and (request_state.operation_id is None or request_state.operation_rebind_required)
1297
+ and session.durable_session_id is not None
1298
+ and session.durable_owner_epoch is not None
1299
+ ):
1300
+ text_data = _text_without_operation_id(text_data)
1301
+ api_key_scope = durable_bridge_api_key_scope(session.key.api_key_id)
1302
+ operation_fingerprint = (
1303
+ request_state.operation_fingerprint
1304
+ if request_state.operation_rebind_required and request_state.operation_fingerprint is not None
1305
+ else _http_bridge_operation_fingerprint(
1306
+ session_id=session.durable_session_id,
1307
+ api_key_scope=api_key_scope,
1308
+ request_state=request_state,
1309
+ text_data=text_data,
1310
+ )
1311
+ )
1312
+ operation_id = (
1313
+ request_state.operation_id
1314
+ if request_state.operation_rebind_required and request_state.operation_id is not None
1315
+ else durable_bridge_operation_id(session.durable_session_id, operation_fingerprint)
1316
+ )
1317
+ operation_parent_response_id = (
1318
+ request_state.operation_parent_response_id
1319
+ if request_state.operation_rebind_required
1320
+ else request_state.previous_response_id
1321
+ )
1322
+ # The operation row must not be committed until the exact
1323
+ # operation-tagged frame is known to fit. Otherwise a local size
1324
+ # rejection before ``send_text`` leaves a submitted ledger row
1325
+ # that fences every identical retry as an unknown in-flight turn.
1326
+ operation_tagged_text = _text_with_operation_id(text_data, operation_id)
1327
+ _enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text)
1328
+ try:
1329
+ get_operation_by_fingerprint = getattr(self._durable_bridge, "get_operation_by_fingerprint", None)
1330
+ get_operation = getattr(self._durable_bridge, "get_operation", None)
1331
+
1332
+ async def lookup_operation() -> Any:
1333
+ operation = None
1334
+ if callable(get_operation_by_fingerprint):
1335
+ operation = await _call_with_supported_optional_kwargs(
1336
+ get_operation_by_fingerprint,
1337
+ optional_kwargs={"api_key_scope": api_key_scope},
1338
+ request_fingerprint=operation_fingerprint,
1339
+ )
1340
+ if operation is None and callable(get_operation):
1341
+ operation = await get_operation(operation_id=operation_id)
1342
+ return operation
1343
+
1344
+ existing_operation = await lookup_operation()
1345
+ if recovery_attempt_consumed and existing_operation is None:
1346
+ raise ProxyResponseError(
1347
+ 502,
1348
+ openai_error(
1349
+ "bridge_continuity_persistence_failed",
1350
+ "The recovery checkpoint was already consumed; retry the request.",
1351
+ ),
1352
+ )
1353
+ hard_turn_chain_advanced = False
1354
+ seen_hard_turn_response_ids: set[str] = set()
1355
+ while not recovery_attempt_consumed:
1356
+ terminal_hard_turn_response_id = _http_bridge_terminal_hard_turn_response_id(
1357
+ request_state,
1358
+ existing_operation,
1359
+ allow_anchored_continuation=hard_turn_chain_advanced,
1360
+ )
1361
+ if (
1362
+ terminal_hard_turn_response_id is not None
1363
+ and terminal_hard_turn_response_id not in seen_hard_turn_response_ids
1364
+ ):
1365
+ # A completed operation with the same body is the
1366
+ # prior hard turn, not a replay request: advance from
1367
+ # its response instead of replaying that transcript.
1368
+ # Keep walking the chain because repeated identical
1369
+ # turns can have several terminal operations with
1370
+ # successive response anchors.
1371
+ seen_hard_turn_response_ids.add(terminal_hard_turn_response_id)
1372
+ hard_turn_chain_advanced = True
1373
+ text_data = _text_with_previous_response_id(text_data, terminal_hard_turn_response_id)
1374
+ request_state.request_text = text_data
1375
+ _bind_http_bridge_proxy_injected_anchor(
1376
+ self,
1377
+ request_state,
1378
+ response_id=terminal_hard_turn_response_id,
1379
+ )
1380
+ request_state.hard_continuity_anchor = True
1381
+ operation_parent_response_id = terminal_hard_turn_response_id
1382
+ operation_fingerprint = durable_bridge_operation_fingerprint(
1383
+ api_key_scope=api_key_scope,
1384
+ request_text=_text_without_account_installation_id(text_data),
1385
+ )
1386
+ operation_id = durable_bridge_operation_id(
1387
+ session.durable_session_id,
1388
+ operation_fingerprint,
1389
+ )
1390
+ operation_tagged_text = _text_with_operation_id(text_data, operation_id)
1391
+ _enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text)
1392
+ existing_operation = await lookup_operation()
1393
+ continue
1394
+
1395
+ # If another worker durably observed the previous turn's
1396
+ # completion, advance a new continuation to that response
1397
+ # anchor instead of replaying the timed-out turn. Re-run
1398
+ # the operation lookup after this race-path advancement so
1399
+ # two completions observed back-to-back are both walked.
1400
+ if existing_operation is None:
1401
+ get_latest_completed = getattr(self._durable_bridge, "get_latest_completed_operation", None)
1402
+ if callable(get_latest_completed):
1403
+ completed_operation = await _call_with_supported_optional_kwargs(
1404
+ get_latest_completed,
1405
+ optional_kwargs={"request_fingerprint": operation_fingerprint},
1406
+ session_id=session.durable_session_id,
1407
+ parent_response_id=operation_parent_response_id,
1408
+ )
1409
+ if completed_operation is None:
1410
+ get_latest_completed_any_session = getattr(
1411
+ self._durable_bridge,
1412
+ "get_latest_completed_operation_any_session",
1413
+ None,
1414
+ )
1415
+ if callable(get_latest_completed_any_session):
1416
+ completed_operation = await _call_with_supported_optional_kwargs(
1417
+ get_latest_completed_any_session,
1418
+ optional_kwargs={
1419
+ "api_key_scope": api_key_scope,
1420
+ "request_fingerprint": operation_fingerprint,
1421
+ },
1422
+ parent_response_id=request_state.previous_response_id,
1423
+ )
1424
+ completed_response_id = getattr(completed_operation, "response_id", None)
1425
+ if completed_response_id and completed_response_id != request_state.previous_response_id:
1426
+ text_data = _text_with_previous_response_id(text_data, completed_response_id)
1427
+ request_state.request_text = text_data
1428
+ _bind_http_bridge_proxy_injected_anchor(
1429
+ self,
1430
+ request_state,
1431
+ response_id=completed_response_id,
1432
+ )
1433
+ operation_parent_response_id = completed_response_id
1434
+ hard_turn_chain_advanced = True
1435
+ seen_hard_turn_response_ids.add(completed_response_id)
1436
+ request_state.hard_continuity_anchor = True
1437
+ operation_fingerprint = durable_bridge_operation_fingerprint(
1438
+ api_key_scope=api_key_scope,
1439
+ request_text=_text_without_account_installation_id(text_data),
1440
+ )
1441
+ operation_id = durable_bridge_operation_id(
1442
+ session.durable_session_id,
1443
+ operation_fingerprint,
1444
+ )
1445
+ operation_tagged_text = _text_with_operation_id(text_data, operation_id)
1446
+ _enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text)
1447
+ existing_operation = await lookup_operation()
1448
+ continue
1449
+ break
1450
+ operation = await _call_with_supported_optional_kwargs(
1451
+ record_operation,
1452
+ optional_kwargs={
1453
+ "recovery_attempt_session_id": request_state.recovery_attempt_session_id
1454
+ if request_state.recovery_attempt_claimed
1455
+ else None,
1456
+ "recovery_attempt_owner_epoch": request_state.recovery_attempt_owner_epoch
1457
+ if request_state.recovery_attempt_claimed
1458
+ else None,
1459
+ "recovery_attempt_fingerprint": request_state.recovery_attempt_fingerprint
1460
+ if request_state.recovery_attempt_claimed
1461
+ else None,
1462
+ "recovery_attempt_consumed": recovery_attempt_consumed,
1463
+ },
1464
+ operation_id=operation_id,
1465
+ session_id=session.durable_session_id,
1466
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
1467
+ owner_epoch=session.durable_owner_epoch,
1468
+ request_fingerprint=operation_fingerprint,
1469
+ api_key_scope=api_key_scope,
1470
+ account_id=session.account.id,
1471
+ model=request_state.model,
1472
+ parent_response_id=operation_parent_response_id,
1473
+ request_text=text_data,
1474
+ )
1475
+ except Exception as exc:
1476
+ session.closed = True
1477
+ session.upstream_control.reconnect_requested = True
1478
+ session.upstream_control.retire_after_drain = True
1479
+ _record_continuity_fail_closed(
1480
+ surface="http_bridge",
1481
+ reason="operation_persistence_failed",
1482
+ previous_response_id=request_state.previous_response_id,
1483
+ session_id=request_state.session_id,
1484
+ upstream_error_code="bridge_continuity_persistence_failed",
1485
+ )
1486
+ raise ProxyResponseError(
1487
+ 502,
1488
+ openai_error(
1489
+ "bridge_continuity_persistence_failed",
1490
+ "Response operation continuity could not be persisted; retry the request.",
1491
+ ),
1492
+ ) from exc
1493
+ if operation is None:
1494
+ session.closed = True
1495
+ session.upstream_control.reconnect_requested = True
1496
+ session.upstream_control.retire_after_drain = True
1497
+ raise ProxyResponseError(
1498
+ 502,
1499
+ openai_error(
1500
+ "bridge_continuity_persistence_failed",
1501
+ "HTTP responses session ownership changed; retry the request.",
1502
+ ),
1503
+ )
1504
+ if recovery_attempt_consumed and operation.created:
1505
+ raise ProxyResponseError(
1506
+ 502,
1507
+ openai_error(
1508
+ "bridge_continuity_persistence_failed",
1509
+ "The recovery checkpoint was already consumed; retry the request.",
1510
+ ),
1511
+ )
1512
+ if not operation.created and operation.state == "abandoned":
1513
+ _record_continuity_fail_closed(
1514
+ surface="http_bridge",
1515
+ reason="abandoned_operation_full_history_recovery",
1516
+ previous_response_id=request_state.previous_response_id,
1517
+ session_id=request_state.session_id,
1518
+ upstream_error_code="previous_response_not_found",
1519
+ )
1520
+ # The recovery journal may already hold an UNKNOWN row for
1521
+ # this request. Release it before rejecting so an identical
1522
+ # anchored resend sees the same 400 instead of a 502
1523
+ # "another recovery request is already in flight".
1524
+ await self._cleanup_http_bridge_submit_interruption(
1525
+ session,
1526
+ request_state=request_state,
1527
+ gate_acquired=False,
1528
+ request_enqueued=False,
1529
+ counted_in_queue=False,
1530
+ )
1531
+ raise ProxyResponseError(
1532
+ 400,
1533
+ _http_bridge_client_full_history_recovery_error(),
1534
+ )
1535
+ if not operation.created and not getattr(operation, "rebound", False):
1536
+ if operation.state in {"completed", "incomplete"}:
1537
+ if getattr(operation, "event_spool_complete", False):
1538
+ get_operation_events = getattr(self._durable_bridge, "get_operation_events", None)
1539
+ replay_events = (
1540
+ await get_operation_events(operation_id=operation.operation_id)
1541
+ if callable(get_operation_events)
1542
+ else []
1543
+ )
1544
+ if replay_events and request_state.event_queue is not None:
1545
+ request_state.operation_replay = True
1546
+ request_state.operation_id = operation.operation_id
1547
+ request_state.operation_fingerprint = operation_fingerprint
1548
+ request_state.operation_registered = True
1549
+ for replay_event in replay_events:
1550
+ await request_state.event_queue.put(replay_event)
1551
+ await request_state.event_queue.put(None)
1552
+ return
1553
+ if recovery_attempt_consumed:
1554
+ raise ProxyResponseError(
1555
+ 502,
1556
+ openai_error(
1557
+ "bridge_continuity_persistence_failed",
1558
+ "The recovery checkpoint was already consumed; retry the request.",
1559
+ ),
1560
+ )
1561
+ # A prior dispatch with the same parent and body may have been
1562
+ # accepted by upstream. Without upstream idempotency/status
1563
+ # proof, never submit it a second time.
1564
+ _record_continuity_fail_closed(
1565
+ surface="http_bridge",
1566
+ reason="operation_already_recorded_no_status_proof",
1567
+ previous_response_id=request_state.previous_response_id,
1568
+ session_id=request_state.session_id,
1569
+ upstream_error_code="upstream_operation_status_unknown",
1570
+ )
1571
+ retry_after_seconds = max(
1572
+ 1,
1573
+ math.ceil(await self._http_bridge_precreated_retry_cooldown_seconds(session)),
1574
+ )
1575
+ raise ProxyResponseError(
1576
+ 503,
1577
+ openai_error(
1578
+ "upstream_operation_status_unknown",
1579
+ "The previous response operation may still be running; retry after the cooldown.",
1580
+ ),
1581
+ retry_after_seconds=retry_after_seconds,
1582
+ )
1583
+ request_state.operation_id = operation.operation_id
1584
+ request_state.operation_fingerprint = operation_fingerprint
1585
+ request_state.operation_parent_response_id = operation_parent_response_id
1586
+ request_state.operation_registered = True
1587
+ request_state.operation_rebind_required = False
1588
+ request_state.operation_created = operation.created
1589
+ request_state.operation_rebound = getattr(operation, "rebound", False)
1590
+ request_state.operation_rebound_from_session_id = getattr(operation, "rebound_from_session_id", None)
1591
+ request_state.operation_rebound_from_account_id = getattr(operation, "rebound_from_account_id", None)
1592
+ request_state.operation_rebound_from_model = getattr(operation, "rebound_from_model", None)
1593
+ request_state.operation_rebound_from_parent_response_id = getattr(
1594
+ operation, "rebound_from_parent_response_id", None
1595
+ )
1596
+ request_state.operation_persisted_response_id = getattr(operation, "response_id", None)
1597
+
1598
+ async def _cleanup_unsubmitted_recovery_claim() -> None:
1599
+ if (
1600
+ not request_state.operation_created and not request_state.operation_rebound
1601
+ ) or request_state.operation_dispatched:
1602
+ return
1603
+ await self._cleanup_http_bridge_submit_interruption(
1604
+ session,
1605
+ request_state=request_state,
1606
+ gate_acquired=False,
1607
+ request_enqueued=False,
1608
+ counted_in_queue=False,
1609
+ )
1610
+
1611
+ text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data)
1612
+ if request_state.response_id is not None or request_state.response_event_count > 0:
1613
+ await _cleanup_unsubmitted_recovery_claim()
1614
+ _log_http_bridge_event(
1615
+ "submit_after_response_event",
1616
+ session.key,
1617
+ account_id=session.account.id,
1618
+ model=session.request_model,
1619
+ detail=(
1620
+ f"response_id={request_state.response_id}, "
1621
+ f"response_events_seen={request_state.response_event_count}"
1622
+ ),
1623
+ cache_key_family=session.key.affinity_kind,
1624
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1625
+ )
1626
+ raise ProxyResponseError(
1627
+ 502,
1628
+ openai_error(
1629
+ "upstream_unavailable",
1630
+ "HTTP responses session bridge request already has upstream response events",
1631
+ error_type="server_error",
1632
+ ),
1633
+ )
1634
+ if session.upstream_control.retire_after_drain and not owned_unanchored_handoff:
1635
+ await _cleanup_unsubmitted_recovery_claim()
1636
+ await self._release_http_bridge_admission_preregistration(session, request_state=request_state)
1637
+ if not session.upstream_close_attempted:
1638
+ await self._retire_http_bridge_after_drain_if_ready(session)
1639
+ raise ProxyResponseError(
1640
+ 502,
1641
+ openai_error("upstream_unavailable", "HTTP responses session bridge is retiring"),
1642
+ )
1643
+ if session.closed:
1644
+ async with session.lifecycle_lock:
1645
+ if session.closed:
1646
+ current_session = session
1647
+ http_bridge_sessions = getattr(self, "_http_bridge_sessions", None)
1648
+ bridge_lock = getattr(self, "_http_bridge_lock", None)
1649
+ if bridge_lock is not None:
1650
+ async with bridge_lock:
1651
+ if http_bridge_sessions is not None:
1652
+ current_session = http_bridge_sessions.get(session.key)
1653
+ elif http_bridge_sessions is not None:
1654
+ current_session = http_bridge_sessions.get(session.key)
1655
+ if current_session is None and _http_bridge_key_strength(session.key) == "hard":
1656
+ await _cleanup_unsubmitted_recovery_claim()
1657
+ _log_http_bridge_event(
1658
+ "submit_on_closed",
1659
+ session.key,
1660
+ account_id=session.account.id,
1661
+ model=session.request_model,
1662
+ detail="session_unregistered_before_reconnect",
1663
+ cache_key_family=session.key.affinity_kind,
1664
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1665
+ )
1666
+ raise ProxyResponseError(
1667
+ 502,
1668
+ openai_error("upstream_unavailable", "HTTP responses session bridge is closed"),
1669
+ )
1670
+ if current_session is not session:
1671
+ _log_http_bridge_event(
1672
+ "submit_on_closed",
1673
+ session.key,
1674
+ account_id=session.account.id,
1675
+ model=session.request_model,
1676
+ detail="session_replaced_before_reconnect",
1677
+ cache_key_family=session.key.affinity_kind,
1678
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1679
+ )
1680
+ # Try reconnecting the upstream websocket first. For requests
1681
+ # carrying previous_response_id we only reconnect (send_request=
1682
+ # False) because the fresh upstream won't recognise the old
1683
+ # response id. If reconnection itself fails, raise 502 so the
1684
+ # client retries with previous_response_id intact rather than
1685
+ # receiving 400 previous_response_not_found (which causes the
1686
+ # CLI to drop previous_response_id and resend the full
1687
+ # conversation history, inflating per-turn context by ~20x).
1688
+ try:
1689
+ recovered = await self._retry_http_bridge_request_on_fresh_upstream(
1690
+ session,
1691
+ request_state=request_state,
1692
+ text_data=text_data,
1693
+ send_request=False,
1694
+ require_same_account=_http_bridge_key_strength(session.key) == "hard",
1695
+ )
1696
+ except BaseException:
1697
+ await _cleanup_unsubmitted_recovery_claim()
1698
+ raise
1699
+ if recovered:
1700
+ session.closed = False
1701
+ else:
1702
+ await _cleanup_unsubmitted_recovery_claim()
1703
+ _log_http_bridge_event(
1704
+ "submit_on_closed",
1705
+ session.key,
1706
+ account_id=session.account.id,
1707
+ model=session.request_model,
1708
+ cache_key_family=session.key.affinity_kind,
1709
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1710
+ )
1711
+ raise ProxyResponseError(
1712
+ 502,
1713
+ openai_error("upstream_unavailable", "HTTP responses session bridge is closed"),
1714
+ )
1715
+ text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data)
1716
+ request_state.session_previous_gap_ms = int(max(0.0, request_state.started_at - session.last_used_at) * 1000)
1717
+ gate_acquired = False
1718
+ request_enqueued = False
1719
+ admission_waiter_registered = False
1720
+ try:
1721
+ # Register the submit as an admission waiter BEFORE any suspension
1722
+ # outside the lock: the waiter count keeps the idle sweeper and
1723
+ # close-time retire from evicting the session while the settings
1724
+ # snapshot below resolves, and a previous turn's finalizer
1725
+ # unwinding concurrently cannot see an apparently idle session
1726
+ # and release the lease the reacquire installs.
1727
+ async with session.pending_lock:
1728
+ if request_state.admission_waiter_preregistered:
1729
+ # The submit entry already counted this turn; take that
1730
+ # registration over instead of counting it twice.
1731
+ request_state.admission_waiter_preregistered = False
1732
+ else:
1733
+ session.admission_waiter_count += 1
1734
+ admission_waiter_registered = True
1735
+ # Snapshot under the same lock: with the waiter registered, a
1736
+ # held lease cannot be idle-released, so a session that does
1737
+ # not need reacquisition here will not need it at the
1738
+ # reacquire either.
1739
+ needs_stream_lease = session.account_lease is None and not session.closed
1740
+ # Resolved before the reacquire's pending_lock — the settings-cache
1741
+ # refresh behind this can run a DB query and must not suspend the
1742
+ # critical section (issue #1971) — and only when the reacquire can
1743
+ # actually run: a session already holding its lease never depended
1744
+ # on a settings read to admit a turn.
1745
+ fair_share_threshold_pct, routing_tunables = (
1746
+ await self._http_bridge_reacquire_snapshot(session) if needs_stream_lease else (0, None)
1747
+ )
1748
+ async with session.pending_lock:
1749
+ await self._ensure_http_bridge_session_stream_lease_locked(
1750
+ session,
1751
+ request_state=request_state,
1752
+ fair_share_threshold_pct=fair_share_threshold_pct,
1753
+ routing_tunables=routing_tunables,
1754
+ )
1755
+ except BaseException:
1756
+ # Recovery claims are made before admission. If reacquiring an
1757
+ # idle session's stream lease fails, no upstream frame can have
1758
+ # been sent; restore that claim before propagating the admission
1759
+ # error so a later reconnect is not fenced as already dispatched.
1760
+ if getattr(session, "unanchored_reservation_id", None) == request_scope_id:
1761
+ session.unanchored_reservation_id = None
1762
+ cleanup_task = scheduler_for(self).create_task(
1763
+ self._cleanup_http_bridge_submit_interruption(
1764
+ session,
1765
+ request_state=request_state,
1766
+ gate_acquired=False,
1767
+ request_enqueued=False,
1768
+ counted_in_queue=False,
1769
+ admission_waiter_registered=admission_waiter_registered,
1770
+ )
1771
+ )
1772
+ await _await_task_deferring_cancellation(cleanup_task)
1773
+ raise
1774
+ try:
1775
+ await self._maybe_prewarm_http_bridge_session(
1776
+ session,
1777
+ request_state=request_state,
1778
+ text_data=text_data,
1779
+ )
1780
+ except BaseException:
1781
+ if getattr(session, "unanchored_reservation_id", None) == request_scope_id:
1782
+ session.unanchored_reservation_id = None
1783
+ cleanup_task = scheduler_for(self).create_task(
1784
+ self._cleanup_http_bridge_submit_interruption(
1785
+ session,
1786
+ request_state=request_state,
1787
+ gate_acquired=False,
1788
+ request_enqueued=False,
1789
+ counted_in_queue=False,
1790
+ admission_waiter_registered=admission_waiter_registered,
1791
+ )
1792
+ )
1793
+ await _await_task_deferring_cancellation(cleanup_task)
1794
+ raise
1795
+ try:
1796
+ # Reuses the pre-prewarm snapshot: the registered admission waiter
1797
+ # kept the lease from idle release, so this reacquire is a no-op
1798
+ # unless the session closed mid-prewarm — a fresh refresh here
1799
+ # could only stall an otherwise admissible request on a
1800
+ # TTL-expired settings read (issue #1971).
1801
+ async with session.pending_lock:
1802
+ if session.queued_request_count >= queue_limit:
1803
+ _log_http_bridge_event(
1804
+ "bridge_queue_full",
1805
+ session.key,
1806
+ account_id=session.account.id,
1807
+ model=session.request_model,
1808
+ pending_count=session.queued_request_count,
1809
+ cache_key_family=session.key.affinity_kind,
1810
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1811
+ )
1812
+ raise ProxyResponseError(
1813
+ 429,
1814
+ openai_error(
1815
+ "bridge_queue_full",
1816
+ "HTTP responses session bridge queue is full",
1817
+ error_type="rate_limit_error",
1818
+ ),
1819
+ )
1820
+ await self._ensure_http_bridge_session_stream_lease_locked(
1821
+ session,
1822
+ request_state=request_state,
1823
+ fair_share_threshold_pct=fair_share_threshold_pct,
1824
+ routing_tunables=routing_tunables,
1825
+ )
1826
+ session.queued_request_count += 1
1827
+ if getattr(session, "unanchored_reservation_id", None) == request_scope_id:
1828
+ session.unanchored_reservation_id = None
1829
+ except BaseException:
1830
+ if getattr(session, "unanchored_reservation_id", None) == request_scope_id:
1831
+ session.unanchored_reservation_id = None
1832
+ cleanup_task = scheduler_for(self).create_task(
1833
+ self._cleanup_http_bridge_submit_interruption(
1834
+ session,
1835
+ request_state=request_state,
1836
+ gate_acquired=False,
1837
+ request_enqueued=False,
1838
+ counted_in_queue=False,
1839
+ admission_waiter_registered=admission_waiter_registered,
1840
+ )
1841
+ )
1842
+ await _await_task_deferring_cancellation(cleanup_task)
1843
+ raise
1844
+ try:
1845
+ text_data = await self._inline_http_bridge_image_urls(text_data, request_state)
1846
+ text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data)
1847
+ self._start_request_state_api_key_reservation_heartbeat(
1848
+ request_state,
1849
+ api_key=request_state.api_key,
1850
+ surface="http_bridge",
1851
+ )
1852
+ _copy_websocket_route_metadata_from_session(request_state, session)
1853
+ request_state.bridge_queue_wait_started_at = clock.monotonic()
1854
+ request_state.response_create_gate_wait_started_at = clock.monotonic()
1855
+ # Bridge ownership is established before this late admission. A
1856
+ # cap race stays a bounded error/wait on that owner; it must not
1857
+ # publish a replacement bridge as a spillover side effect.
1858
+ await self._acquire_request_state_response_create_admission(
1859
+ request_state,
1860
+ response_create_gate=session.response_create_gate,
1861
+ account_id=session.account.id,
1862
+ routing_tunables=routing_tunables,
1863
+ surface="http_bridge",
1864
+ bridge_session=session,
1865
+ )
1866
+ gate_acquired = True
1867
+ if request_state.bridge_queue_wait_started_at is not None:
1868
+ request_state.latency_bridge_queue_wait_ms = int(
1869
+ max(0.0, clock.monotonic() - request_state.bridge_queue_wait_started_at) * 1000
1870
+ )
1871
+ async with session.lifecycle_lock:
1872
+ current_session = session
1873
+ http_bridge_sessions = getattr(self, "_http_bridge_sessions", None)
1874
+ bridge_lock = getattr(self, "_http_bridge_lock", None)
1875
+ if bridge_lock is not None:
1876
+ async with bridge_lock:
1877
+ if http_bridge_sessions is not None:
1878
+ current_session = http_bridge_sessions.get(session.key)
1879
+ elif http_bridge_sessions is not None:
1880
+ current_session = http_bridge_sessions.get(session.key)
1881
+ session_unregistered = current_session is None and _http_bridge_key_strength(session.key) == "hard"
1882
+ session_replaced = current_session is not None and current_session is not session
1883
+ # Queue publication clears the mutable reservation marker. The
1884
+ # proof captured before the first await still authorizes exactly
1885
+ # that request to submit on its detached, draining generation.
1886
+ detached_handoff_can_submit = (
1887
+ owned_unanchored_handoff and session.upstream_control.retire_after_drain and not session.closed
1888
+ )
1889
+ if session.closed and current_session is session and not session.upstream_control.retire_after_drain:
1890
+ recovered = await self._retry_http_bridge_request_on_fresh_upstream(
1891
+ session,
1892
+ request_state=request_state,
1893
+ text_data=text_data,
1894
+ send_request=False,
1895
+ require_same_account=_http_bridge_key_strength(session.key) == "hard",
1896
+ )
1897
+ if recovered:
1898
+ session.closed = False
1899
+ if session.closed or ((session_unregistered or session_replaced) and not detached_handoff_can_submit):
1900
+ _log_http_bridge_event(
1901
+ "submit_on_closed",
1902
+ session.key,
1903
+ account_id=session.account.id,
1904
+ model=session.request_model,
1905
+ detail=(
1906
+ "session_retired_after_admission"
1907
+ if session.closed
1908
+ else (
1909
+ "session_unregistered_after_admission"
1910
+ if session_unregistered
1911
+ else "session_replaced_after_admission"
1912
+ )
1913
+ ),
1914
+ cache_key_family=session.key.affinity_kind,
1915
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1916
+ )
1917
+ raise ProxyResponseError(
1918
+ 502,
1919
+ openai_error("upstream_unavailable", "HTTP responses session bridge is closed"),
1920
+ )
1921
+ recovery_receipt: DurableBridgeAliasRegistrationReceipt | None = None
1922
+ upstream_send_started = False
1923
+ try:
1924
+ if recovery_turn_state is not None:
1925
+ registration_cancellation: asyncio.CancelledError | None = None
1926
+ try:
1927
+ async with session.recovery_alias_lock:
1928
+ registration_task = scheduler_for(self).create_task(
1929
+ self._register_http_bridge_recovery_turn_state_locked(
1930
+ session,
1931
+ recovery_turn_state,
1932
+ )
1933
+ )
1934
+ (
1935
+ (
1936
+ recovery_alias_registered,
1937
+ recovery_receipt,
1938
+ ),
1939
+ registration_cancellation,
1940
+ ) = await _await_task_deferring_cancellation(registration_task)
1941
+ if registration_cancellation is not None or not recovery_alias_registered:
1942
+ session.closed = True
1943
+ session.upstream_control.reconnect_requested = True
1944
+ session.upstream_control.retire_after_drain = True
1945
+ if (
1946
+ recovery_receipt is not None
1947
+ and recovery_receipt.status == DurableBridgeAliasRegistration.REGISTERED
1948
+ ):
1949
+ try:
1950
+ (
1951
+ rolled_back,
1952
+ rollback_cancellation,
1953
+ ) = await _rollback_http_bridge_recovery_turn_state_registration(
1954
+ self,
1955
+ recovery_receipt,
1956
+ )
1957
+ except Exception:
1958
+ rolled_back = False
1959
+ rollback_cancellation = None
1960
+ logger.warning(
1961
+ "Failed to roll back recovered HTTP bridge turn-state alias",
1962
+ exc_info=True,
1963
+ )
1964
+ registration_cancellation = registration_cancellation or rollback_cancellation
1965
+ recovery_receipt = None
1966
+ if not rolled_back:
1967
+ _record_continuity_fail_closed(
1968
+ surface="http_bridge",
1969
+ reason="recovery_alias_rollback_failed",
1970
+ previous_response_id=request_state.previous_response_id,
1971
+ session_id=request_state.session_id,
1972
+ upstream_error_code="bridge_continuity_persistence_failed",
1973
+ )
1974
+ if registration_cancellation is not None:
1975
+ raise registration_cancellation
1976
+ except asyncio.CancelledError:
1977
+ raise
1978
+ except Exception:
1979
+ logger.warning(
1980
+ "Failed to persist recovered HTTP bridge turn-state before upstream dispatch",
1981
+ exc_info=True,
1982
+ )
1983
+ recovery_alias_registered = False
1984
+ if not recovery_alias_registered:
1985
+ session.closed = True
1986
+ session.upstream_control.reconnect_requested = True
1987
+ session.upstream_control.retire_after_drain = True
1988
+ _record_continuity_fail_closed(
1989
+ surface="http_bridge",
1990
+ reason="recovery_alias_registration_failed",
1991
+ previous_response_id=request_state.previous_response_id,
1992
+ session_id=request_state.session_id,
1993
+ upstream_error_code="bridge_continuity_persistence_failed",
1994
+ )
1995
+ raise ProxyResponseError(
1996
+ 502,
1997
+ openai_error(
1998
+ "bridge_continuity_persistence_failed",
1999
+ "Recovered response continuity could not be persisted; retry the request.",
2000
+ ),
2001
+ )
2002
+ if request_state.recovery_attempt_fingerprint is not None:
2003
+ try:
2004
+ owner_lookup = await self._durable_bridge.renew_live_session(
2005
+ session_id=session.durable_session_id,
2006
+ api_key_id=session.key.api_key_id,
2007
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
2008
+ owner_epoch=session.durable_owner_epoch,
2009
+ lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(),
2010
+ latest_turn_state=session.downstream_turn_state,
2011
+ latest_response_id=None,
2012
+ )
2013
+ except Exception as exc:
2014
+ session.closed = True
2015
+ session.upstream_control.reconnect_requested = True
2016
+ session.upstream_control.retire_after_drain = True
2017
+ raise ProxyResponseError(
2018
+ 502,
2019
+ openai_error(
2020
+ "bridge_continuity_persistence_failed",
2021
+ "HTTP responses session ownership could not be renewed; retry the request.",
2022
+ ),
2023
+ ) from exc
2024
+ if (
2025
+ owner_lookup is None
2026
+ or owner_lookup.owner_instance_id
2027
+ != _service_get_settings().http_responses_session_bridge_instance_id
2028
+ or owner_lookup.owner_epoch != session.durable_owner_epoch
2029
+ ):
2030
+ session.closed = True
2031
+ session.upstream_control.reconnect_requested = True
2032
+ session.upstream_control.retire_after_drain = True
2033
+ raise ProxyResponseError(
2034
+ 502,
2035
+ openai_error(
2036
+ "bridge_continuity_persistence_failed",
2037
+ "HTTP responses session ownership changed before dispatch; retry the request.",
2038
+ ),
2039
+ )
2040
+ # The journal entry is created before queue admission so
2041
+ # concurrent requests cannot both enter the gate without
2042
+ # a recovery generation. Revalidate it after the gate and
2043
+ # lifecycle locks, immediately before dispatch: a waiter
2044
+ # may have observed UNKNOWN while the first request
2045
+ # settled the row REPLAYED.
2046
+ if (
2047
+ request_state.recovery_attempt_fingerprint is not None
2048
+ and not request_state.recovery_attempt_claimed
2049
+ ):
2050
+ if (
2051
+ request_state.recovery_attempt_session_id != session.durable_session_id
2052
+ or request_state.recovery_attempt_owner_epoch != session.durable_owner_epoch
2053
+ ):
2054
+ session.closed = True
2055
+ session.upstream_control.reconnect_requested = True
2056
+ session.upstream_control.retire_after_drain = True
2057
+ raise ProxyResponseError(
2058
+ 502,
2059
+ openai_error(
2060
+ "bridge_continuity_persistence_failed",
2061
+ "HTTP responses session ownership changed before dispatch; retry the request.",
2062
+ ),
2063
+ )
2064
+ try:
2065
+ dispatch_attempt = await self._durable_bridge.record_recovery_attempt(
2066
+ session_id=session.durable_session_id,
2067
+ api_key_id=session.key.api_key_id,
2068
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
2069
+ owner_epoch=session.durable_owner_epoch,
2070
+ request_fingerprint=request_state.recovery_attempt_fingerprint,
2071
+ request_id=request_state.request_id,
2072
+ account_id=session.account.id,
2073
+ model=request_state.model,
2074
+ replay_safe=True,
2075
+ )
2076
+ except Exception as exc:
2077
+ session.closed = True
2078
+ session.upstream_control.reconnect_requested = True
2079
+ session.upstream_control.retire_after_drain = True
2080
+ raise ProxyResponseError(
2081
+ 502,
2082
+ openai_error(
2083
+ "bridge_continuity_persistence_failed",
2084
+ "Recovered response continuity could not be revalidated; retry the request.",
2085
+ ),
2086
+ ) from exc
2087
+ if (
2088
+ dispatch_attempt is None
2089
+ or getattr(dispatch_attempt.state, "value", dispatch_attempt.state) != "unknown"
2090
+ or getattr(dispatch_attempt, "request_id", request_state.request_id)
2091
+ != request_state.request_id
2092
+ ):
2093
+ session.closed = True
2094
+ session.upstream_control.reconnect_requested = True
2095
+ session.upstream_control.retire_after_drain = True
2096
+ raise ProxyResponseError(
2097
+ 502,
2098
+ openai_error(
2099
+ "bridge_continuity_persistence_failed",
2100
+ "The recovery checkpoint was consumed before dispatch; retry the request.",
2101
+ ),
2102
+ )
2103
+ if (
2104
+ request_state.proxy_injected_previous_response_id
2105
+ and request_state.previous_response_id is not None
2106
+ and (
2107
+ request_state.previous_response_id in session.denied_proxy_injected_anchor_ids
2108
+ or request_state.denied_proxy_injected_anchor_fence_was_already_denied
2109
+ or _http_bridge_denied_anchor_fence_advanced(self, request_state)
2110
+ )
2111
+ ):
2112
+ _record_continuity_fail_closed(
2113
+ surface="http_bridge",
2114
+ reason="denied_proxy_anchor_before_dispatch",
2115
+ previous_response_id=request_state.previous_response_id,
2116
+ session_id=request_state.session_id,
2117
+ upstream_error_code="previous_response_not_found",
2118
+ )
2119
+ # The fence sits above the pending-request enqueue and
2120
+ # the upstream send, so this refusal is proven to be
2121
+ # ours rather than an upstream transport failure. Say
2122
+ # so on the exception: without the flag the native
2123
+ # Codex lifecycle aborts the committed body with no
2124
+ # terminal event and the client sees an empty 200
2125
+ # (issue #2364).
2126
+ raise ProxyResponseError(
2127
+ 502,
2128
+ openai_error(
2129
+ "stream_incomplete",
2130
+ "The previous response anchor was rejected upstream; retry the request.",
2131
+ ),
2132
+ local_pre_dispatch_refusal=True,
2133
+ )
2134
+ if (
2135
+ request_state.verified_stale_anchor_replay
2136
+ and request_state.verified_stale_anchor_retry_circuit_generation_captured
2137
+ ):
2138
+ circuit_key = request_state.verified_stale_anchor_retry_circuit_key
2139
+ claim_outcome: bool | None = False
2140
+ if circuit_key is not None:
2141
+ claim_outcome = await self._claim_http_bridge_retry_circuit_generation(
2142
+ key=circuit_key,
2143
+ captured=request_state.verified_stale_anchor_retry_circuit_generation_captured,
2144
+ generation=request_state.verified_stale_anchor_retry_circuit_generation,
2145
+ )
2146
+ generation_claimed = claim_outcome is True
2147
+ if not generation_claimed:
2148
+ remote_probe_holds_lease = False
2149
+ if claim_outcome is False and circuit_key is not None:
2150
+ remote_probe_holds_lease = await self._http_bridge_claim_miss_shows_remote_probe(
2151
+ circuit_key,
2152
+ request_state.verified_stale_anchor_retry_circuit_generation,
2153
+ )
2154
+ # The block lives on the source hard key: this
2155
+ # replacement session's own unique key never has a
2156
+ # circuit, and the source key's cooldown alone is
2157
+ # already zero during its half-open lease.
2158
+ (
2159
+ suppressed_block_seconds,
2160
+ suppressed_block_reason,
2161
+ ) = await self._http_bridge_precreated_retry_block_for_key(
2162
+ circuit_key or session.key,
2163
+ assume_remote_half_open_lease=remote_probe_holds_lease,
2164
+ )
2165
+ suppressed_retry_after_seconds = max(1, math.ceil(suppressed_block_seconds))
2166
+ _log_http_bridge_event(
2167
+ "stale_anchor_replay_generation_suppressed",
2168
+ circuit_key or session.key,
2169
+ account_id=session.account.id,
2170
+ model=session.request_model,
2171
+ detail="circuit_generation_advanced_before_dispatch_claim",
2172
+ cache_key_family=(circuit_key or session.key).affinity_kind,
2173
+ model_class=(
2174
+ _extract_model_class(session.request_model) if session.request_model else None
2175
+ ),
2176
+ )
2177
+ raise ProxyResponseError(
2178
+ 503,
2179
+ openai_error(
2180
+ "upstream_request_timeout",
2181
+ _http_bridge_retry_circuit_suppression_message(
2182
+ suppressed_block_reason, suppressed_retry_after_seconds
2183
+ ),
2184
+ ),
2185
+ retry_after_seconds=suppressed_retry_after_seconds,
2186
+ )
2187
+ async with session.pending_lock:
2188
+ session.pending_requests.append(request_state)
2189
+ session.admission_waiter_count = max(0, session.admission_waiter_count - 1)
2190
+ admission_waiter_registered = False
2191
+ request_enqueued = True
2192
+
2193
+ def mark_upstream_send_started() -> None:
2194
+ nonlocal upstream_send_started
2195
+ # The helper invokes this only after the final frame
2196
+ # size preflight. A payload_too_large rejection must
2197
+ # therefore remain proven pre-dispatch so cleanup can
2198
+ # roll back a newly-created operation.
2199
+ upstream_send_started = True
2200
+
2201
+ try:
2202
+ await _send_http_bridge_request_text_with_archive_id(
2203
+ session,
2204
+ request_state,
2205
+ text_data,
2206
+ on_send_started=mark_upstream_send_started,
2207
+ clock=clock,
2208
+ )
2209
+ except BaseException as exc:
2210
+ request_state.recovery_attempt_dispatched = upstream_send_started
2211
+ request_state.operation_dispatched = (
2212
+ request_state.operation_id is not None and upstream_send_started
2213
+ )
2214
+ # Publish retirement while lifecycle ownership is still
2215
+ # held; a gate waiter must never reuse an ambiguously sent
2216
+ # response.create socket between unlock and cleanup.
2217
+ session.closed = True
2218
+ session.upstream_control.reconnect_requested = True
2219
+ session.upstream_control.retire_after_drain = True
2220
+ if (
2221
+ isinstance(exc, UpstreamWebSocketTransportError)
2222
+ and exc.error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE
2223
+ ):
2224
+ # Only this narrow claim, not ``closed``, tells the
2225
+ # reader that the submitter will settle siblings.
2226
+ # Keep it inside lifecycle_lock with the failing
2227
+ # send so the reader cannot observe an ownership gap.
2228
+ session.claim_liveness_settlement()
2229
+ raise
2230
+ request_state.recovery_attempt_dispatched = True
2231
+ request_state.operation_dispatched = request_state.operation_id is not None
2232
+ session.last_used_at = clock.monotonic()
2233
+ except asyncio.CancelledError:
2234
+ if recovery_receipt is not None and not upstream_send_started:
2235
+ session.closed = True
2236
+ session.upstream_control.reconnect_requested = True
2237
+ session.upstream_control.retire_after_drain = True
2238
+ async with session.recovery_alias_lock:
2239
+ try:
2240
+ (
2241
+ rolled_back,
2242
+ _rollback_cancellation,
2243
+ ) = await _rollback_http_bridge_recovery_turn_state_registration(
2244
+ self,
2245
+ recovery_receipt,
2246
+ )
2247
+ except Exception:
2248
+ rolled_back = False
2249
+ logger.warning(
2250
+ "Failed to roll back cancelled HTTP bridge recovery alias",
2251
+ exc_info=True,
2252
+ )
2253
+ if not rolled_back:
2254
+ _record_continuity_fail_closed(
2255
+ surface="http_bridge",
2256
+ reason="recovery_alias_rollback_failed",
2257
+ previous_response_id=request_state.previous_response_id,
2258
+ session_id=request_state.session_id,
2259
+ upstream_error_code="bridge_continuity_persistence_failed",
2260
+ )
2261
+ raise
2262
+ except ProxyResponseError:
2263
+ await self._cleanup_http_bridge_submit_interruption(
2264
+ session,
2265
+ request_state=request_state,
2266
+ gate_acquired=gate_acquired,
2267
+ request_enqueued=request_enqueued,
2268
+ counted_in_queue=True,
2269
+ admission_waiter_registered=admission_waiter_registered,
2270
+ )
2271
+ if not session.upstream_close_attempted:
2272
+ await self._retire_http_bridge_after_drain_if_ready(session)
2273
+ raise
2274
+ except asyncio.CancelledError as cancellation:
2275
+ cleanup_task = scheduler_for(self).create_task(
2276
+ self._cleanup_http_bridge_submit_interruption(
2277
+ session,
2278
+ request_state=request_state,
2279
+ gate_acquired=gate_acquired,
2280
+ request_enqueued=request_enqueued,
2281
+ counted_in_queue=True,
2282
+ admission_waiter_registered=admission_waiter_registered,
2283
+ )
2284
+ )
2285
+ try:
2286
+ await _await_task_deferring_cancellation(cleanup_task)
2287
+ except Exception:
2288
+ logger.warning("Failed to clean up cancelled HTTP bridge submit", exc_info=True)
2289
+ if session.upstream_control.retire_after_drain and not session.upstream_close_attempted:
2290
+ retire_task = scheduler_for(self).create_task(self._retire_http_bridge_after_drain_if_ready(session))
2291
+ try:
2292
+ await _await_task_deferring_cancellation(retire_task)
2293
+ except Exception:
2294
+ logger.warning("Failed to retire cancelled HTTP bridge recovery", exc_info=True)
2295
+ raise cancellation
2296
+ except Exception as exc:
2297
+ _log_http_bridge_event(
2298
+ "send_failure",
2299
+ session.key,
2300
+ account_id=session.account.id,
2301
+ model=session.request_model,
2302
+ detail=str(exc) or None,
2303
+ cache_key_family=session.key.affinity_kind,
2304
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
2305
+ )
2306
+ # send_text may fail after the complete response.create frame was
2307
+ # handed to the kernel. Never reconnect-and-resend from this path;
2308
+ # only failures proven to precede dispatch may be replayed.
2309
+ error_code = exc.error_code if isinstance(exc, UpstreamWebSocketTransportError) else "stream_incomplete"
2310
+ failure_error_message = str(exc) or "Upstream websocket closed before response.completed"
2311
+ # Liveness expiry and local network loss are transport failures,
2312
+ # not evidence against the selected account. Keep this in sync
2313
+ # with the reader path's shared provenance classification.
2314
+ account_neutral = is_account_neutral_websocket_error_code(error_code)
2315
+ if error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE:
2316
+ # The sender claimed ownership beside the failing send while
2317
+ # holding lifecycle_lock. It therefore owns the entire session
2318
+ # deque, including older in-flight requests; settling only this
2319
+ # request would strand its siblings after the reader yields.
2320
+ # Publish the cleanup task before the first await after the
2321
+ # claim. Shielding it makes cancellation wait for settlement,
2322
+ # so the claim can never outlive its exactly-once owner.
2323
+ settlement_task = scheduler_for(self).create_task(
2324
+ _settle_claimed_http_bridge_liveness_failure(
2325
+ self,
2326
+ session,
2327
+ error_message=str(exc) or "Upstream websocket liveness failed",
2328
+ ),
2329
+ name="http-bridge-liveness-send-settlement",
2330
+ )
2331
+ _, settlement_cancellation = await _await_task_deferring_cancellation(settlement_task)
2332
+ if settlement_cancellation is not None:
2333
+ raise settlement_cancellation
2334
+ else:
2335
+ # Once the operation-tagged frame has been handed to the
2336
+ # socket, the transport exception is ambiguous: upstream may
2337
+ # have accepted it even though this worker saw no
2338
+ # acknowledgement. Persist UNKNOWN under the owner fence
2339
+ # before cleanup can retire the closed session and release
2340
+ # that fence.
2341
+ if (
2342
+ request_state.operation_dispatched
2343
+ and request_state.operation_registered
2344
+ and request_state.operation_id is not None
2345
+ and session.durable_session_id is not None
2346
+ and session.durable_owner_epoch is not None
2347
+ ):
2348
+ mark_operation_unknown = getattr(self._durable_bridge, "mark_operation_unknown", None)
2349
+ marked_unknown = False
2350
+ if callable(mark_operation_unknown):
2351
+ try:
2352
+ marked_unknown = await mark_operation_unknown(
2353
+ operation_id=request_state.operation_id,
2354
+ session_id=session.durable_session_id,
2355
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
2356
+ owner_epoch=session.durable_owner_epoch,
2357
+ )
2358
+ except Exception:
2359
+ logger.warning(
2360
+ "Failed to mark ambiguous HTTP bridge operation UNKNOWN operation_id=%s",
2361
+ request_state.operation_id,
2362
+ exc_info=True,
2363
+ )
2364
+ if not marked_unknown:
2365
+ request_state.operation_registered = False
2366
+ error_code = "bridge_continuity_persistence_failed"
2367
+ failure_error_message = (
2368
+ "Ambiguous response operation could not be persisted; retry the request."
2369
+ )
2370
+ _record_continuity_fail_closed(
2371
+ surface="http_bridge",
2372
+ reason="ambiguous_operation_unknown_persistence_failed",
2373
+ previous_response_id=request_state.previous_response_id,
2374
+ session_id=request_state.session_id,
2375
+ upstream_error_code=error_code,
2376
+ )
2377
+ await self._cleanup_http_bridge_submit_interruption(
2378
+ session,
2379
+ request_state=request_state,
2380
+ gate_acquired=gate_acquired,
2381
+ request_enqueued=request_enqueued,
2382
+ counted_in_queue=True,
2383
+ admission_waiter_registered=admission_waiter_registered,
2384
+ )
2385
+ await self._fail_pending_websocket_requests(
2386
+ account=session.account,
2387
+ account_id_value=session.account.id,
2388
+ pending_requests=deque([request_state]),
2389
+ pending_lock=fast_lock(),
2390
+ error_code=error_code,
2391
+ error_message=failure_error_message,
2392
+ api_key=None,
2393
+ response_create_gate=session.response_create_gate,
2394
+ penalize_account=not account_neutral,
2395
+ )
2396
+ session.closed = True
2397
+ try:
2398
+ await session.upstream.close()
2399
+ except Exception:
2400
+ logger.debug("Failed to close HTTP bridge upstream websocket after send failure", exc_info=True)
2401
+ # Always raise 502 so the client can retry with
2402
+ # previous_response_id intact. Returning 400
2403
+ # previous_response_not_found causes the client to drop
2404
+ # previous_response_id and resend the full conversation
2405
+ # history, inflating per-turn context by ~20x.
2406
+ raise ProxyResponseError(
2407
+ 502,
2408
+ openai_error(error_code, failure_error_message),
2409
+ ) from exc
2410
+
2411
+ async def _maybe_prewarm_http_bridge_session(
2412
+ self: Any,
2413
+ session: "_HTTPBridgeSession",
2414
+ *,
2415
+ request_state: _WebSocketRequestState,
2416
+ text_data: str,
2417
+ ) -> None:
2418
+ clock = clock_for(self)
2419
+ settings = _service_get_settings()
2420
+ if not session.codex_session or session.prewarmed or request_state.previous_response_id is not None:
2421
+ request_state.prewarm_status = request_state.prewarm_status or "not_applicable"
2422
+ return
2423
+ # M3 codex prewarm: the switch is dashboard-managed, and it reaches this
2424
+ # path through the request-bound overlay already applied by
2425
+ # ``_service_get_settings()`` above (the field is in
2426
+ # ``DASHBOARD_OVERRIDE_SETTINGS``, and the entry-point middleware bound
2427
+ # the snapshot once for this request). Resolving it is therefore a plain
2428
+ # memory read that adds neither a settings read nor an ``await`` -- here
2429
+ # or, more importantly, under ``prewarm_lock`` below, where a cache
2430
+ # refresh could run a DB query and suspend (issues #1971 and #1972
2431
+ # wedged every keyed submit on exactly that pattern).
2432
+ if not _http_bridge_prewarm_enabled(settings):
2433
+ request_state.prewarm_status = "not_applicable"
2434
+ return
2435
+ prewarm_lock = session.prewarm_lock
2436
+ if prewarm_lock is None:
2437
+ request_state.prewarm_status = "skipped"
2438
+ _record_http_bridge_prewarm_outcome(outcome="skipped")
2439
+ return
2440
+ # Build the warm-up outside the lock. It is a pure function of
2441
+ # ``text_data``, so the locked body gets the same answer, and this keeps
2442
+ # a parse of a possibly large payload off the critical section.
2443
+ warmup_text = _build_http_bridge_prewarm_text(text_data)
2444
+ # The two settings readers the locked body reaches -- the admission
2445
+ # gate's account caps/tunables and the reconnect the timeout path takes
2446
+ # -- get this one row instead of reading it themselves, so neither
2447
+ # awaits the settings cache while ``prewarm_lock`` is held. A refresh
2448
+ # behind that read runs a DB query under a process-global lock; one
2449
+ # stalled query would then hold this session's prewarm lock and stall
2450
+ # every later turn on it (issues #1971 and #1972). A payload with no
2451
+ # warm-up to send reaches neither reader, so it resolves nothing and
2452
+ # reads nothing, exactly as before the snapshot existed.
2453
+ dashboard_settings: DashboardSettings | None = None
2454
+ if warmup_text is not None:
2455
+ settings_cache = _service_get_settings_cache()
2456
+ try:
2457
+ dashboard_settings = await settings_cache.get()
2458
+ except Exception: # noqa: BLE001 - a prewarm must never fail a servable request
2459
+ # The same fallback the request entry point's dashboard-overrides
2460
+ # middleware applies to this row: prefer the last one this replica
2461
+ # loaded. With no row at all the prewarm is skipped rather than run
2462
+ # without a snapshot, which would put the read back under the lock.
2463
+ dashboard_settings = settings_cache.cached_row()
2464
+ logger.warning(
2465
+ "HTTP bridge prewarm settings snapshot unavailable; %s",
2466
+ "using the last loaded dashboard values"
2467
+ if dashboard_settings is not None
2468
+ else "skipping the prewarm",
2469
+ exc_info=True,
2470
+ )
2471
+ if dashboard_settings is None:
2472
+ request_state.prewarm_status = "skipped"
2473
+ _record_http_bridge_prewarm_outcome(outcome="skipped")
2474
+ return
2475
+ async with prewarm_lock:
2476
+ if session.prewarmed:
2477
+ request_state.prewarm_status = "skipped"
2478
+ _record_http_bridge_prewarm_outcome(outcome="skipped")
2479
+ return
2480
+ session.prewarmed = True
2481
+ if warmup_text is None:
2482
+ request_state.prewarm_status = "skipped"
2483
+ _record_http_bridge_prewarm_outcome(outcome="skipped")
2484
+ return
2485
+
2486
+ prewarm_started_at = clock.monotonic()
2487
+ warmup_state = _WebSocketRequestState(
2488
+ request_id=f"http_prewarm_{uuid4().hex}",
2489
+ model=request_state.model,
2490
+ service_tier=request_state.service_tier,
2491
+ reasoning_effort=request_state.reasoning_effort,
2492
+ api_key_reservation=None,
2493
+ started_at=clock.monotonic(),
2494
+ thread_affinity_last_touch_at=clock.monotonic(),
2495
+ api_key_reservation_last_touch_at=clock.monotonic(),
2496
+ requested_service_tier=request_state.requested_service_tier,
2497
+ actual_service_tier=request_state.actual_service_tier,
2498
+ awaiting_response_created=True,
2499
+ event_queue=asyncio.Queue(),
2500
+ transport=_REQUEST_TRANSPORT_HTTP,
2501
+ request_text=warmup_text,
2502
+ skip_request_log=True,
2503
+ )
2504
+ gate_acquired = False
2505
+ request_enqueued = False
2506
+ warmup_send_started = False
2507
+ try:
2508
+ event_queue = warmup_state.event_queue
2509
+ assert event_queue is not None
2510
+ await self._acquire_request_state_response_create_admission(
2511
+ warmup_state,
2512
+ response_create_gate=session.response_create_gate,
2513
+ account_id=session.account.id,
2514
+ surface="http_bridge_prewarm",
2515
+ bridge_session=session,
2516
+ dashboard_settings=dashboard_settings,
2517
+ )
2518
+ gate_acquired = True
2519
+ async with session.lifecycle_lock:
2520
+ current_session = session
2521
+ http_bridge_sessions = getattr(self, "_http_bridge_sessions", None)
2522
+ bridge_lock = getattr(self, "_http_bridge_lock", None)
2523
+ if bridge_lock is not None:
2524
+ async with bridge_lock:
2525
+ if http_bridge_sessions is not None:
2526
+ current_session = http_bridge_sessions.get(session.key)
2527
+ elif http_bridge_sessions is not None:
2528
+ current_session = http_bridge_sessions.get(session.key)
2529
+ session_replaced = current_session is not session
2530
+ if session.closed or session_replaced:
2531
+ _log_http_bridge_event(
2532
+ "submit_on_closed",
2533
+ session.key,
2534
+ account_id=session.account.id,
2535
+ model=session.request_model,
2536
+ detail=(
2537
+ "prewarm_session_retired_after_admission"
2538
+ if session.closed
2539
+ else "prewarm_session_replaced_after_admission"
2540
+ ),
2541
+ cache_key_family=session.key.affinity_kind,
2542
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
2543
+ )
2544
+ session.prewarmed = False
2545
+ await self._cleanup_http_bridge_submit_interruption(
2546
+ session,
2547
+ request_state=warmup_state,
2548
+ gate_acquired=gate_acquired,
2549
+ request_enqueued=request_enqueued,
2550
+ counted_in_queue=False,
2551
+ )
2552
+ gate_acquired = False
2553
+ return
2554
+ async with session.pending_lock:
2555
+ session.pending_requests.append(warmup_state)
2556
+ request_enqueued = True
2557
+ warmup_send_started = True
2558
+ await _send_http_bridge_request_text_with_archive_id(
2559
+ session, warmup_state, warmup_text, clock=clock
2560
+ )
2561
+ while True:
2562
+ try:
2563
+ event_block = await scheduler_for(self).wait_for(
2564
+ event_queue.get(),
2565
+ timeout=_prewarm_response_timeout_seconds(),
2566
+ )
2567
+ except asyncio.TimeoutError:
2568
+ request_state.prewarm_latency_ms = int(max(0.0, clock.monotonic() - prewarm_started_at) * 1000)
2569
+ request_state.prewarm_status = "timeout"
2570
+ _record_http_bridge_prewarm_outcome(outcome="timeout")
2571
+ logger.warning(
2572
+ "HTTP bridge prewarm timed out request_id=%s model=%s",
2573
+ request_state.request_id,
2574
+ request_state.model,
2575
+ )
2576
+ session.prewarmed = False
2577
+ try:
2578
+ # The warmup request has already been sent upstream. Close/reconnect the
2579
+ # socket while the warmup state is still attached so any late warmup
2580
+ # response cannot be assigned to the next visible request on this session.
2581
+ await self._reconnect_http_bridge_session(
2582
+ session,
2583
+ request_state=request_state,
2584
+ restart_reader=True,
2585
+ require_same_account=is_http_bridge_account_neutral_replay(
2586
+ kind=session.key.affinity_kind,
2587
+ key=session.key.affinity_key,
2588
+ ),
2589
+ dashboard_settings=dashboard_settings,
2590
+ )
2591
+ except Exception:
2592
+ session.closed = True
2593
+ raise
2594
+ finally:
2595
+ async with session.pending_lock:
2596
+ if warmup_state in session.pending_requests:
2597
+ session.pending_requests.remove(warmup_state)
2598
+ self._cancel_request_state_api_key_reservation_heartbeat(warmup_state)
2599
+ if gate_acquired:
2600
+ await _release_websocket_response_create_gate(
2601
+ warmup_state,
2602
+ session.response_create_gate,
2603
+ scheduler=scheduler_for(self),
2604
+ )
2605
+ return
2606
+ if event_block is None:
2607
+ break
2608
+ payload = parse_sse_data_json(event_block)
2609
+ event = parse_sse_event(event_block)
2610
+ event_type = _event_type_from_payload(event, payload)
2611
+ if event_type in {"response.failed", "response.incomplete", "error"}:
2612
+ raise ProxyResponseError(
2613
+ 502,
2614
+ openai_error(
2615
+ "upstream_unavailable",
2616
+ "HTTP responses session bridge prewarm failed",
2617
+ ),
2618
+ )
2619
+ session.last_used_at = clock.monotonic()
2620
+ request_state.prewarm_latency_ms = int(max(0.0, session.last_used_at - prewarm_started_at) * 1000)
2621
+ request_state.prewarm_status = "success"
2622
+ _record_http_bridge_prewarm_outcome(outcome="success")
2623
+ except ProxyResponseError as exc:
2624
+ error = _parse_openai_error(exc.payload)
2625
+ code = _normalize_error_code(error.code if error else None, error.type if error else None)
2626
+ await self._cleanup_http_bridge_submit_interruption(
2627
+ session,
2628
+ request_state=warmup_state,
2629
+ gate_acquired=gate_acquired,
2630
+ request_enqueued=request_enqueued,
2631
+ counted_in_queue=False,
2632
+ )
2633
+ if is_local_overload_error_code(code):
2634
+ session.prewarmed = False
2635
+ request_state.prewarm_latency_ms = int(max(0.0, clock.monotonic() - prewarm_started_at) * 1000)
2636
+ request_state.prewarm_status = "skipped"
2637
+ _record_http_bridge_prewarm_outcome(outcome="skipped")
2638
+ return
2639
+ session.prewarmed = False
2640
+ request_state.prewarm_latency_ms = int(max(0.0, clock.monotonic() - prewarm_started_at) * 1000)
2641
+ request_state.prewarm_status = "error"
2642
+ _record_http_bridge_prewarm_outcome(outcome="error")
2643
+ raise
2644
+ except BaseException:
2645
+ if warmup_send_started:
2646
+ session.closed = True
2647
+ session.upstream_control.reconnect_requested = True
2648
+ session.upstream_control.retire_after_drain = True
2649
+ session.prewarmed = False
2650
+ request_state.prewarm_latency_ms = int(max(0.0, clock.monotonic() - prewarm_started_at) * 1000)
2651
+ request_state.prewarm_status = "error"
2652
+ _record_http_bridge_prewarm_outcome(outcome="error")
2653
+ cleanup_task = scheduler_for(self).create_task(
2654
+ self._cleanup_http_bridge_submit_interruption(
2655
+ session,
2656
+ request_state=warmup_state,
2657
+ gate_acquired=gate_acquired,
2658
+ request_enqueued=request_enqueued,
2659
+ counted_in_queue=False,
2660
+ )
2661
+ )
2662
+ await _await_task_deferring_cancellation(cleanup_task)
2663
+ if (
2664
+ warmup_send_started
2665
+ and session.upstream_control.retire_after_drain
2666
+ and not session.upstream_close_attempted
2667
+ ):
2668
+ retire_task = scheduler_for(self).create_task(
2669
+ self._retire_http_bridge_after_drain_if_ready(session)
2670
+ )
2671
+ await _await_task_deferring_cancellation(retire_task)
2672
+ raise
2673
+
2674
+ async def _cleanup_http_bridge_submit_interruption(
2675
+ self: Any,
2676
+ session: "_HTTPBridgeSession",
2677
+ *,
2678
+ request_state: _WebSocketRequestState,
2679
+ gate_acquired: bool,
2680
+ request_enqueued: bool,
2681
+ counted_in_queue: bool,
2682
+ admission_waiter_registered: bool = False,
2683
+ ) -> None:
2684
+ retire_closed_session = False
2685
+ async with session.pending_lock:
2686
+ if request_enqueued and request_state in session.pending_requests:
2687
+ session.pending_requests.remove(request_state)
2688
+ if counted_in_queue:
2689
+ session.queued_request_count = max(0, session.queued_request_count - 1)
2690
+ if admission_waiter_registered or request_state.admission_waiter_preregistered:
2691
+ session.admission_waiter_count = max(0, session.admission_waiter_count - 1)
2692
+ request_state.admission_waiter_preregistered = False
2693
+ retire_closed_session = session.closed and session.admission_waiter_count == 0
2694
+ if (
2695
+ request_state.recovery_attempt_fingerprint is not None
2696
+ and not request_state.recovery_attempt_claimed
2697
+ and not request_state.recovery_attempt_dispatched
2698
+ and session.durable_session_id is not None
2699
+ and session.durable_owner_epoch is not None
2700
+ ):
2701
+ rollback_recovery_attempt = getattr(self._durable_bridge, "rollback_recovery_attempt_before_dispatch", None)
2702
+ if callable(rollback_recovery_attempt):
2703
+ try:
2704
+ await _call_with_supported_optional_kwargs(
2705
+ rollback_recovery_attempt,
2706
+ optional_kwargs={},
2707
+ session_id=session.durable_session_id,
2708
+ api_key_id=session.key.api_key_id,
2709
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
2710
+ owner_epoch=session.durable_owner_epoch,
2711
+ request_fingerprint=request_state.recovery_attempt_fingerprint,
2712
+ )
2713
+ except Exception:
2714
+ logger.warning(
2715
+ "Failed to roll back pre-dispatch HTTP bridge recovery checkpoint request_id=%s",
2716
+ request_state.request_id,
2717
+ exc_info=True,
2718
+ )
2719
+ if (
2720
+ (request_state.operation_created or request_state.operation_rebound)
2721
+ and request_state.operation_registered
2722
+ and request_state.operation_id is not None
2723
+ and not request_state.operation_dispatched
2724
+ and session.durable_session_id is not None
2725
+ and session.durable_owner_epoch is not None
2726
+ ):
2727
+ rollback_operation = getattr(self._durable_bridge, "rollback_operation_before_dispatch", None)
2728
+ if callable(rollback_operation):
2729
+ try:
2730
+ rolled_back = await _call_with_supported_optional_kwargs(
2731
+ rollback_operation,
2732
+ optional_kwargs={
2733
+ "restore_rebound": request_state.operation_rebound,
2734
+ "rebound_from_session_id": request_state.operation_rebound_from_session_id,
2735
+ "rebound_from_account_id": request_state.operation_rebound_from_account_id,
2736
+ "rebound_from_model": request_state.operation_rebound_from_model,
2737
+ "rebound_from_parent_response_id": request_state.operation_rebound_from_parent_response_id,
2738
+ },
2739
+ operation_id=request_state.operation_id,
2740
+ session_id=session.durable_session_id,
2741
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
2742
+ owner_epoch=session.durable_owner_epoch,
2743
+ )
2744
+ except Exception:
2745
+ rolled_back = False
2746
+ logger.warning(
2747
+ "Failed to roll back pre-dispatch HTTP bridge operation operation_id=%s",
2748
+ request_state.operation_id,
2749
+ exc_info=True,
2750
+ )
2751
+ if rolled_back:
2752
+ request_state.operation_registered = False
2753
+ request_state.operation_created = False
2754
+ if request_state.operation_rebound:
2755
+ # The failed durable row was restored rather than
2756
+ # deleted. Preserve its identity so a capacity/gate
2757
+ # retry must rebind that exact fence before send.
2758
+ request_state.operation_rebind_required = True
2759
+ request_state.operation_rebound = False
2760
+ else:
2761
+ request_state.operation_id = None
2762
+ request_state.operation_fingerprint = None
2763
+ request_state.operation_parent_response_id = None
2764
+ self._cancel_request_state_api_key_reservation_heartbeat(request_state)
2765
+ if request_state.response_create_gate is not None:
2766
+ if gate_acquired or request_state.response_create_gate_acquired:
2767
+ await _release_websocket_response_create_gate(
2768
+ request_state, session.response_create_gate, scheduler=scheduler_for(self)
2769
+ )
2770
+ else:
2771
+ account_response_create_lease = request_state.account_response_create_lease
2772
+ account_response_create_release = request_state.account_response_create_release
2773
+ request_state.account_response_create_lease = None
2774
+ request_state.account_response_create_release = None
2775
+ if account_response_create_lease is not None and account_response_create_release is not None:
2776
+ await account_response_create_release(account_response_create_lease)
2777
+ if request_state.response_create_admission is not None:
2778
+ request_state.response_create_admission.release()
2779
+ request_state.response_create_admission = None
2780
+ request_state.awaiting_response_created = False
2781
+ request_state.response_create_gate = None
2782
+ request_state.response_create_gate_acquired = False
2783
+ elif gate_acquired:
2784
+ await _release_websocket_response_create_gate(
2785
+ request_state, session.response_create_gate, scheduler=scheduler_for(self)
2786
+ )
2787
+ if retire_closed_session:
2788
+ await self._retire_stale_pending_http_bridge_session(
2789
+ session,
2790
+ detail="last_admission_waiter_cancelled",
2791
+ response_events_seen=max(
2792
+ request_state.response_event_count,
2793
+ int(request_state.response_id is not None or request_state.latency_response_created_ms is not None),
2794
+ ),
2795
+ # The session was already closed when the last admission
2796
+ # waiter cancelled; its reader is gone, so post-suspension
2797
+ # liveness signals cannot make it serviceable again.
2798
+ allow_liveness_revive=False,
2799
+ )
2800
+ await self._maybe_release_idle_http_bridge_session_lease(session)
2801
+
2802
+ async def _http_bridge_reacquire_snapshot(
2803
+ self: Any,
2804
+ session: "_HTTPBridgeSession",
2805
+ ) -> tuple[int, RoutingTunables | None]:
2806
+ """Resolve the reacquire's settings inputs WITHOUT holding pending_lock.
2807
+
2808
+ For a keyed session one cached dashboard row yields both the fair-share
2809
+ threshold and the routing tunables the lease is judged against (C2-2
2810
+ routing/overload: the lease TTL). An unkeyed session reads no settings
2811
+ here, as before: its threshold is ``0`` and the balancer falls back to
2812
+ the snapshot of its most recent request. The settings-cache refresh
2813
+ runs a DB query behind a process-global lock; awaiting it while holding
2814
+ a session's ``pending_lock`` let one stalled query wedge every keyed
2815
+ submit process-wide (issue #1971). Callers resolve the snapshot first
2816
+ and pass both values into ``_ensure_http_bridge_session_stream_lease_locked``.
2817
+ """
2818
+ if session.key.api_key_id is None:
2819
+ return 0, None
2820
+ dashboard_settings = await _service_get_settings_cache().get()
2821
+ return (
2822
+ _api_key_fair_share_threshold_pct_from_settings(dashboard_settings),
2823
+ effective_routing_tunables(dashboard_settings),
2824
+ )
2825
+
2826
+ async def _ensure_http_bridge_session_stream_lease_locked(
2827
+ self: Any,
2828
+ session: "_HTTPBridgeSession",
2829
+ *,
2830
+ request_state: _WebSocketRequestState | None = None,
2831
+ fair_share_threshold_pct: int | None = None,
2832
+ routing_tunables: RoutingTunables | None = None,
2833
+ ) -> None:
2834
+ """Reacquire the account stream lease for a session idled between turns.
2835
+
2836
+ Callers hold ``session.pending_lock`` and MUST pass both
2837
+ ``fair_share_threshold_pct`` and ``routing_tunables`` (see
2838
+ ``_http_bridge_reacquire_snapshot``) computed before acquiring it:
2839
+ resolving either reads the settings cache, whose refresh runs a DB
2840
+ query behind a process-global lock — one stalled refresh under
2841
+ ``pending_lock`` wedged every keyed submit for days (issue #1971).
2842
+ The ``None`` fallback resolves them inline from one cached row and
2843
+ exists for lock-free callers only.
2844
+
2845
+ The lease is released when the
2846
+ session's last in-flight turn detaches, so an idle session does not
2847
+ occupy a per-account stream slot; the next turn must pass normal cap
2848
+ admission again. Denial raises the standard local-cap envelope so the
2849
+ existing recoverable capacity wait applies.
2850
+
2851
+ The lease stays per-session (one upstream stream): a session that
2852
+ already holds a lease admits further queued turns without acquiring
2853
+ another, because those turns multiplex over the session's single
2854
+ upstream WebSocket — unchanged from the pre-existing per-session
2855
+ lease lifecycle.
2856
+
2857
+ Keyed sessions thread their API key through the reacquire so the
2858
+ turn joins the per-key stream accounting and passes the same
2859
+ congestion-gated fair-share admission as initial selection; a
2860
+ fair-share denial raises the standard local-cap envelope with the
2861
+ fair-share code so the recoverable capacity wait applies.
2862
+ """
2863
+ if session.account_lease is not None or session.closed:
2864
+ return
2865
+ load_balancer = getattr(self, "_load_balancer", None)
2866
+ if load_balancer is None:
2867
+ return
2868
+ api_key_id = session.key.api_key_id
2869
+ if api_key_id is None:
2870
+ fair_share_threshold_pct = 0
2871
+ elif fair_share_threshold_pct is None:
2872
+ # Lock-free callers only (see the docstring): one cached row serves
2873
+ # both inputs.
2874
+ dashboard_settings = await _service_get_settings_cache().get()
2875
+ fair_share_threshold_pct = _api_key_fair_share_threshold_pct_from_settings(dashboard_settings)
2876
+ if routing_tunables is None:
2877
+ routing_tunables = effective_routing_tunables(dashboard_settings)
2878
+ try:
2879
+ lease = await load_balancer.acquire_account_lease(
2880
+ session.account.id,
2881
+ kind="stream",
2882
+ routing_tunables=routing_tunables,
2883
+ # Carry the turn's usage-budget estimate like initial selection
2884
+ # and reconnect do, so capacity-weighted routing pressure still
2885
+ # sees large turns on reused warm sessions.
2886
+ estimated_tokens=_estimated_lease_tokens_from_request_usage_budget(
2887
+ request_state.request_usage_budget if request_state is not None else None
2888
+ ),
2889
+ api_key_id=api_key_id,
2890
+ api_key_stream_fair_share_threshold_pct=fair_share_threshold_pct,
2891
+ )
2892
+ except ApiKeyFairShareDenialError as denial:
2893
+ raise ProxyResponseError(
2894
+ 429,
2895
+ openai_error(
2896
+ API_KEY_STREAM_FAIR_SHARE_ERROR_CODE,
2897
+ str(denial),
2898
+ error_type="rate_limit_error",
2899
+ ),
2900
+ ) from None
2901
+ if lease is None:
2902
+ raise ProxyResponseError(
2903
+ 429,
2904
+ openai_error(
2905
+ "account_stream_cap",
2906
+ "Account stream capacity is exhausted; wait for active streams to finish.",
2907
+ error_type="rate_limit_error",
2908
+ ),
2909
+ )
2910
+ if session.closed:
2911
+ # A close or eviction ran while the acquire await was suspended
2912
+ # (the close path does not take pending_lock before settling the
2913
+ # session's lease). Installing this lease on the closed session
2914
+ # would leak the slot: close already settled, and the idle
2915
+ # release helper skips closed sessions. Return the slot and fail
2916
+ # the turn like any other submit on a closed bridge.
2917
+ async def release_detached_lease() -> None:
2918
+ try:
2919
+ await load_balancer.release_account_lease(lease)
2920
+ except Exception:
2921
+ logger.warning(
2922
+ "Failed to release stream lease acquired for closed HTTP bridge session",
2923
+ exc_info=True,
2924
+ )
2925
+
2926
+ release_task = scheduler_for(self).create_task(release_detached_lease())
2927
+ _, cancellation = await _await_task_deferring_cancellation(release_task)
2928
+ if cancellation is not None:
2929
+ raise cancellation
2930
+ raise ProxyResponseError(
2931
+ 502,
2932
+ openai_error("upstream_unavailable", "HTTP responses session bridge is closed"),
2933
+ )
2934
+ session.account_lease = lease
2935
+
2936
+ async def _maybe_release_idle_http_bridge_session_lease(
2937
+ self: Any,
2938
+ session: "_HTTPBridgeSession",
2939
+ ) -> None:
2940
+ """Release the account stream lease once a session has no in-flight work.
2941
+
2942
+ The per-account stream cap exists to bound concurrent upstream
2943
+ streams (one bridge session holds one slot for its single upstream
2944
+ WebSocket); an idle bridge session keeping its upstream WebSocket warm
2945
+ must not occupy a slot for its whole idle TTL. Session close releases
2946
+ via its own path, so a session that closed keeps that settlement.
2947
+ """
2948
+ load_balancer = getattr(self, "_load_balancer", None)
2949
+ if load_balancer is None:
2950
+ return
2951
+ lease = None
2952
+ async with session.pending_lock:
2953
+ if (
2954
+ not session.closed
2955
+ and session.account_lease is not None
2956
+ and session.queued_request_count == 0
2957
+ and session.admission_waiter_count == 0
2958
+ and not session.pending_requests
2959
+ ):
2960
+ lease = session.account_lease
2961
+ session.account_lease = None
2962
+ if lease is None:
2963
+ return
2964
+
2965
+ async def release_idle_lease() -> None:
2966
+ try:
2967
+ await load_balancer.release_account_lease(lease)
2968
+ except Exception:
2969
+ logger.warning("Failed to release idle HTTP bridge account lease", exc_info=True)
2970
+
2971
+ release_task = scheduler_for(self).create_task(release_idle_lease())
2972
+ _, cancellation = await _await_task_deferring_cancellation(release_task)
2973
+ if cancellation is not None:
2974
+ raise cancellation
2975
+
2976
+ async def _detach_http_bridge_request(
2977
+ self: Any,
2978
+ session: "_HTTPBridgeSession",
2979
+ *,
2980
+ request_state: _WebSocketRequestState,
2981
+ ) -> bool:
2982
+ detached = False
2983
+ async with session.pending_lock:
2984
+ if request_state in session.pending_requests and not request_state.draining_until_terminal:
2985
+ request_state.draining_until_terminal = True
2986
+ request_state.downstream_visible = False
2987
+ session.queued_request_count = max(0, session.queued_request_count - 1)
2988
+ session.upstream_control.reconnect_requested = True
2989
+ session.upstream_control.retire_after_drain = True
2990
+ detached = True
2991
+ # Queue revocation and pending ownership use the same lock. A
2992
+ # completed handler that wins first keeps its local queue reference;
2993
+ # a detach that wins first leaves no queue for that handler to claim.
2994
+ request_state.event_queue = None
2995
+ await _release_websocket_response_create_gate(
2996
+ request_state, session.response_create_gate, scheduler=scheduler_for(self)
2997
+ )
2998
+ if not detached:
2999
+ if request_state.operation_replay:
3000
+ # Replay requests are delivered from the durable transcript
3001
+ # without entering pending ownership, so the normal detach
3002
+ # branch cannot settle their API-key reservation.
3003
+ self._cancel_request_state_api_key_reservation_heartbeat(request_state)
3004
+ await self._release_websocket_request_state_reservation(request_state)
3005
+ request_state.api_key_reservation = None
3006
+ request_state.operation_replay = False
3007
+ return False
3008
+ if request_state.terminal_settlement_phase == "abandoned":
3009
+ # Belt-and-braces for issue #1594: terminal bookkeeping
3010
+ # claimed this request out of pending ownership, aborted, and
3011
+ # its shielded abort settlement also failed. Nothing else owns
3012
+ # the reservation any more, so reclaim settlement here instead
3013
+ # of keying solely on pending-deque membership.
3014
+ self._cancel_request_state_api_key_reservation_heartbeat(request_state)
3015
+ await self._release_websocket_request_state_reservation(request_state)
3016
+ request_state.api_key_reservation = None
3017
+ request_state.terminal_settlement_phase = None
3018
+ return False
3019
+ self._cancel_request_state_api_key_reservation_heartbeat(request_state)
3020
+ await self._release_websocket_request_state_reservation(request_state)
3021
+ request_state.api_key_reservation = None
3022
+ await self._retire_http_bridge_after_drain_if_ready(session)
3023
+ return True
3024
+
3025
+ async def _fail_stale_http_bridge_pending_requests(
3026
+ self: Any,
3027
+ session: "_HTTPBridgeSession",
3028
+ request_states: list[_WebSocketRequestState],
3029
+ *,
3030
+ detail: str,
3031
+ retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None,
3032
+ ) -> None:
3033
+ if retry_circuit_attempt_selection is None:
3034
+ # Capture the physical sends before waiting for pending ownership.
3035
+ # A concurrent recovery may replace request_state.response_create_attempt
3036
+ # while this task is suspended on pending_lock.
3037
+ retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
3038
+ request_states
3039
+ )
3040
+ stale_requests: deque[_WebSocketRequestState] = deque()
3041
+ response_events_seen = 0
3042
+ async with session.pending_lock:
3043
+ for request_state in request_states:
3044
+ if request_state not in session.pending_requests:
3045
+ continue
3046
+ response_events_seen = max(
3047
+ response_events_seen,
3048
+ request_state.response_event_count,
3049
+ int(
3050
+ request_state.response_id is not None
3051
+ or request_state.latency_response_created_ms is not None
3052
+ or request_state.downstream_visible
3053
+ ),
3054
+ )
3055
+ session.pending_requests.remove(request_state)
3056
+ if _http_bridge_request_counts_against_queue(request_state):
3057
+ session.queued_request_count = max(0, session.queued_request_count - 1)
3058
+ stale_requests.append(request_state)
3059
+ if not stale_requests:
3060
+ return
3061
+ # A stale gate holder that streamed response events without ever
3062
+ # receiving ``response.created`` proves the reattach wedge (#1534)
3063
+ # even when the session itself survives with other active requests.
3064
+ _record_http_bridge_quarantine_wedged_pending(self, session, stale_requests)
3065
+ partial_strike_detail = detail
3066
+ # ``_fail_pending_websocket_requests`` empties the deque it is handed
3067
+ # when it claims the states, so the settle predicate below must read
3068
+ # a frozen copy or a removed safe-replay holder silently vanishes
3069
+ # from the settlement decision.
3070
+ removed_request_states = list(stale_requests)
3071
+
3072
+ async def _finalize_and_settle_partial_cleanup() -> None:
3073
+ # The strike lands before the failure frames are published, so a
3074
+ # client that resends the moment it sees the end-of-stream
3075
+ # sentinel already meets the cooldown and quarantine this strike
3076
+ # armed. The durable abandonment stays after publication per the
3077
+ # ordering contract.
3078
+ consecutive_failures: int | None = None
3079
+ if response_events_seen == 0:
3080
+ consecutive_failures = await self._record_http_bridge_retry_circuit_failure_for_attempt_selection(
3081
+ session,
3082
+ detail=partial_strike_detail,
3083
+ selection=retry_circuit_attempt_selection,
3084
+ )
3085
+ await self._fail_pending_websocket_requests(
3086
+ account=session.account,
3087
+ account_id_value=session.account.id,
3088
+ pending_requests=stale_requests,
3089
+ pending_lock=session.pending_lock,
3090
+ error_code="upstream_request_timeout",
3091
+ error_message="HTTP bridge response-create gate holder timed out",
3092
+ api_key=None,
3093
+ response_create_gate=session.response_create_gate,
3094
+ penalize_account=False,
3095
+ )
3096
+ if response_events_seen == 0:
3097
+ poison_detail = await self._http_bridge_effective_anchor_poison_detail(session, partial_strike_detail)
3098
+ if poison_detail is None:
3099
+ return
3100
+ # The consult returns the exact episode it validated; the
3101
+ # marker below scopes to it.
3102
+ poison_episode, poison_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
3103
+ session,
3104
+ consecutive_failures=consecutive_failures,
3105
+ )
3106
+ if poison_episode is None:
3107
+ return
3108
+ # A surviving safe-replay holder is about to re-dispatch
3109
+ # and claim the circuit generation; settling under it would
3110
+ # strip that fence, so survivors count alongside the removed
3111
+ # holders. Snapshotted here, not before finalization: a
3112
+ # request that joined the still-live session while the
3113
+ # removed holders were being failed must count too.
3114
+ async with session.pending_lock:
3115
+ surviving_request_states = list(session.pending_requests)
3116
+ durable_cleared = await _abandon_durable_http_bridge_continuity(
3117
+ self,
3118
+ session,
3119
+ detail=poison_detail,
3120
+ settle_circuit=_http_bridge_abandonment_may_settle_circuit(
3121
+ [*removed_request_states, *surviving_request_states]
3122
+ ),
3123
+ expected_continuity=poison_expected_anchor,
3124
+ authorized_episode=poison_episode,
3125
+ )
3126
+ if durable_cleared:
3127
+ await self._http_bridge_mark_poison_anchor_cleared(session, episode=poison_episode)
3128
+ return
3129
+ if session.durable_session_id is not None:
3130
+ _log_http_bridge_event(
3131
+ "durable_anchor_poison_clear_failed",
3132
+ session.key,
3133
+ account_id=session.account.id,
3134
+ model=session.request_model,
3135
+ pending_count=len(removed_request_states),
3136
+ detail=poison_detail,
3137
+ cache_key_family=session.key.affinity_kind,
3138
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
3139
+ )
3140
+
3141
+ # One owned task covers finalization and settlement together: a
3142
+ # cancellation landing while the holders are being failed would
3143
+ # otherwise re-raise out of finalization before the settlement task
3144
+ # even exists, with the holders already removed from pending and no
3145
+ # lifecycle left to retry the strike, the consult, or the
3146
+ # poisoned-anchor abandonment. Defer cancellation across the whole
3147
+ # sequence like the other funnels, then re-raise it.
3148
+ partial_cleanup_task = scheduler_for(self).create_task(
3149
+ _finalize_and_settle_partial_cleanup(),
3150
+ name=f"http-bridge-partial-cleanup-poison-settlement-{session.durable_session_id}",
3151
+ )
3152
+ _partial_result, partial_cancellation = await _await_task_deferring_cancellation(partial_cleanup_task)
3153
+ if partial_cancellation is not None:
3154
+ raise partial_cancellation
3155
+
3156
+ def _classify_http_bridge_stale_gate_holders(
3157
+ self: Any,
3158
+ pending_states: list[_WebSocketRequestState],
3159
+ *,
3160
+ now: float,
3161
+ threshold_seconds: float,
3162
+ session_closed: bool,
3163
+ ) -> tuple[list[_WebSocketRequestState], bool]:
3164
+ stale_states = [
3165
+ state
3166
+ for state in pending_states
3167
+ if not state.draining_until_terminal
3168
+ and self._http_bridge_pending_state_is_stale(
3169
+ state,
3170
+ now=now,
3171
+ threshold_seconds=threshold_seconds,
3172
+ session_closed=session_closed,
3173
+ )
3174
+ ]
3175
+ active_states = [
3176
+ # A draining request still owns terminal response continuity. It
3177
+ # must keep the session alive while stale holders are cleaned up.
3178
+ state
3179
+ for state in pending_states
3180
+ if state not in stale_states
3181
+ ]
3182
+ if stale_states and active_states:
3183
+ return stale_states, False
3184
+ return [], bool(stale_states)
3185
+
3186
+ async def _snapshot_http_bridge_stale_gate_state(
3187
+ self: Any,
3188
+ session: "_HTTPBridgeSession",
3189
+ *,
3190
+ now: float,
3191
+ ) -> _HTTPBridgeStaleGateSnapshot:
3192
+ threshold_seconds = float(_http_bridge_helpers.HTTP_BRIDGE_STUCK_GATE_RETIRE_AFTER_SECONDS)
3193
+ async with session.pending_lock:
3194
+ pending_states = list(session.pending_requests)
3195
+ stale_request_states, should_retire = self._classify_http_bridge_stale_gate_holders(
3196
+ pending_states,
3197
+ now=now,
3198
+ threshold_seconds=threshold_seconds,
3199
+ session_closed=session.closed,
3200
+ )
3201
+ retry_circuit_request_states = (
3202
+ stale_request_states if stale_request_states else (pending_states if should_retire else ())
3203
+ )
3204
+ return _HTTPBridgeStaleGateSnapshot(
3205
+ pending_states=pending_states,
3206
+ queued_count=session.queued_request_count,
3207
+ threshold_seconds=threshold_seconds,
3208
+ stale_request_states=stale_request_states,
3209
+ should_retire=should_retire,
3210
+ retry_circuit_attempt_selection=(
3211
+ _http_bridge_retry_circuit_attempt_selection_for_pending_requests(retry_circuit_request_states)
3212
+ ),
3213
+ )
3214
+
3215
+ async def _release_http_bridge_admission_preregistration(
3216
+ self: Any,
3217
+ session: "_HTTPBridgeSession",
3218
+ *,
3219
+ request_state: _WebSocketRequestState,
3220
+ ) -> bool:
3221
+ """Hand back a submit-entry admission registration the dispatch never took over.
3222
+
3223
+ Mirrors the interruption cleanup's waiter release for a turn that
3224
+ never reached dispatch: the last waiter to leave a session that was
3225
+ closed under it retires the session when nothing else owns it (a
3226
+ fence-closed session with pending work and a live reader is settled
3227
+ by that reader, not here), and a session left with no in-flight work
3228
+ gives up an idle lease.
3229
+ """
3230
+ if not request_state.admission_waiter_preregistered:
3231
+ return False
3232
+ async with session.pending_lock:
3233
+ session.admission_waiter_count = max(0, session.admission_waiter_count - 1)
3234
+ request_state.admission_waiter_preregistered = False
3235
+ retire_closed_session = session.closed and _http_bridge_session_unowned_locked(session)
3236
+ if retire_closed_session:
3237
+ await self._retire_stale_pending_http_bridge_session(
3238
+ session,
3239
+ detail="last_admission_waiter_cancelled",
3240
+ response_events_seen=max(
3241
+ request_state.response_event_count,
3242
+ int(request_state.response_id is not None or request_state.latency_response_created_ms is not None),
3243
+ ),
3244
+ allow_liveness_revive=False,
3245
+ )
3246
+ await self._maybe_release_idle_http_bridge_session_lease(session)
3247
+ return True
3248
+
3249
+ async def _retire_http_bridge_after_drain_if_ready(
3250
+ self: Any,
3251
+ session: "_HTTPBridgeSession",
3252
+ *,
3253
+ lock_wait_timeout_seconds: float | None = None,
3254
+ ) -> bool:
3255
+ """Retire a drained session flagged ``retire_after_drain``.
3256
+
3257
+ ``lock_wait_timeout_seconds`` bounds the ``pending_lock`` wait for
3258
+ callers on a shared hot path (the per-request fail-safe sweep over
3259
+ detached sessions). When the bound elapses the check is skipped for
3260
+ this pass and ``False`` is returned; the session stays tracked and is
3261
+ reconsidered by the next sweep or by its own drain/close paths.
3262
+ ``None`` keeps the unbounded wait for owners of the session lifecycle.
3263
+ """
3264
+ if not (session.upstream_control.reconnect_requested and session.upstream_control.retire_after_drain):
3265
+ return False
3266
+
3267
+ def decide_locked() -> bool:
3268
+ unowned = _http_bridge_session_unowned_locked(session)
3269
+ if unowned:
3270
+ session.pending_requests.clear()
3271
+ session.upstream_close_attempted = True
3272
+ return unowned
3273
+
3274
+ if lock_wait_timeout_seconds is None:
3275
+ async with session.pending_lock:
3276
+ should_reconnect = decide_locked()
3277
+ else:
3278
+ try:
3279
+ await scheduler_for(self).wait_for(session.pending_lock.acquire(), timeout=lock_wait_timeout_seconds)
3280
+ except TimeoutError:
3281
+ logger.warning(
3282
+ "Skipping detached HTTP bridge retire check: pending_lock busy for %.1fs session_key=%s",
3283
+ lock_wait_timeout_seconds,
3284
+ _hash_identifier(session.key.affinity_key),
3285
+ )
3286
+ return False
3287
+ try:
3288
+ should_reconnect = decide_locked()
3289
+ finally:
3290
+ session.pending_lock.release()
3291
+ if not should_reconnect:
3292
+ return False
3293
+
3294
+ await self._close_http_bridge_session_bounded(session, reason="retire_after_drain")
3295
+ return True
3296
+
3297
+ async def _retire_idle_http_bridge_session_on_cooldown_suppression(
3298
+ self: Any,
3299
+ session: "_HTTPBridgeSession",
3300
+ *,
3301
+ owned_unanchored_handoff: bool,
3302
+ ) -> bool:
3303
+ """Retire a cooldown-suppressed session unless another turn owns it.
3304
+
3305
+ The bridge opens or selects the session before the hard-key retry
3306
+ circuit decides admission, so a suppressed request can leave a socket
3307
+ that never carried ``response.create`` in the reuse pool, where the
3308
+ next half-open probe picks it up (issue #1943). Flagging the session
3309
+ for retirement and running the bounded drain path closes that socket
3310
+ — but only when no other turn owns the session. A pending or queued
3311
+ turn, a registered admission waiter (the admitted probe between its
3312
+ gate and dispatch), or a foreign unanchored handoff is a live owner
3313
+ whose own lifecycle governs retirement; flagging the session under it
3314
+ would trip the pre-dispatch retiring fence for the one submission the
3315
+ cooldown permits. The caller's own unanchored handoff does not count:
3316
+ its submit finalizer releases it and retires the flagged session.
3317
+ """
3318
+ async with session.pending_lock:
3319
+ idle = _http_bridge_session_unowned_locked(
3320
+ session,
3321
+ ignore_unanchored_reservation=owned_unanchored_handoff,
3322
+ )
3323
+ if idle:
3324
+ session.upstream_control.reconnect_requested = True
3325
+ session.upstream_control.retire_after_drain = True
3326
+ if not idle:
3327
+ return False
3328
+ await self._retire_http_bridge_after_drain_if_ready(session)
3329
+ return True
3330
+
3331
+ async def _http_bridge_claim_miss_shows_remote_probe(
3332
+ self: Any,
3333
+ circuit_key: Any,
3334
+ captured_generation: Any,
3335
+ ) -> bool:
3336
+ """Whether a claim CAS miss actually means a probe holds the lease.
3337
+
3338
+ A confirmed CAS loss only says the row moved: a sibling completion
3339
+ resets the row and changes its version without touching the
3340
+ admission generation. Only an advanced admission generation proves a
3341
+ probe won; a reset, purge, strike, or lookup outage must report the
3342
+ timer the fresh row actually carries instead of a 600-second wait no
3343
+ probe owns.
3344
+ """
3345
+ captured_admission_generation = captured_generation[0] if captured_generation else 0
3346
+ captured_lineage_epoch = captured_generation[1] if captured_generation else 0.0
3347
+ try:
3348
+ moved_row = await self._durable_bridge.lookup_retry_circuit(
3349
+ session_key_kind=circuit_key.affinity_kind,
3350
+ session_key_value=circuit_key.affinity_key,
3351
+ api_key_id=circuit_key.api_key_id,
3352
+ )
3353
+ except Exception:
3354
+ return False
3355
+ return bool(
3356
+ moved_row is not None
3357
+ # A lease exists only for an active at-threshold lineage: a
3358
+ # probe-then-reset sequence keeps the advanced generation on a
3359
+ # zero-count row with no timer left, and a purge-and-recreate
3360
+ # restarts generations below the captured one.
3361
+ and moved_row.consecutive_failures >= _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD
3362
+ and getattr(moved_row, "admission_generation", 0) > captured_admission_generation
3363
+ # A reset preserves the admission generation while starting a
3364
+ # new lineage under a new version; only an advance within the
3365
+ # captured lineage is a probe in THIS episode. A same-lineage
3366
+ # strike moves the version too and under-reports here, which
3367
+ # falls back to the timer the fresh row actually carries.
3368
+ and getattr(moved_row, "updated_at_epoch", None) == captured_lineage_epoch
3369
+ )
3370
+
3371
+ async def _retire_stale_pending_http_bridge_session(
3372
+ self: Any,
3373
+ session: "_HTTPBridgeSession",
3374
+ *,
3375
+ detail: str,
3376
+ retry_circuit_detail: str | None = None,
3377
+ response_events_seen: int | None = None,
3378
+ retired_request_count: int | None = None,
3379
+ retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None,
3380
+ retired_request_states: list[_WebSocketRequestState] | None = None,
3381
+ allow_liveness_revive: bool = True,
3382
+ ) -> None:
3383
+ async with session.pending_lock:
3384
+ if retired_request_states is None:
3385
+ retired_request_states = list(session.pending_requests)
3386
+ baseline_completed_response_id = session.last_completed_response_id
3387
+ # Baseline the upstream event generation at entry, alongside the
3388
+ # completed-response baseline, so a prelude-only upstream event
3389
+ # arriving during the retry-circuit strike await (which bumps the
3390
+ # generation without touching per-request response counters) is
3391
+ # still detected as post-suspension liveness.
3392
+ baseline_event_generation = session.last_upstream_event_generation
3393
+ # Snapshot the retirement fences at entry. Fence owners (durable
3394
+ # stale-owner rejection, never-graft replacement rejection,
3395
+ # previous-response-owner unavailability, alias-registration
3396
+ # failure) set ``closed``/``reconnect_requested``/
3397
+ # ``retire_after_drain`` without detaching the session or claiming
3398
+ # the upstream close, so identity and close-claim guards alone
3399
+ # cannot see them. A fence raised after this snapshot must survive
3400
+ # the liveness revive below; erasing it would turn the fence
3401
+ # owner's drain-retirement into a permanent no-op and let a
3402
+ # condemned socket stay registered and reusable.
3403
+ entry_closed = session.closed
3404
+ entry_reconnect_requested = session.upstream_control.reconnect_requested
3405
+ entry_retire_after_drain = session.upstream_control.retire_after_drain
3406
+ if retired_request_count is None:
3407
+ retired_request_count = sum(
3408
+ 1
3409
+ for request_state in retired_request_states
3410
+ if _http_bridge_request_counts_against_queue(request_state)
3411
+ )
3412
+ if response_events_seen is None:
3413
+ # Direct retirement must derive event evidence from the same
3414
+ # locked ownership snapshot as the pending count. Otherwise an
3415
+ # eventful stale-gate owner looks eventless merely because its
3416
+ # caller omitted this optional handoff, creating a false
3417
+ # circuit strike. Explicit values remain authoritative for
3418
+ # reader-failure callers whose pending deque was already
3419
+ # drained before entering this shared boundary.
3420
+ response_events_seen = max(
3421
+ (
3422
+ max(
3423
+ request_state.response_event_count,
3424
+ int(
3425
+ request_state.response_id is not None
3426
+ or request_state.latency_response_created_ms is not None
3427
+ or request_state.downstream_visible
3428
+ ),
3429
+ )
3430
+ for request_state in retired_request_states
3431
+ ),
3432
+ default=0,
3433
+ )
3434
+ if retry_circuit_attempt_selection is None:
3435
+ retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
3436
+ retired_request_states
3437
+ )
3438
+ # Direct retirement (for example the all-stale stuck-gate path, where
3439
+ # the wedged reattach is the only pending request) cancels the reader
3440
+ # and fails the pendings without passing the partial-cleanup hook or
3441
+ # the reader-failure funnel, so evaluate the wedge shape (#1534) here
3442
+ # too; recording is idempotent for callers that already quarantined.
3443
+ _record_http_bridge_quarantine_wedged_pending(self, session, retired_request_states)
3444
+ # This circuit measures failed request lifecycles, not upstream socket
3445
+ # churn. ``response_events_seen == 0`` is also true when an idle reader
3446
+ # closes with an empty pending deque. Charging that idle close creates a
3447
+ # phantom first strike, so one later response-create timeout opens the
3448
+ # nominally "repeated" 60-second cooldown and interrupts the client.
3449
+ # Keep the ownership proof at this shared retirement boundary unless a
3450
+ # caller already claimed and drained the deque. The reader-failure
3451
+ # funnel must pass its pre-drain count because terminal notification
3452
+ # deliberately empties ``pending_requests`` before retirement. Without
3453
+ # that handoff, genuine pre-response failures disappear from circuit
3454
+ # accounting while idle closes and request failures look identical.
3455
+ # Mirror the terminal and grouped settlement paths: a retired request
3456
+ # that still holds a verified safe replay is about to be re-dispatched
3457
+ # and will claim the circuit generation, so it must not strike the
3458
+ # circuit or trigger the poison clear. Only the safe-replay half of
3459
+ # the settlement predicate applies here — an unanchored eventless
3460
+ # owner has no replay to protect and keeps striking. A pre-drain
3461
+ # handoff supplies the drained states through
3462
+ # ``retired_request_states``, so a safe-replay holder that was
3463
+ # already claimed by finalization still counts; only a handoff with
3464
+ # genuinely no request states keeps striking.
3465
+ retired_states_present = [state for state in retired_request_states if state is not None]
3466
+ retirement_strike_eligible = not retired_states_present or any(
3467
+ not _http_bridge_request_state_holds_safe_replay(state) for state in retired_states_present
3468
+ )
3469
+ if retired_request_count > 0 and response_events_seen == 0 and retirement_strike_eligible:
3470
+ consecutive_failures = await self._record_http_bridge_retry_circuit_failure_for_attempt_selection(
3471
+ session,
3472
+ detail=retry_circuit_detail or detail,
3473
+ selection=retry_circuit_attempt_selection,
3474
+ )
3475
+ poison_detail = await self._http_bridge_effective_anchor_poison_detail(
3476
+ session, retry_circuit_detail or detail
3477
+ )
3478
+ poison_episode = None
3479
+ poison_expected_anchor: object = _POISON_ANCHOR_CAPTURE_UNAVAILABLE
3480
+ if poison_detail is not None:
3481
+ # The consult returns the exact episode it validated; the
3482
+ # marker below scopes to it, so a settle landing between the
3483
+ # consult and the rebind cannot be marked onto a replacement.
3484
+ poison_episode, poison_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
3485
+ session,
3486
+ consecutive_failures=consecutive_failures,
3487
+ )
3488
+ if poison_detail is not None and poison_episode is not None:
3489
+ # Consecutive eventless failures on one bridge key are
3490
+ # same-anchor failures (the anchor only advances on a
3491
+ # completed response, which resets the circuit). Clear the
3492
+ # poisoned durable anchor while this session still owns the
3493
+ # lease so the next attempt is not re-anchored into the same
3494
+ # failure. Without this, only the admission-waiter reader
3495
+ # path could ever poison an anchor, and an anchored session
3496
+ # failing without waiters cooled down forever (issue #1830).
3497
+ durable_cleared = await _abandon_durable_http_bridge_continuity(
3498
+ self,
3499
+ session,
3500
+ detail=poison_detail,
3501
+ settle_circuit=_http_bridge_abandonment_may_settle_circuit(retired_request_states),
3502
+ expected_continuity=poison_expected_anchor,
3503
+ authorized_episode=poison_episode,
3504
+ )
3505
+ if durable_cleared:
3506
+ await self._http_bridge_mark_poison_anchor_cleared(session, episode=poison_episode)
3507
+ if not durable_cleared and session.durable_session_id is not None:
3508
+ # Keep failed waiterless clears visible in the same
3509
+ # poison-clear telemetry the admission-waiter path emits;
3510
+ # the next threshold failure re-attempts the clear.
3511
+ _log_http_bridge_event(
3512
+ "durable_anchor_poison_clear_failed",
3513
+ session.key,
3514
+ account_id=session.account.id,
3515
+ model=session.request_model,
3516
+ pending_count=retired_request_count,
3517
+ detail=poison_detail,
3518
+ cache_key_family=session.key.affinity_kind,
3519
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
3520
+ )
3521
+
3522
+ async with session.pending_lock:
3523
+ current_response_events_seen = max(
3524
+ (getattr(request_state, "response_event_count", 0) for request_state in session.pending_requests),
3525
+ default=0,
3526
+ )
3527
+ current_response_created = any(
3528
+ request_state.response_id is not None or request_state.latency_response_created_ms is not None
3529
+ for request_state in session.pending_requests
3530
+ )
3531
+ completed_response_id = session.last_completed_response_id
3532
+ # The snapshot above avoids awaiting pending_lock while the global
3533
+ # registry lock is held, preserving bounded cleanup for other sessions.
3534
+ caller_response_events_seen = response_events_seen or 0
3535
+ observed_new_completed_response = (
3536
+ completed_response_id is not None and completed_response_id != baseline_completed_response_id
3537
+ )
3538
+ became_healthy_during_suspend = current_response_events_seen > caller_response_events_seen or (
3539
+ caller_response_events_seen == 0 and (current_response_created or observed_new_completed_response)
3540
+ )
3541
+ should_close = False
3542
+ async with self._http_bridge_lock:
3543
+ if session.last_upstream_event_generation != baseline_event_generation:
3544
+ became_healthy_during_suspend = True
3545
+ # A fence raised while this coroutine was suspended condemns the
3546
+ # session even if it also showed liveness: the fence owner leaves
3547
+ # the session registered and never claims the upstream close, so
3548
+ # neither guard below can see it. Only fences newly raised since
3549
+ # the entry snapshot block the revive; the entry-time retirement
3550
+ # state belongs to this retirement and is cleared on revive.
3551
+ fence_raised_during_suspend = (
3552
+ (session.closed and not entry_closed)
3553
+ or (session.upstream_control.reconnect_requested and not entry_reconnect_requested)
3554
+ or (session.upstream_control.retire_after_drain and not entry_retire_after_drain)
3555
+ )
3556
+ # Bounded close may return while resource finalization is still
3557
+ # running. Detachment transfers ownership instead of freeing the
3558
+ # capacity slot at canonical removal, and leaves a failed close
3559
+ # discoverable by shutdown/account invalidation for a later retry.
3560
+ if session.upstream_close_attempted:
3561
+ # A close owner already claimed retirement; converge with its
3562
+ # bookkeeping and fall through so this path still emits the
3563
+ # shared retirement telemetry (``should_close`` resolves to
3564
+ # False below, so the close is not re-attempted).
3565
+ session.closed = True
3566
+ self._detach_http_bridge_session_locked(session.key, expected_session=session)
3567
+ elif (
3568
+ allow_liveness_revive
3569
+ and became_healthy_during_suspend
3570
+ and not fence_raised_during_suspend
3571
+ and self._http_bridge_sessions.get(session.key) is session
3572
+ ):
3573
+ # The pending snapshot is only advisory; a terminal response
3574
+ # may already have left the deque. A close owner cannot have
3575
+ # claimed retirement in between: the branch above falls
3576
+ # through under this same lock hold without awaiting.
3577
+ #
3578
+ # Revive only while this session is still the registered owner
3579
+ # of its key. The acquisition loop can detach a
3580
+ # ``retiring_with_visible_requests`` generation with
3581
+ # ``mark_closed=False`` while this coroutine is suspended;
3582
+ # clearing the retirement flags on that detached generation
3583
+ # would make its drain-retirement a permanent no-op and leak
3584
+ # the socket, durable/account leases, and capacity slot. A
3585
+ # detached generation falls through to the bounded close
3586
+ # below instead (the detach call is a no-op for it), as does
3587
+ # a session fenced during the suspension: the bounded close
3588
+ # conservatively satisfies the fence owner's intent.
3589
+ #
3590
+ # ``allow_liveness_revive=False`` callers (the reader-failure
3591
+ # funnel and deferred retirement of an already-closed session)
3592
+ # never reach this branch. Their pending turns were already
3593
+ # terminally failed and their reader is condemned, so there is
3594
+ # no turn left for the revive to save — and the
3595
+ # completed-response signal above can be advanced by durable
3596
+ # anchor rehydration, which copies registry state without any
3597
+ # upstream evidence. Reviving there would leave a condemned,
3598
+ # readerless session registered and reusable.
3599
+ session.closed = False
3600
+ session.upstream_control.reconnect_requested = False
3601
+ session.upstream_control.retire_after_drain = False
3602
+ return
3603
+ else:
3604
+ self._detach_http_bridge_session_locked(session.key, expected_session=session)
3605
+ async with session.pending_lock:
3606
+ should_close = not session.upstream_close_attempted
3607
+ if should_close:
3608
+ session.upstream_close_attempted = True
3609
+ if should_close:
3610
+ await self._close_http_bridge_session_bounded(session, reason="retire_stale_pending")
3611
+ _log_http_bridge_event(
3612
+ "retire_stale_pending",
3613
+ session.key,
3614
+ account_id=session.account.id,
3615
+ model=session.request_model,
3616
+ pending_count=await self._http_bridge_pending_count(session),
3617
+ detail=detail,
3618
+ cache_key_family=session.key.affinity_kind,
3619
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
3620
+ )
3621
+
3622
+ async def _retry_http_bridge_request_on_fresh_upstream(
3623
+ self: Any,
3624
+ session: "_HTTPBridgeSession",
3625
+ *,
3626
+ request_state: _WebSocketRequestState,
3627
+ text_data: str,
3628
+ send_request: bool = True,
3629
+ require_same_account: bool = False,
3630
+ ) -> bool:
3631
+ clock = clock_for(self)
3632
+ require_same_account = require_same_account or is_http_bridge_account_neutral_replay(
3633
+ kind=session.key.affinity_kind,
3634
+ key=session.key.affinity_key,
3635
+ )
3636
+ retry_text_data = text_data
3637
+ using_fresh_replay = False
3638
+ if request_state.previous_response_id is not None and send_request:
3639
+ # After an ambiguous websocket send failure we cannot prove whether
3640
+ # upstream already accepted the continuation. Re-sending the same
3641
+ # previous_response_id request can fork continuity with duplicate
3642
+ # child responses, so only reconnect-without-resend is allowed.
3643
+ # The single exception is a proof-gated, trim-safe full-resend
3644
+ # payload: dropping the anchor and replaying the original
3645
+ # unanchored request is equivalent to the client's own retry.
3646
+ # The proof is independent of where the anchor came from; a
3647
+ # client-provided full resend is as safe as a durable injection.
3648
+ # Session-level follow-ups do not opt in because their context may
3649
+ # depend on the anchor.
3650
+ if not request_state.fresh_upstream_request_text or not request_state.fresh_upstream_request_is_retry_safe:
3651
+ return False
3652
+ retry_text_data = request_state.fresh_upstream_request_text
3653
+ using_fresh_replay = True
3654
+ if request_state.replay_count >= 1:
3655
+ return False
3656
+ if request_state.response_event_count > 0:
3657
+ return False
3658
+ request_state.replay_count += 1
3659
+ _log_http_bridge_event(
3660
+ "retry_fresh_upstream",
3661
+ session.key,
3662
+ account_id=session.account.id,
3663
+ model=session.request_model,
3664
+ pending_count=1,
3665
+ cache_key_family=session.key.affinity_kind,
3666
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
3667
+ )
3668
+ try:
3669
+ await self._reconnect_http_bridge_session(
3670
+ session,
3671
+ request_state=request_state,
3672
+ restart_reader=True,
3673
+ require_same_account=require_same_account,
3674
+ require_preferred_account=request_state.file_required_preferred_account,
3675
+ )
3676
+ if send_request:
3677
+ retry_text_data = self._http_bridge_text_with_account_installation_id(
3678
+ session,
3679
+ request_state,
3680
+ retry_text_data,
3681
+ )
3682
+ if using_fresh_replay:
3683
+ _bind_http_bridge_proxy_injected_anchor(
3684
+ self,
3685
+ request_state,
3686
+ response_id=None,
3687
+ proxy_injected=False,
3688
+ )
3689
+ request_state.request_text = retry_text_data
3690
+ await _send_http_bridge_request_text_with_archive_id(
3691
+ session, request_state, retry_text_data, clock=clock
3692
+ )
3693
+ _clear_websocket_request_error_overrides(request_state)
3694
+ session.last_used_at = clock.monotonic()
3695
+ return True
3696
+ except UpstreamWebSocketTransportError:
3697
+ # The new socket may have accepted response.create. Let the reader
3698
+ # owner retire the whole session with the typed, non-replayable
3699
+ # failure instead of falling back to the earlier close reason.
3700
+ raise
3701
+ except ProxyResponseError as exc:
3702
+ if _http_bridge_is_previous_response_owner_unavailable(exc):
3703
+ raise
3704
+ logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True)
3705
+ return False
3706
+ except Exception:
3707
+ logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True)
3708
+ return False
3709
+
3710
+ async def _retry_http_bridge_precreated_request(
3711
+ self: Any,
3712
+ session: "_HTTPBridgeSession",
3713
+ *,
3714
+ request_state: _WebSocketRequestState | None = None,
3715
+ restart_reader: bool = False,
3716
+ ) -> bool:
3717
+ # The admitted body's admission gate can claim the half-open probe;
3718
+ # any exit that never advances a send attempt past its baseline must
3719
+ # hand that probe back, or a failed reconnect leaves the lease
3720
+ # suppressing traffic for up to the full window with no probe in
3721
+ # flight — the same handback contract the submit finalizer applies.
3722
+ admission_claimed_leases: list[float] = []
3723
+ retry_send_baselines: list[tuple[_WebSocketRequestState, int]] = []
3724
+ try:
3725
+ return await self._retry_http_bridge_precreated_request_admitted(
3726
+ session,
3727
+ request_state=request_state,
3728
+ restart_reader=restart_reader,
3729
+ admission_claimed_leases=admission_claimed_leases,
3730
+ retry_send_baselines=retry_send_baselines,
3731
+ )
3732
+ finally:
3733
+ retry_claimed_half_open_until = admission_claimed_leases[-1] if admission_claimed_leases else 0.0
3734
+ if retry_claimed_half_open_until > 0.0:
3735
+ send_attempt_advanced = bool(retry_send_baselines) and (
3736
+ retry_send_baselines[-1][0].response_create_attempt_count > retry_send_baselines[-1][1]
3737
+ )
3738
+ if not send_attempt_advanced:
3739
+ async with self._http_bridge_retry_circuit_lock:
3740
+ unused_probe_state = self._http_bridge_retry_circuits.get(session.key)
3741
+ if (
3742
+ unused_probe_state is not None
3743
+ and unused_probe_state.half_open_until == retry_claimed_half_open_until
3744
+ ):
3745
+ unused_probe_state.half_open_until = 0.0
3746
+ unused_probe_state.cooldown_until = clock_for(self).monotonic() - 1.0
3747
+
3748
+ async def _retry_http_bridge_precreated_request_admitted(
3749
+ self: Any,
3750
+ session: "_HTTPBridgeSession",
3751
+ *,
3752
+ request_state: _WebSocketRequestState | None = None,
3753
+ restart_reader: bool = False,
3754
+ admission_claimed_leases: list[float] | None = None,
3755
+ retry_send_baselines: list[tuple[_WebSocketRequestState, int]] | None = None,
3756
+ ) -> bool:
3757
+ clock = clock_for(self)
3758
+ clean_close_retry_max_count = self._http_bridge_clean_close_retry_max_count()
3759
+ account_neutral_recovery = is_http_bridge_account_neutral_replay(
3760
+ kind=session.key.affinity_kind,
3761
+ key=session.key.affinity_key,
3762
+ )
3763
+
3764
+ def request_is_retryable(request_state: _WebSocketRequestState) -> bool:
3765
+ transport_only_unanchored_replay = bool(
3766
+ request_state.precreated_replay_reason is None
3767
+ and (
3768
+ request_state.verified_stale_anchor_replay
3769
+ or (
3770
+ request_state.previous_response_id is not None
3771
+ and not request_state.proxy_injected_previous_response_id
3772
+ and request_state.fresh_upstream_request_is_retry_safe
3773
+ and request_state.fresh_upstream_request_text
3774
+ )
3775
+ )
3776
+ )
3777
+ if transport_only_unanchored_replay:
3778
+ return False
3779
+ if _websocket_request_can_replay_before_visible_output(request_state):
3780
+ return True
3781
+ if (
3782
+ clean_close_retry_max_count <= 0
3783
+ or request_state.verified_stale_anchor_replay
3784
+ or request_state.replay_count != 1
3785
+ or request_state.response_event_count != 0
3786
+ or request_state.clean_close_replay_count >= clean_close_retry_max_count
3787
+ or _classify_upstream_close(
3788
+ session.last_upstream_close_code,
3789
+ response_events_seen=request_state.response_event_count,
3790
+ )
3791
+ != "clean"
3792
+ ):
3793
+ return False
3794
+ return _websocket_request_can_replay_before_visible_output(
3795
+ request_state,
3796
+ allow_clean_close_retry=True,
3797
+ )
3798
+
3799
+ fresh_hard_request_account_switch_candidate = False
3800
+ proof_gated_continuity_replay_candidate = False
3801
+ if session.key.strength == "hard":
3802
+ async with session.pending_lock:
3803
+ retryable_candidates = [
3804
+ request_state
3805
+ for request_state in session.pending_requests
3806
+ if not request_state.draining_until_terminal and request_is_retryable(request_state)
3807
+ ]
3808
+ if len(retryable_candidates) == 1:
3809
+ candidate = retryable_candidates[0]
3810
+ fresh_hard_request_account_switch_candidate = (
3811
+ candidate.previous_response_id is None
3812
+ and not candidate.hard_continuity_anchor
3813
+ and not candidate.proxy_injected_previous_response_id
3814
+ and not candidate.file_required_preferred_account
3815
+ and candidate.response_event_count == 0
3816
+ and candidate.replay_count == 0
3817
+ )
3818
+ proof_gated_continuity_replay_candidate = (
3819
+ candidate.previous_response_id is not None
3820
+ and candidate.fresh_upstream_request_is_retry_safe
3821
+ and bool(candidate.fresh_upstream_request_text)
3822
+ and candidate.response_event_count == 0
3823
+ and candidate.replay_count == 0
3824
+ )
3825
+ if not await self._http_bridge_precreated_retry_allowed(
3826
+ session,
3827
+ allow_fresh_hard_account_switch=fresh_hard_request_account_switch_candidate,
3828
+ allow_proof_gated_continuity_replay=proof_gated_continuity_replay_candidate,
3829
+ claimed_lease_out=admission_claimed_leases,
3830
+ ):
3831
+ return False
3832
+
3833
+ account_neutral_recovery = is_http_bridge_account_neutral_replay(
3834
+ kind=session.key.affinity_kind,
3835
+ key=session.key.affinity_key,
3836
+ )
3837
+ hard_owner_bound = _http_bridge_key_strength(session.key) == "hard"
3838
+ async with session.pending_lock:
3839
+ if request_state is not None:
3840
+ if (
3841
+ request_state not in session.pending_requests
3842
+ or any(pending_request is not request_state for pending_request in session.pending_requests)
3843
+ or request_state.draining_until_terminal
3844
+ or not _http_bridge_request_counts_against_queue(request_state)
3845
+ or not request_is_retryable(request_state)
3846
+ ):
3847
+ return False
3848
+ else:
3849
+ retryable_requests = [
3850
+ request_state
3851
+ for request_state in session.pending_requests
3852
+ if not request_state.draining_until_terminal and request_is_retryable(request_state)
3853
+ ]
3854
+ if len(retryable_requests) != 1:
3855
+ return False
3856
+ request_state = retryable_requests[0]
3857
+ # An accepted lifecycle is replayed only when it is the sole
3858
+ # request on the socket (the terminal path's other-pending
3859
+ # guard, ``_websocket_accepted_replay_candidate``). Reconnecting
3860
+ # it would strand every sibling still bound to the dead
3861
+ # upstream, so both fail closed with ``stream_incomplete`` as
3862
+ # they did before accepted replays existed.
3863
+ if (
3864
+ request_state.response_id is not None
3865
+ and not request_state.awaiting_response_created
3866
+ and any(pending_request is not request_state for pending_request in session.pending_requests)
3867
+ ):
3868
+ return False
3869
+ # A request that already saw response.created released the session
3870
+ # create gate. Its replay must hold the gate again before it can
3871
+ # own the pre-created identity, and never waits for a contended one.
3872
+ if not await _claim_websocket_replay_create_gate(request_state, session.response_create_gate):
3873
+ return False
3874
+ if retry_send_baselines is not None:
3875
+ # The send-attempt baseline: a retried request already carries
3876
+ # prior attempts, so the release keys on advancement past
3877
+ # this point, not on a zero count.
3878
+ retry_send_baselines.append((request_state, request_state.response_create_attempt_count))
3879
+ model_fallback_replay = request_state.precreated_replay_reason == _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
3880
+ if request_state.previous_response_id is not None and not (
3881
+ request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text
3882
+ ):
3883
+ # Once a continuation is pending upstream, reconnecting without
3884
+ # replay cannot complete the current request, while replaying it
3885
+ # is unsafe without upstream idempotency guarantees. Proxy-
3886
+ # injected anchors and proof-gated client full resends are
3887
+ # equivalent to the client's own retry once the anchor is
3888
+ # stripped. The latter remains pinned to the current owner.
3889
+ return False
3890
+ close_classification = _classify_upstream_close(
3891
+ session.last_upstream_close_code,
3892
+ response_events_seen=request_state.response_event_count,
3893
+ )
3894
+ close_generation = session.last_upstream_close_generation
3895
+ hard_session_affinity = session.key.strength == "hard"
3896
+ fresh_hard_request_account_switch_allowed = (
3897
+ hard_session_affinity
3898
+ and request_state.previous_response_id is None
3899
+ and not request_state.hard_continuity_anchor
3900
+ and not request_state.proxy_injected_previous_response_id
3901
+ and not request_state.file_required_preferred_account
3902
+ )
3903
+ clean_close_hard_continuation = (
3904
+ close_classification == "clean"
3905
+ and hard_session_affinity
3906
+ and request_state.previous_response_id is not None
3907
+ )
3908
+ clean_close_hard_continuity_anchor = (
3909
+ close_classification == "clean" and hard_session_affinity and request_state.hard_continuity_anchor
3910
+ )
3911
+ clean_close_retry_for_current_close = (
3912
+ close_classification == "clean"
3913
+ and request_state.clean_close_retry_close_generation != close_generation
3914
+ and not request_state.clean_close_retry_in_progress
3915
+ )
3916
+ additional_clean_close_retry = (
3917
+ clean_close_retry_for_current_close
3918
+ and request_state.replay_count == 1
3919
+ and request_state.response_event_count == 0
3920
+ and request_state.clean_close_replay_count < clean_close_retry_max_count
3921
+ )
3922
+ if request_state.replay_count >= 1 and not additional_clean_close_retry:
3923
+ return False
3924
+ account_bound_replay = False
3925
+ if request_state.previous_response_id is not None:
3926
+ require_preferred_reconnect = False
3927
+ if account_neutral_recovery:
3928
+ request_state.preferred_account_id = session.account.id
3929
+ switch_text = None
3930
+ else:
3931
+ switch_text = _prepare_websocket_request_state_for_account_switch(request_state)
3932
+ if switch_text is None:
3933
+ # The retained full body may be retry-safe for continuity
3934
+ # while still naming an account-scoped uploaded file. In
3935
+ # that case retry on the same owner-bound anchor instead of
3936
+ # letting visible-output replay strip the anchor and migrate.
3937
+ fresh_retry_safe = request_state.fresh_upstream_request_is_retry_safe
3938
+ request_state.fresh_upstream_request_is_retry_safe = False
3939
+ try:
3940
+ request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state)
3941
+ finally:
3942
+ request_state.fresh_upstream_request_is_retry_safe = fresh_retry_safe
3943
+ if request_text is None:
3944
+ return False
3945
+ require_preferred_reconnect = request_state.preferred_account_id is not None
3946
+ else:
3947
+ request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state)
3948
+ if request_text is None:
3949
+ return False
3950
+ if not hard_owner_bound:
3951
+ request_state.excluded_account_ids.add(session.account.id)
3952
+ else:
3953
+ # Account-scoped uploaded files cannot be replayed on a
3954
+ # different owner. Keep the preferred account mandatory for
3955
+ # both silent recovery and clean-close recovery.
3956
+ candidate_text = (
3957
+ request_state.fresh_upstream_request_text
3958
+ if request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text
3959
+ else request_state.request_text
3960
+ )
3961
+ # The send boundary decorates durable operations with
3962
+ # codex_lb_operation_id after selection. Keep that operation
3963
+ # identity on its owner unless a dedicated rebind path has
3964
+ # already replaced the operation ID.
3965
+ candidate_portable = request_state.operation_id is None and (
3966
+ _websocket_request_text_is_account_neutral_fresh_replay(candidate_text)
3967
+ )
3968
+ request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state)
3969
+ if request_text is None or request_text != candidate_text:
3970
+ return False
3971
+ account_bound_replay = not candidate_portable
3972
+ require_preferred_reconnect = (
3973
+ account_neutral_recovery or account_bound_replay or request_state.file_required_preferred_account
3974
+ )
3975
+ if account_neutral_recovery or account_bound_replay:
3976
+ request_state.preferred_account_id = session.account.id
3977
+ elif not request_state.file_required_preferred_account:
3978
+ if hard_owner_bound and not model_fallback_replay and not fresh_hard_request_account_switch_allowed:
3979
+ request_state.preferred_account_id = session.account.id
3980
+ else:
3981
+ request_state.preferred_account_id = None
3982
+ # An accepted replay whose session affinity may resolve
3983
+ # a hard sticky owner reconnects unexcluded: the owner
3984
+ # is the only account selection can return, so the
3985
+ # exclusion would spin on ``hard_affinity_saturated``
3986
+ # until the bridge request budget ran out.
3987
+ if model_fallback_replay or _http_bridge_accepted_replay_may_exclude_account(
3988
+ request_state, session
3989
+ ):
3990
+ request_state.excluded_account_ids.add(session.account.id)
3991
+ if session.account.id in request_state.excluded_account_ids:
3992
+ session.upstream_turn_state = None
3993
+ session.downstream_turn_state = None
3994
+ session.headers = {
3995
+ key: value for key, value in session.headers.items() if key.lower() != "x-codex-turn-state"
3996
+ }
3997
+ if close_classification == "clean":
3998
+ if not clean_close_retry_for_current_close:
3999
+ return False
4000
+ request_state.clean_close_retry_in_progress = True
4001
+ request_state.clean_close_retry_result = None
4002
+ request_state.clean_close_retry_close_generation = close_generation
4003
+ if additional_clean_close_retry:
4004
+ request_state.clean_close_replay_count += 1
4005
+ retry_jitter_seconds = (
4006
+ self._http_bridge_clean_close_retry_jitter_seconds() if additional_clean_close_retry else 0.0
4007
+ )
4008
+ retry_event = "retry_precreated_clean_close" if additional_clean_close_retry else "retry_precreated"
4009
+ _log_http_bridge_event(
4010
+ retry_event,
4011
+ session.key,
4012
+ account_id=session.account.id,
4013
+ model=session.request_model,
4014
+ detail="accepted_lifecycle_replay" if request_state.replay_downstream_response_id is not None else None,
4015
+ pending_count=1,
4016
+ cache_key_family=session.key.affinity_kind,
4017
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
4018
+ )
4019
+ reconnect_reader_kwargs = {"restart_reader": True} if restart_reader else {}
4020
+ try:
4021
+ if retry_jitter_seconds > 0:
4022
+ logger.info(
4023
+ "HTTP bridge clean-close retry jitter request_id=%s sleep_seconds=%.3f",
4024
+ request_state.request_id,
4025
+ retry_jitter_seconds,
4026
+ )
4027
+ await scheduler_for(self).sleep(retry_jitter_seconds)
4028
+ request_deadline = request_state.bridge_request_deadline
4029
+ if request_deadline is None:
4030
+ request_deadline = request_state.started_at + _http_bridge_request_budget_seconds(
4031
+ _service_get_settings()
4032
+ )
4033
+ now_monotonic = clock.monotonic()
4034
+ async with session.pending_lock:
4035
+ request_still_owned = (
4036
+ request_state in session.pending_requests and not request_state.draining_until_terminal
4037
+ )
4038
+ if not request_still_owned or now_monotonic >= request_deadline:
4039
+ logger.info(
4040
+ "HTTP bridge clean-close retry abandoned after jitter request_id=%s "
4041
+ "still_owned=%s deadline_expired=%s",
4042
+ request_state.request_id,
4043
+ request_still_owned,
4044
+ now_monotonic >= request_deadline,
4045
+ )
4046
+ request_state.clean_close_retry_result = False
4047
+ return False
4048
+ # A fresh hard-session replay may select a replacement account.
4049
+ # The admission lease is account-scoped, so release the old
4050
+ # account's lease before reconnecting; the post-reconnect path
4051
+ # below acquires a lease for the account actually selected.
4052
+ if fresh_hard_request_account_switch_allowed:
4053
+ await self._release_request_state_account_response_create_lease(request_state)
4054
+ if hard_owner_bound and not model_fallback_replay and not fresh_hard_request_account_switch_allowed:
4055
+ await self._reconnect_http_bridge_session(
4056
+ session,
4057
+ request_state=request_state,
4058
+ require_same_account=True,
4059
+ **reconnect_reader_kwargs,
4060
+ )
4061
+ elif require_preferred_reconnect:
4062
+ await self._reconnect_http_bridge_session(
4063
+ session,
4064
+ request_state=request_state,
4065
+ require_same_account=account_neutral_recovery or account_bound_replay,
4066
+ require_preferred_account=True,
4067
+ **reconnect_reader_kwargs,
4068
+ )
4069
+ elif clean_close_hard_continuation or clean_close_hard_continuity_anchor:
4070
+ await self._reconnect_http_bridge_session(
4071
+ session,
4072
+ request_state=request_state,
4073
+ # Continuity anchors (previous_response_id and turn-state)
4074
+ # are account-bound. Do not migrate them while recovering a
4075
+ # clean handoff close.
4076
+ require_same_account=True,
4077
+ **reconnect_reader_kwargs,
4078
+ )
4079
+ else:
4080
+ await self._reconnect_http_bridge_session(
4081
+ session,
4082
+ request_state=request_state,
4083
+ **reconnect_reader_kwargs,
4084
+ )
4085
+ if request_state.account_response_create_lease is None:
4086
+ current_settings = await _service_get_settings_cache().get()
4087
+ request_state.account_response_create_lease = (
4088
+ await self._acquire_account_response_create_lease_or_overload(
4089
+ account_id=session.account.id,
4090
+ request_id=request_state.request_log_id or request_state.request_id,
4091
+ surface="http_bridge",
4092
+ concurrency_caps=effective_account_concurrency_caps(current_settings),
4093
+ )
4094
+ )
4095
+ request_state.account_response_create_release = self._load_balancer.release_account_lease
4096
+ enforce_capacity_retry_deadline = request_state.response_create_admission_reacquire_required
4097
+ if enforce_capacity_retry_deadline:
4098
+ retry_deadline = request_state.bridge_request_deadline
4099
+ if retry_deadline is None:
4100
+ retry_deadline = request_state.started_at + _http_bridge_request_budget_seconds(
4101
+ _service_get_settings()
4102
+ )
4103
+ remaining_retry_budget_seconds = retry_deadline - clock.monotonic()
4104
+ if remaining_retry_budget_seconds <= 0:
4105
+ request_state.response_create_admission_reacquire_required = False
4106
+ await self._release_request_state_account_response_create_lease(request_state)
4107
+ return False
4108
+ try:
4109
+ request_state.response_create_admission = await scheduler_for(self).wait_for(
4110
+ self._get_work_admission().acquire_response_create(),
4111
+ timeout=remaining_retry_budget_seconds,
4112
+ )
4113
+ except TimeoutError:
4114
+ request_state.response_create_admission_reacquire_required = False
4115
+ await self._release_request_state_account_response_create_lease(request_state)
4116
+ return False
4117
+ request_state.response_create_admission_reacquire_required = False
4118
+ if clock.monotonic() >= retry_deadline:
4119
+ request_state.response_create_admission.release()
4120
+ request_state.response_create_admission = None
4121
+ await self._release_request_state_account_response_create_lease(request_state)
4122
+ return False
4123
+ async with session.pending_lock:
4124
+ retry_consumer_attached = (
4125
+ request_state in session.pending_requests
4126
+ and request_state.event_queue is not None
4127
+ and not request_state.draining_until_terminal
4128
+ )
4129
+ if not retry_consumer_attached:
4130
+ request_state.response_create_admission.release()
4131
+ request_state.response_create_admission = None
4132
+ await self._release_request_state_account_response_create_lease(request_state)
4133
+ return False
4134
+ request_text = self._http_bridge_text_with_account_installation_id(session, request_state, request_text)
4135
+ await _send_http_bridge_request_text_with_archive_id(session, request_state, request_text, clock=clock)
4136
+ session.last_used_at = clock.monotonic()
4137
+ request_state.clean_close_retry_result = True
4138
+ return True
4139
+ except asyncio.CancelledError:
4140
+ request_state.clean_close_retry_result = False
4141
+ raise
4142
+ except UpstreamWebSocketTransportError:
4143
+ request_state.clean_close_retry_result = False
4144
+ raise
4145
+ except Exception as exc:
4146
+ request_state.clean_close_retry_result = False
4147
+ (
4148
+ request_state.error_http_status_override,
4149
+ request_state.error_code_override,
4150
+ request_state.error_message_override,
4151
+ request_state.error_type_override,
4152
+ request_state.error_param_override,
4153
+ ) = _http_bridge_precreated_retry_failure_error(exc)
4154
+ if isinstance(exc, ProxyResponseError):
4155
+ logger.info(
4156
+ "HTTP bridge pre-created retry failed with terminal proxy error code=%s message=%s",
4157
+ request_state.error_code_override,
4158
+ request_state.error_message_override,
4159
+ )
4160
+ else:
4161
+ logger.warning("HTTP bridge pre-created retry failed", exc_info=True)
4162
+ return False
4163
+ finally:
4164
+ request_state.clean_close_retry_in_progress = False
4165
+
4166
+ async def _retry_http_bridge_precreated_auth_request(
4167
+ self: Any,
4168
+ session: "_HTTPBridgeSession",
4169
+ request_state: _WebSocketRequestState,
4170
+ *,
4171
+ error_message: str | None,
4172
+ ) -> Literal["not_replayable", "retried", "failed"]:
4173
+ clock = clock_for(self)
4174
+ permanent_failure_code = _websocket_auth_failure_permanent_code(error_message)
4175
+ bound_to_current_account = request_state.replay_required_account_id == session.account.id
4176
+ if bound_to_current_account and (
4177
+ _websocket_auth_failure_requires_reauth(error_message)
4178
+ or request_state.auth_replay_counts_by_account.get(session.account.id, 0) > 0
4179
+ ):
4180
+ failure_code = permanent_failure_code or _WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE
4181
+ await self._load_balancer.mark_permanent_failure(session.account, failure_code)
4182
+ setattr(request_state, "account_health_error_handled", True)
4183
+ request_state.force_refresh_account_id = None
4184
+ request_state.preferred_account_id = None
4185
+ request_state.excluded_account_ids.add(session.account.id)
4186
+ return "not_replayable"
4187
+ request_text = _prepare_websocket_request_state_for_auth_replay(
4188
+ request_state,
4189
+ current_account_id=session.account.id,
4190
+ )
4191
+ if request_text is None:
4192
+ await self._load_balancer.mark_permanent_failure(session.account, permanent_failure_code)
4193
+ setattr(request_state, "account_health_error_handled", True)
4194
+ request_state.force_refresh_account_id = None
4195
+ request_state.preferred_account_id = None
4196
+ request_state.excluded_account_ids.add(session.account.id)
4197
+ return "not_replayable"
4198
+
4199
+ if _websocket_auth_failure_requires_reauth(error_message):
4200
+ failure_code = permanent_failure_code
4201
+ elif request_state.auth_replay_counts_by_account.get(session.account.id, 0) == 0:
4202
+ failure_code = None
4203
+ request_state.auth_replay_counts_by_account[session.account.id] = 1
4204
+ request_state.force_refresh_account_id = session.account.id
4205
+ request_state.preferred_account_id = session.account.id
4206
+ else:
4207
+ failure_code = _WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE
4208
+
4209
+ if failure_code is not None:
4210
+ await self._load_balancer.mark_permanent_failure(session.account, failure_code)
4211
+ request_state.force_refresh_account_id = None
4212
+ request_state.preferred_account_id = None
4213
+ request_state.excluded_account_ids.add(session.account.id)
4214
+ if is_http_bridge_account_neutral_replay(
4215
+ kind=session.key.affinity_kind,
4216
+ key=session.key.affinity_key,
4217
+ ):
4218
+ setattr(request_state, "account_health_error_handled", True)
4219
+ return "not_replayable"
4220
+
4221
+ async with session.pending_lock:
4222
+ if request_state not in session.pending_requests:
4223
+ session.pending_requests.appendleft(request_state)
4224
+ session.queued_request_count += 1
4225
+
4226
+ _log_http_bridge_event(
4227
+ "retry_precreated_auth",
4228
+ session.key,
4229
+ account_id=session.account.id,
4230
+ model=session.request_model,
4231
+ pending_count=await self._http_bridge_pending_count(session),
4232
+ cache_key_family=session.key.affinity_kind,
4233
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
4234
+ )
4235
+ try:
4236
+ await self._reconnect_http_bridge_session(
4237
+ session,
4238
+ request_state=request_state,
4239
+ require_same_account=(
4240
+ bound_to_current_account
4241
+ or is_http_bridge_account_neutral_replay(
4242
+ kind=session.key.affinity_kind,
4243
+ key=session.key.affinity_key,
4244
+ )
4245
+ ),
4246
+ require_preferred_account=bound_to_current_account,
4247
+ )
4248
+ request_text = self._http_bridge_text_with_account_installation_id(session, request_state, request_text)
4249
+ await _send_http_bridge_request_text_with_archive_id(session, request_state, request_text, clock=clock)
4250
+ session.last_used_at = clock.monotonic()
4251
+ return "retried"
4252
+ except UpstreamWebSocketTransportError:
4253
+ raise
4254
+ except Exception as exc:
4255
+ (
4256
+ request_state.error_http_status_override,
4257
+ request_state.error_code_override,
4258
+ request_state.error_message_override,
4259
+ request_state.error_type_override,
4260
+ request_state.error_param_override,
4261
+ ) = _http_bridge_precreated_retry_failure_error(exc)
4262
+ if isinstance(exc, ProxyResponseError):
4263
+ logger.info(
4264
+ "HTTP bridge pre-created auth retry failed with terminal proxy error code=%s message=%s",
4265
+ request_state.error_code_override,
4266
+ request_state.error_message_override,
4267
+ )
4268
+ else:
4269
+ logger.warning("HTTP bridge pre-created auth retry failed", exc_info=True)
4270
+ return "failed"
4271
+
4272
+ async def _retry_http_bridge_security_work_request(
4273
+ self: Any,
4274
+ session: "_HTTPBridgeSession",
4275
+ request_state: _WebSocketRequestState,
4276
+ ) -> bool:
4277
+ clock = clock_for(self)
4278
+ if session.account.security_work_authorized:
4279
+ return False
4280
+ if request_state.response_id is not None:
4281
+ return False
4282
+ if request_state.replay_count >= 1:
4283
+ return False
4284
+ if is_http_bridge_account_neutral_replay(
4285
+ kind=session.key.affinity_kind,
4286
+ key=session.key.affinity_key,
4287
+ ):
4288
+ return False
4289
+ if request_state.file_required_preferred_account:
4290
+ return False
4291
+ if not _websocket_request_can_replay_before_visible_output(request_state):
4292
+ return False
4293
+ retry_text = _prepare_websocket_request_state_for_account_switch(request_state)
4294
+ if retry_text is None:
4295
+ return False
4296
+
4297
+ owner_account_id = session.account.id
4298
+ previous_replay_count = request_state.replay_count
4299
+ previous_response_id = request_state.response_id
4300
+ previous_response_event_count = request_state.response_event_count
4301
+ previous_upstream_model_output_seen = request_state.upstream_model_output_seen
4302
+ previous_affinity_policy = request_state.affinity_policy
4303
+ previous_affinity_observation = request_state.affinity_observation
4304
+ previous_session_affinity = session.affinity
4305
+ previous_session_codex_session = session.codex_session
4306
+ previous_session_upstream_turn_state = session.upstream_turn_state
4307
+ previous_session_downstream_turn_state = session.downstream_turn_state
4308
+ previous_session_turn_state_aliases = set(session.downstream_turn_state_aliases)
4309
+ previous_session_turn_state_alias_registration_generations = dict(
4310
+ session.turn_state_alias_registration_generations
4311
+ )
4312
+ previous_session_headers = session.headers
4313
+ request_state.preferred_account_id = None
4314
+ request_state.excluded_account_ids.add(owner_account_id)
4315
+ request_state.affinity_policy = replace(
4316
+ request_state.affinity_policy,
4317
+ key=None,
4318
+ kind=None,
4319
+ reallocate_sticky=True,
4320
+ )
4321
+ if previous_affinity_observation is not None:
4322
+ request_state.affinity_observation = AffinityObservation.retaining_source(
4323
+ previous_affinity_observation.source, request_state.affinity_policy
4324
+ )
4325
+ replacement_session_affinity = replace(
4326
+ session.affinity,
4327
+ key=None,
4328
+ kind=None,
4329
+ reallocate_sticky=True,
4330
+ codex_session_source=None,
4331
+ )
4332
+
4333
+ request_state.replay_count += 1
4334
+ request_state.response_id = None
4335
+ request_state.response_event_count = 0
4336
+ request_state.upstream_model_output_seen = False
4337
+ request_state.deferred_reasoning_downstream_texts = []
4338
+ request_state.awaiting_response_created = True
4339
+ if retry_text != request_state.request_text:
4340
+ _bind_http_bridge_proxy_injected_anchor(
4341
+ self,
4342
+ request_state,
4343
+ response_id=None,
4344
+ proxy_injected=False,
4345
+ )
4346
+ request_state.request_text = retry_text
4347
+
4348
+ async with session.pending_lock:
4349
+ if request_state not in session.pending_requests:
4350
+ session.pending_requests.append(request_state)
4351
+ session.queued_request_count += 1
4352
+
4353
+ _log_http_bridge_event(
4354
+ "retry_security_work_authorized",
4355
+ session.key,
4356
+ account_id=session.account.id,
4357
+ model=session.request_model,
4358
+ pending_count=await self._http_bridge_pending_count(session),
4359
+ cache_key_family=session.key.affinity_kind,
4360
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
4361
+ )
4362
+ reconnected = False
4363
+ operation_rebound_for_retry = False
4364
+ security_retry_send_started = False
4365
+
4366
+ def mark_security_retry_send_started() -> None:
4367
+ nonlocal security_retry_send_started
4368
+ security_retry_send_started = True
4369
+
4370
+ try:
4371
+ request_state.precreated_replay_account_id = session.account.id
4372
+ await self._release_request_state_account_response_create_lease(request_state)
4373
+ await _call_with_supported_optional_kwargs(
4374
+ self._reconnect_http_bridge_session,
4375
+ session,
4376
+ optional_kwargs={
4377
+ "owner_rebind_affinity": previous_session_affinity,
4378
+ "selection_affinity": replacement_session_affinity,
4379
+ },
4380
+ request_state=request_state,
4381
+ require_security_work_authorized=True,
4382
+ )
4383
+ reconnected = True
4384
+ settings = await _service_get_settings_cache().get()
4385
+ request_state.account_response_create_lease = await self._acquire_account_response_create_lease_or_overload(
4386
+ account_id=session.account.id,
4387
+ request_id=request_state.request_id,
4388
+ surface="http_bridge_security_retry",
4389
+ concurrency_caps=effective_account_concurrency_caps(settings),
4390
+ )
4391
+ request_state.account_response_create_release = self._load_balancer.release_account_lease
4392
+ if session.account.id != owner_account_id:
4393
+ if (
4394
+ previous_session_affinity.codex_session_source in {"session_header", "thread_header"}
4395
+ and previous_session_affinity.selection_key is not None
4396
+ and previous_session_affinity.kind is not None
4397
+ ):
4398
+ async with self._repo_factory() as repos:
4399
+ await repos.sticky_sessions.upsert(
4400
+ previous_session_affinity.selection_key,
4401
+ session.account.id,
4402
+ kind=previous_session_affinity.kind,
4403
+ )
4404
+ if (
4405
+ request_state.operation_registered
4406
+ and request_state.operation_id is not None
4407
+ and request_state.operation_fingerprint is not None
4408
+ and session.durable_session_id is not None
4409
+ and session.durable_owner_epoch is not None
4410
+ ):
4411
+ record_operation = getattr(self._durable_bridge, "record_operation", None)
4412
+ if not callable(record_operation):
4413
+ raise ProxyResponseError(
4414
+ 502,
4415
+ openai_error(
4416
+ "bridge_continuity_persistence_failed",
4417
+ "Security-work recovery operation could not be re-fenced; retry the request.",
4418
+ ),
4419
+ )
4420
+ rebound_operation = await record_operation(
4421
+ operation_id=request_state.operation_id,
4422
+ session_id=session.durable_session_id,
4423
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4424
+ owner_epoch=session.durable_owner_epoch,
4425
+ request_fingerprint=request_state.operation_fingerprint,
4426
+ api_key_scope=durable_bridge_api_key_scope(session.key.api_key_id),
4427
+ account_id=session.account.id,
4428
+ model=request_state.model,
4429
+ parent_response_id=request_state.operation_parent_response_id or request_state.previous_response_id,
4430
+ )
4431
+ if rebound_operation is None or getattr(rebound_operation, "state", None) != "submitted":
4432
+ raise ProxyResponseError(
4433
+ 502,
4434
+ openai_error(
4435
+ "bridge_continuity_persistence_failed",
4436
+ "Security-work recovery operation could not be re-fenced; retry the request.",
4437
+ ),
4438
+ )
4439
+ operation_rebound_for_retry = True
4440
+ retry_text = self._http_bridge_text_with_account_installation_id(session, request_state, retry_text)
4441
+ await _send_http_bridge_request_text_with_archive_id(
4442
+ session,
4443
+ request_state,
4444
+ retry_text,
4445
+ on_send_started=mark_security_retry_send_started,
4446
+ clock=clock,
4447
+ )
4448
+ session.last_used_at = clock.monotonic()
4449
+ return True
4450
+ except UpstreamWebSocketTransportError:
4451
+ raise
4452
+ except Exception as exc:
4453
+ logger.warning("HTTP bridge security-work retry failed", exc_info=True)
4454
+ if (
4455
+ operation_rebound_for_retry
4456
+ and not security_retry_send_started
4457
+ and request_state.operation_id is not None
4458
+ and session.durable_session_id is not None
4459
+ and session.durable_owner_epoch is not None
4460
+ ):
4461
+ update_operation = getattr(self._durable_bridge, "update_operation", None)
4462
+ if callable(update_operation):
4463
+ try:
4464
+ restored = await update_operation(
4465
+ operation_id=request_state.operation_id,
4466
+ session_id=session.durable_session_id,
4467
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4468
+ owner_epoch=session.durable_owner_epoch,
4469
+ state="failed",
4470
+ )
4471
+ if not restored:
4472
+ logger.info(
4473
+ "HTTP bridge security retry failed to restore operation fence operation_id=%s",
4474
+ request_state.operation_id,
4475
+ )
4476
+ except Exception:
4477
+ logger.warning(
4478
+ "Failed to restore HTTP bridge security retry operation operation_id=%s",
4479
+ request_state.operation_id,
4480
+ exc_info=True,
4481
+ )
4482
+ if isinstance(exc, ProxyResponseError):
4483
+ error = _parse_openai_error(exc.payload)
4484
+ code = _normalize_error_code(error.code if error else None, error.type if error else None)
4485
+ if code == _NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE and request_state.event_queue is not None:
4486
+ await request_state.event_queue.put(
4487
+ format_sse_event(
4488
+ _security_work_advisory_event(
4489
+ code=_NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE,
4490
+ message=_SECURITY_WORK_NO_AUTHORIZED_ACCOUNTS_MESSAGE,
4491
+ request_id=request_state.request_log_id or request_state.request_id,
4492
+ action="forward_original_security_work_error",
4493
+ )
4494
+ )
4495
+ )
4496
+ async with session.pending_lock:
4497
+ if request_state in session.pending_requests:
4498
+ session.pending_requests.remove(request_state)
4499
+ session.queued_request_count = max(0, session.queued_request_count - 1)
4500
+ if reconnected:
4501
+ # Ownership and the replacement socket were already swapped,
4502
+ # but response.create never reached that socket. Retire the
4503
+ # replacement instead of leaving a live bridge with aliases
4504
+ # rebound to a turn that was never submitted.
4505
+ session.upstream_control.reconnect_requested = True
4506
+ session.upstream_control.retire_after_drain = True
4507
+ await self._retire_http_bridge_after_drain_if_ready(session)
4508
+ else:
4509
+ request_state.replay_count = previous_replay_count
4510
+ request_state.response_id = previous_response_id
4511
+ request_state.response_event_count = previous_response_event_count
4512
+ request_state.upstream_model_output_seen = previous_upstream_model_output_seen
4513
+ request_state.deferred_reasoning_downstream_texts = []
4514
+ request_state.affinity_policy = previous_affinity_policy
4515
+ request_state.affinity_observation = previous_affinity_observation
4516
+ session.affinity = previous_session_affinity
4517
+ session.codex_session = previous_session_codex_session
4518
+ session.upstream_turn_state = previous_session_upstream_turn_state
4519
+ session.downstream_turn_state = previous_session_downstream_turn_state
4520
+ async with self._http_bridge_lock:
4521
+ session.downstream_turn_state_aliases.update(previous_session_turn_state_aliases)
4522
+ session.turn_state_alias_registration_generations.update(
4523
+ previous_session_turn_state_alias_registration_generations
4524
+ )
4525
+ _register_http_bridge_turn_state_aliases_locked(self, session)
4526
+ session.headers = previous_session_headers
4527
+ return False
4528
+
4529
+ async def _claim_http_bridge_replacement_before_swap(
4530
+ self: Any,
4531
+ session: "_HTTPBridgeSession",
4532
+ *,
4533
+ account_id: str,
4534
+ upstream: Any,
4535
+ release_selected_account_lease: Any,
4536
+ owner_rebind_affinity: _AffinityPolicy,
4537
+ ) -> None:
4538
+ if account_id == session.account.id:
4539
+ return
4540
+ try:
4541
+ if owner_rebind_affinity.legacy_selection_key is not None:
4542
+ async with self._repo_factory() as repos:
4543
+ # A goal restart abandons only session-header interpretation
4544
+ # of the legacy raw row. Preserve that typed capability here:
4545
+ # omitting it would resurrect the retained turn-state owner
4546
+ # during a later security-authorized replacement.
4547
+ legacy_owner_id = await repos.sticky_sessions.get_account_id(
4548
+ owner_rebind_affinity.legacy_selection_key,
4549
+ # The new thread row may be PROMPT_CACHE, but the raw
4550
+ # compatibility row has always been CODEX_SESSION and
4551
+ # remains durable hard ownership.
4552
+ kind=StickySessionKind.CODEX_SESSION,
4553
+ max_age_seconds=None,
4554
+ continuity_source=(owner_rebind_affinity.legacy_continuity_source or "session_header"),
4555
+ )
4556
+ if legacy_owner_id is not None and legacy_owner_id != account_id:
4557
+ raise ProxyResponseError(
4558
+ 502,
4559
+ openai_error(
4560
+ "continuity_owner_conflict",
4561
+ "Security retry conflicts with a legacy continuity owner.",
4562
+ error_type="server_error",
4563
+ ),
4564
+ )
4565
+ if session.durable_session_id is not None:
4566
+ await _call_with_supported_optional_kwargs(
4567
+ self._claim_durable_http_bridge_session,
4568
+ session,
4569
+ optional_kwargs={
4570
+ "claim_account_id": account_id,
4571
+ "clear_latest_turn_state": True,
4572
+ },
4573
+ allow_takeover=True,
4574
+ force_owner_epoch_advance=True,
4575
+ )
4576
+ except BaseException:
4577
+ try:
4578
+ await upstream.close()
4579
+ except Exception:
4580
+ logger.debug("Failed to close unclaimed HTTP bridge replacement websocket", exc_info=True)
4581
+ await release_selected_account_lease()
4582
+ raise