codexLB-up 1.25.0b10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (827) hide show
  1. app/__init__.py +10 -0
  2. app/admin_cli.py +494 -0
  3. app/cli.py +339 -0
  4. app/codex_sessions_retag.py +487 -0
  5. app/core/__init__.py +0 -0
  6. app/core/audit/__init__.py +0 -0
  7. app/core/audit/service.py +216 -0
  8. app/core/audit/types.py +94 -0
  9. app/core/auth/__init__.py +236 -0
  10. app/core/auth/api_key_cache.py +65 -0
  11. app/core/auth/dashboard_access.py +429 -0
  12. app/core/auth/dashboard_mode.py +171 -0
  13. app/core/auth/dashboard_session_ttl.py +74 -0
  14. app/core/auth/dashboard_users_cache.py +120 -0
  15. app/core/auth/dependencies.py +710 -0
  16. app/core/auth/external_identity.py +78 -0
  17. app/core/auth/guardian.py +374 -0
  18. app/core/auth/models.py +49 -0
  19. app/core/auth/providers/__init__.py +129 -0
  20. app/core/auth/providers/oidc.py +838 -0
  21. app/core/auth/providers/registry.py +118 -0
  22. app/core/auth/refresh.py +394 -0
  23. app/core/auth/step_up.py +117 -0
  24. app/core/auth/totp.py +81 -0
  25. app/core/balancer/__init__.py +71 -0
  26. app/core/balancer/logic.py +1612 -0
  27. app/core/balancer/rendezvous_hash.py +21 -0
  28. app/core/balancer/types.py +21 -0
  29. app/core/bootstrap.py +172 -0
  30. app/core/cache/invalidation.py +426 -0
  31. app/core/clients/__init__.py +0 -0
  32. app/core/clients/codex.py +996 -0
  33. app/core/clients/codex_version.py +187 -0
  34. app/core/clients/codex_version_snapshot.py +2 -0
  35. app/core/clients/files.py +534 -0
  36. app/core/clients/headers.py +29 -0
  37. app/core/clients/http.py +511 -0
  38. app/core/clients/model_fetcher.py +276 -0
  39. app/core/clients/native_egress.py +1304 -0
  40. app/core/clients/oauth.py +477 -0
  41. app/core/clients/proxy.py +6089 -0
  42. app/core/clients/proxy_websocket.py +1411 -0
  43. app/core/clients/rate_limit_reset_credits.py +412 -0
  44. app/core/clients/stream_errors.py +12 -0
  45. app/core/clients/thread_cache_identity.py +347 -0
  46. app/core/clients/usage.py +467 -0
  47. app/core/clock.py +170 -0
  48. app/core/config/__init__.py +0 -0
  49. app/core/config/background_jobs.py +73 -0
  50. app/core/config/context_window_overrides.py +166 -0
  51. app/core/config/dashboard_overrides.py +145 -0
  52. app/core/config/inheritable.py +47 -0
  53. app/core/config/key_fingerprint.py +115 -0
  54. app/core/config/settings.py +872 -0
  55. app/core/config/settings_cache.py +100 -0
  56. app/core/config/spool_retention.py +217 -0
  57. app/core/config/tiers.py +174 -0
  58. app/core/conversation_archive.py +568 -0
  59. app/core/crypto.py +37 -0
  60. app/core/draining.py +11 -0
  61. app/core/errors.py +365 -0
  62. app/core/exceptions.py +155 -0
  63. app/core/handlers/__init__.py +3 -0
  64. app/core/handlers/exceptions.py +478 -0
  65. app/core/http_protocol.py +176 -0
  66. app/core/http_protocol_httptools.py +204 -0
  67. app/core/ingress_limits.py +17 -0
  68. app/core/metadata_store.py +18 -0
  69. app/core/metrics/__init__.py +0 -0
  70. app/core/metrics/middleware.py +74 -0
  71. app/core/metrics/prometheus.py +604 -0
  72. app/core/middleware/__init__.py +25 -0
  73. app/core/middleware/api_firewall.py +111 -0
  74. app/core/middleware/app_version.py +37 -0
  75. app/core/middleware/dashboard_auth_proxy.py +55 -0
  76. app/core/middleware/dashboard_csrf.py +177 -0
  77. app/core/middleware/dashboard_gzip.py +41 -0
  78. app/core/middleware/dashboard_overrides.py +59 -0
  79. app/core/middleware/firewall_cache.py +91 -0
  80. app/core/middleware/inflight.py +126 -0
  81. app/core/middleware/multipart_content_encoding.py +129 -0
  82. app/core/middleware/path_rewrite.py +148 -0
  83. app/core/middleware/request_body_limit.py +196 -0
  84. app/core/middleware/request_decompression.py +209 -0
  85. app/core/middleware/request_id.py +72 -0
  86. app/core/middleware/required_capability_http.py +116 -0
  87. app/core/middleware/trusted_proxy_headers.py +36 -0
  88. app/core/multipart.py +237 -0
  89. app/core/multipart_fields.py +91 -0
  90. app/core/openai/__init__.py +0 -0
  91. app/core/openai/chat_requests.py +493 -0
  92. app/core/openai/chat_responses.py +931 -0
  93. app/core/openai/contracts.py +46 -0
  94. app/core/openai/exceptions.py +24 -0
  95. app/core/openai/host_models.py +14 -0
  96. app/core/openai/images.py +413 -0
  97. app/core/openai/message_coercion.py +376 -0
  98. app/core/openai/model_refresh_scheduler.py +423 -0
  99. app/core/openai/model_registry.py +997 -0
  100. app/core/openai/model_registry_store.py +548 -0
  101. app/core/openai/models.py +172 -0
  102. app/core/openai/parsing.py +97 -0
  103. app/core/openai/requests.py +1980 -0
  104. app/core/openai/strict_schema.py +199 -0
  105. app/core/openai/tool_call_safety.py +99 -0
  106. app/core/openai/v1_requests.py +187 -0
  107. app/core/plan_types.py +90 -0
  108. app/core/prestop.py +202 -0
  109. app/core/rate_limiter/__init__.py +0 -0
  110. app/core/rate_limiter/db_rate_limiter.py +215 -0
  111. app/core/request_locality.py +473 -0
  112. app/core/resilience/__init__.py +0 -0
  113. app/core/resilience/backpressure.py +57 -0
  114. app/core/resilience/bulkhead.py +202 -0
  115. app/core/resilience/circuit_breaker.py +150 -0
  116. app/core/resilience/degradation.py +55 -0
  117. app/core/resilience/loop_lag_monitor.py +60 -0
  118. app/core/resilience/memory_monitor.py +137 -0
  119. app/core/resilience/network_recovery.py +256 -0
  120. app/core/resilience/overload.py +98 -0
  121. app/core/resilience/toggles.py +114 -0
  122. app/core/retention/__init__.py +0 -0
  123. app/core/retention/job.py +240 -0
  124. app/core/retention/scheduler.py +69 -0
  125. app/core/runtime_logging.py +601 -0
  126. app/core/scheduling/__init__.py +0 -0
  127. app/core/scheduling/leader_election.py +993 -0
  128. app/core/scheduling/leader_election_handle.py +25 -0
  129. app/core/server.py +140 -0
  130. app/core/shutdown.py +326 -0
  131. app/core/socket_peer.py +22 -0
  132. app/core/startup.py +41 -0
  133. app/core/timeout_invariants.py +347 -0
  134. app/core/tracing/__init__.py +0 -0
  135. app/core/tracing/otel.py +207 -0
  136. app/core/types.py +6 -0
  137. app/core/upstream_proxy/__init__.py +13 -0
  138. app/core/upstream_proxy/cache.py +130 -0
  139. app/core/upstream_proxy/resolver.py +189 -0
  140. app/core/upstream_proxy/types.py +104 -0
  141. app/core/usage/__init__.py +407 -0
  142. app/core/usage/depletion.py +109 -0
  143. app/core/usage/live_hub.py +41 -0
  144. app/core/usage/live_snapshots.py +174 -0
  145. app/core/usage/logs.py +212 -0
  146. app/core/usage/metadata_scheduler.py +118 -0
  147. app/core/usage/models.py +54 -0
  148. app/core/usage/pricing.py +576 -0
  149. app/core/usage/pricing_catalog.py +241 -0
  150. app/core/usage/pricing_snapshot.json +821 -0
  151. app/core/usage/quota.py +133 -0
  152. app/core/usage/refresh_policy.py +21 -0
  153. app/core/usage/refresh_scheduler.py +713 -0
  154. app/core/usage/reset_credits_refresh_scheduler.py +435 -0
  155. app/core/usage/types.py +168 -0
  156. app/core/utils/__init__.py +0 -0
  157. app/core/utils/json_guards.py +18 -0
  158. app/core/utils/locks.py +33 -0
  159. app/core/utils/masking.py +20 -0
  160. app/core/utils/proxy_env.py +104 -0
  161. app/core/utils/request_id.py +65 -0
  162. app/core/utils/retry.py +77 -0
  163. app/core/utils/shared_future.py +172 -0
  164. app/core/utils/sse.py +286 -0
  165. app/core/utils/time.py +24 -0
  166. app/db/__init__.py +0 -0
  167. app/db/account_identity_lock.py +56 -0
  168. app/db/alembic/__init__.py +1 -0
  169. app/db/alembic/env.py +70 -0
  170. app/db/alembic/revision_ids.py +43 -0
  171. app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
  172. app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
  173. app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
  174. app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
  175. app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
  176. app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
  177. app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
  178. app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
  179. app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
  180. app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
  181. app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
  182. app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
  183. app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
  184. app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
  185. app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
  186. app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
  187. app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
  188. app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
  189. app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
  190. app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
  191. app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
  192. app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
  193. app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
  194. app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
  195. app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
  196. app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
  197. app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
  198. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
  199. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
  200. app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
  201. app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
  202. app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
  203. app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
  204. app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
  205. app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
  206. app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
  207. app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
  208. app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
  209. app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
  210. app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
  211. app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
  212. app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
  213. app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
  214. app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
  215. app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
  216. app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
  217. app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
  218. app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
  219. app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
  220. app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
  221. app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
  222. app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
  223. app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
  224. app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
  225. app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
  226. app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
  227. app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
  228. app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
  229. app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
  230. app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
  231. app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
  232. app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
  233. app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
  234. app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
  235. app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
  236. app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
  237. app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
  238. app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
  239. app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
  240. app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
  241. app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
  242. app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
  243. app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
  244. app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
  245. app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
  246. app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
  247. app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
  248. app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
  249. app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
  250. app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
  251. app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
  252. app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
  253. app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
  254. app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
  255. app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
  256. app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
  257. app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
  258. app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
  259. app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
  260. app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
  261. app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
  262. app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
  263. app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
  264. app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
  265. app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
  266. app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
  267. app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
  268. app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
  269. app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
  270. app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
  271. app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
  272. app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
  273. app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
  274. app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
  275. app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
  276. app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
  277. app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
  278. app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
  279. app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
  280. app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
  281. app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
  282. app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
  283. app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
  284. app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
  285. app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
  286. app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
  287. app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
  288. app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
  289. app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
  290. app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
  291. app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
  292. app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
  293. app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
  294. app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
  295. app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
  296. app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
  297. app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
  298. app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
  299. app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
  300. app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
  301. app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
  302. app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
  303. app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
  304. app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
  305. app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
  306. app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
  307. app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
  308. app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
  309. app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
  310. app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
  311. app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
  312. app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
  313. app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
  314. app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
  315. app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
  316. app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
  317. app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
  318. app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
  319. app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
  320. app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
  321. app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
  322. app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
  323. app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
  324. app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
  325. app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
  326. app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
  327. app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
  328. app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
  329. app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
  330. app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
  331. app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
  332. app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
  333. app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
  334. app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
  335. app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
  336. app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
  337. app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
  338. app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
  339. app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
  340. app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
  341. app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
  342. app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
  343. app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
  344. app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
  345. app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
  346. app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
  347. app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
  348. app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
  349. app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
  350. app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
  351. app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
  352. app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
  353. app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
  354. app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
  355. app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
  356. app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
  357. app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
  358. app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
  359. app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
  360. app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
  361. app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
  362. app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
  363. app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
  364. app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
  365. app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
  366. app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
  367. app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
  368. app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
  369. app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
  370. app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
  371. app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
  372. app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
  373. app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
  374. app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
  375. app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
  376. app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
  377. app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
  378. app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
  379. app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
  380. app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
  381. app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
  382. app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
  383. app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
  384. app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
  385. app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
  386. app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
  387. app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
  388. app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
  389. app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
  390. app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
  391. app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
  392. app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
  393. app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
  394. app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
  395. app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
  396. app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
  397. app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
  398. app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
  399. app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
  400. app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
  401. app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
  402. app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
  403. app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
  404. app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
  405. app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
  406. app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
  407. app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
  408. app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
  409. app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
  410. app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
  411. app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
  412. app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
  413. app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
  414. app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
  415. app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
  416. app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
  417. app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
  418. app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
  419. app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
  420. app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
  421. app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
  422. app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
  423. app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
  424. app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
  425. app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
  426. app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
  427. app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
  428. app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
  429. app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
  430. app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
  431. app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
  432. app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
  433. app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
  434. app/db/alembic/versions/__init__.py +1 -0
  435. app/db/backup.py +64 -0
  436. app/db/migrate.py +1249 -0
  437. app/db/migration_lock.py +160 -0
  438. app/db/migration_url.py +18 -0
  439. app/db/models.py +3100 -0
  440. app/db/recover.py +244 -0
  441. app/db/session.py +1295 -0
  442. app/db/snapshot.py +46 -0
  443. app/db/sqlite_lock_retry.py +190 -0
  444. app/db/sqlite_utils.py +549 -0
  445. app/dependencies.py +462 -0
  446. app/main.py +1240 -0
  447. app/modules/__init__.py +0 -0
  448. app/modules/accounts/__init__.py +0 -0
  449. app/modules/accounts/api.py +426 -0
  450. app/modules/accounts/auth_manager.py +1439 -0
  451. app/modules/accounts/background_repository.py +160 -0
  452. app/modules/accounts/deletion.py +540 -0
  453. app/modules/accounts/mappers.py +654 -0
  454. app/modules/accounts/refresh_claims.py +352 -0
  455. app/modules/accounts/repository.py +1679 -0
  456. app/modules/accounts/schemas.py +308 -0
  457. app/modules/accounts/service.py +809 -0
  458. app/modules/accounts/usage_rollup.py +486 -0
  459. app/modules/accounts/usage_rollup_scheduler.py +75 -0
  460. app/modules/accounts/usage_time_rollup.py +1037 -0
  461. app/modules/accounts/usage_time_rollup_read.py +522 -0
  462. app/modules/api_keys/__init__.py +1 -0
  463. app/modules/api_keys/api.py +344 -0
  464. app/modules/api_keys/last_used_coalescer.py +216 -0
  465. app/modules/api_keys/limit_windows.py +41 -0
  466. app/modules/api_keys/repository.py +1200 -0
  467. app/modules/api_keys/reset_scheduler.py +85 -0
  468. app/modules/api_keys/schemas.py +135 -0
  469. app/modules/api_keys/service.py +2123 -0
  470. app/modules/audit/__init__.py +0 -0
  471. app/modules/audit/api.py +88 -0
  472. app/modules/audit/repository.py +74 -0
  473. app/modules/audit/schemas.py +33 -0
  474. app/modules/audit/service.py +71 -0
  475. app/modules/auth_providers/__init__.py +0 -0
  476. app/modules/auth_providers/api.py +127 -0
  477. app/modules/auth_providers/config.py +343 -0
  478. app/modules/auth_providers/repository.py +101 -0
  479. app/modules/auth_providers/schemas.py +88 -0
  480. app/modules/auth_providers/seed.py +105 -0
  481. app/modules/auth_providers/service.py +289 -0
  482. app/modules/automations/__init__.py +1 -0
  483. app/modules/automations/api.py +369 -0
  484. app/modules/automations/repository.py +1896 -0
  485. app/modules/automations/scheduler.py +107 -0
  486. app/modules/automations/schemas.py +158 -0
  487. app/modules/automations/service.py +2548 -0
  488. app/modules/cache_isolation_probe/__init__.py +0 -0
  489. app/modules/cache_isolation_probe/api.py +188 -0
  490. app/modules/cache_isolation_probe/prefix.py +144 -0
  491. app/modules/cache_isolation_probe/schemas.py +94 -0
  492. app/modules/cache_isolation_probe/sender.py +229 -0
  493. app/modules/cache_isolation_probe/service.py +525 -0
  494. app/modules/conversation_archive/__init__.py +1 -0
  495. app/modules/conversation_archive/api.py +101 -0
  496. app/modules/conversation_archive/schemas.py +28 -0
  497. app/modules/conversation_archive/service.py +165 -0
  498. app/modules/dashboard/__init__.py +1 -0
  499. app/modules/dashboard/api.py +106 -0
  500. app/modules/dashboard/builders.py +107 -0
  501. app/modules/dashboard/repository.py +304 -0
  502. app/modules/dashboard/schemas.py +148 -0
  503. app/modules/dashboard/service.py +534 -0
  504. app/modules/dashboard/timeframes.py +19 -0
  505. app/modules/dashboard/weekly_pace.py +617 -0
  506. app/modules/dashboard_auth/__init__.py +1 -0
  507. app/modules/dashboard_auth/api.py +1344 -0
  508. app/modules/dashboard_auth/oidc_api.py +753 -0
  509. app/modules/dashboard_auth/oidc_flows.py +330 -0
  510. app/modules/dashboard_auth/repository.py +452 -0
  511. app/modules/dashboard_auth/schemas.py +232 -0
  512. app/modules/dashboard_auth/service.py +1384 -0
  513. app/modules/dashboard_roles/__init__.py +0 -0
  514. app/modules/dashboard_roles/api.py +64 -0
  515. app/modules/dashboard_roles/repository.py +54 -0
  516. app/modules/dashboard_roles/schemas.py +29 -0
  517. app/modules/dashboard_roles/seed.py +80 -0
  518. app/modules/dashboard_roles/service.py +142 -0
  519. app/modules/dashboard_users/__init__.py +0 -0
  520. app/modules/dashboard_users/api.py +278 -0
  521. app/modules/dashboard_users/break_glass.py +169 -0
  522. app/modules/dashboard_users/credentials.py +18 -0
  523. app/modules/dashboard_users/identity_resolver.py +665 -0
  524. app/modules/dashboard_users/repository.py +725 -0
  525. app/modules/dashboard_users/schemas.py +117 -0
  526. app/modules/dashboard_users/service.py +1054 -0
  527. app/modules/firewall/__init__.py +1 -0
  528. app/modules/firewall/api.py +78 -0
  529. app/modules/firewall/repository.py +51 -0
  530. app/modules/firewall/schemas.py +28 -0
  531. app/modules/firewall/service.py +93 -0
  532. app/modules/fleet/__init__.py +1 -0
  533. app/modules/fleet/api.py +177 -0
  534. app/modules/fleet/mappers.py +56 -0
  535. app/modules/fleet/observability.py +372 -0
  536. app/modules/fleet/schemas.py +119 -0
  537. app/modules/health/__init__.py +0 -0
  538. app/modules/health/api.py +246 -0
  539. app/modules/health/schemas.py +27 -0
  540. app/modules/limit_warmup/__init__.py +1 -0
  541. app/modules/limit_warmup/repository.py +159 -0
  542. app/modules/limit_warmup/service.py +1001 -0
  543. app/modules/model_sources/__init__.py +1 -0
  544. app/modules/model_sources/api.py +99 -0
  545. app/modules/model_sources/catalog.py +307 -0
  546. app/modules/model_sources/forwarding.py +1435 -0
  547. app/modules/model_sources/projection.py +55 -0
  548. app/modules/model_sources/repository.py +201 -0
  549. app/modules/model_sources/schemas.py +79 -0
  550. app/modules/model_sources/selection.py +166 -0
  551. app/modules/model_sources/service.py +236 -0
  552. app/modules/oauth/__init__.py +0 -0
  553. app/modules/oauth/api.py +101 -0
  554. app/modules/oauth/repository.py +238 -0
  555. app/modules/oauth/schemas.py +49 -0
  556. app/modules/oauth/service.py +1196 -0
  557. app/modules/oauth/templates/oauth_success.html +122 -0
  558. app/modules/proxy/__init__.py +0 -0
  559. app/modules/proxy/_load_balancer/__init__.py +1 -0
  560. app/modules/proxy/_load_balancer/error_rate.py +116 -0
  561. app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
  562. app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
  563. app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
  564. app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
  565. app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
  566. app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
  567. app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
  568. app/modules/proxy/_load_balancer/tunables.py +97 -0
  569. app/modules/proxy/_load_balancer/types.py +96 -0
  570. app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
  571. app/modules/proxy/_service/__init__.py +6 -0
  572. app/modules/proxy/_service/api_key_usage.py +745 -0
  573. app/modules/proxy/_service/clock_budget.py +38 -0
  574. app/modules/proxy/_service/codex_control.py +611 -0
  575. app/modules/proxy/_service/compact.py +2211 -0
  576. app/modules/proxy/_service/file_ops.py +749 -0
  577. app/modules/proxy/_service/http_bridge/__init__.py +3 -0
  578. app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
  579. app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
  580. app/modules/proxy/_service/http_bridge/activity.py +178 -0
  581. app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
  582. app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
  583. app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
  584. app/modules/proxy/_service/http_bridge/protocol.py +167 -0
  585. app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
  586. app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
  587. app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
  588. app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
  589. app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
  590. app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
  591. app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
  592. app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
  593. app/modules/proxy/_service/observability.py +410 -0
  594. app/modules/proxy/_service/rate_limit.py +361 -0
  595. app/modules/proxy/_service/realtime_live.py +655 -0
  596. app/modules/proxy/_service/refresh.py +154 -0
  597. app/modules/proxy/_service/request_log.py +585 -0
  598. app/modules/proxy/_service/response_create.py +930 -0
  599. app/modules/proxy/_service/streaming/__init__.py +3 -0
  600. app/modules/proxy/_service/streaming/helpers.py +1317 -0
  601. app/modules/proxy/_service/streaming/mixin.py +1100 -0
  602. app/modules/proxy/_service/streaming/protocol.py +32 -0
  603. app/modules/proxy/_service/streaming/retry.py +3762 -0
  604. app/modules/proxy/_service/support.py +2122 -0
  605. app/modules/proxy/_service/transcribe.py +434 -0
  606. app/modules/proxy/_service/warmup.py +483 -0
  607. app/modules/proxy/_service/websocket/__init__.py +3 -0
  608. app/modules/proxy/_service/websocket/helpers.py +2360 -0
  609. app/modules/proxy/_service/websocket/mixin.py +7285 -0
  610. app/modules/proxy/_service/websocket/protocol.py +74 -0
  611. app/modules/proxy/account_cache.py +229 -0
  612. app/modules/proxy/account_eligibility.py +87 -0
  613. app/modules/proxy/additional_model_limits.py +40 -0
  614. app/modules/proxy/affinity.py +830 -0
  615. app/modules/proxy/affinity_observation.py +101 -0
  616. app/modules/proxy/api.py +10254 -0
  617. app/modules/proxy/api_key_usage.py +113 -0
  618. app/modules/proxy/cap_partitioning.py +280 -0
  619. app/modules/proxy/capability_lineage.py +65 -0
  620. app/modules/proxy/capability_lineage_repository.py +82 -0
  621. app/modules/proxy/capability_routing.py +253 -0
  622. app/modules/proxy/continuity.py +112 -0
  623. app/modules/proxy/downstream_delivery.py +277 -0
  624. app/modules/proxy/durable_bridge_coordinator.py +1217 -0
  625. app/modules/proxy/durable_bridge_repository.py +4516 -0
  626. app/modules/proxy/durable_bridge_runtime.py +9 -0
  627. app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
  628. app/modules/proxy/fair_share.py +131 -0
  629. app/modules/proxy/file_pin_repository.py +210 -0
  630. app/modules/proxy/helpers.py +439 -0
  631. app/modules/proxy/http_bridge_event_batcher.py +613 -0
  632. app/modules/proxy/http_bridge_forwarding.py +797 -0
  633. app/modules/proxy/http_continuation.py +76 -0
  634. app/modules/proxy/images_observability.py +62 -0
  635. app/modules/proxy/images_service.py +1054 -0
  636. app/modules/proxy/load_balancer.py +3016 -0
  637. app/modules/proxy/rate_limit_cache.py +46 -0
  638. app/modules/proxy/replay_safety.py +1063 -0
  639. app/modules/proxy/repo_bundle.py +31 -0
  640. app/modules/proxy/request_policy.py +1054 -0
  641. app/modules/proxy/ring_membership.py +251 -0
  642. app/modules/proxy/schemas.py +345 -0
  643. app/modules/proxy/selection_errors.py +41 -0
  644. app/modules/proxy/service.py +2600 -0
  645. app/modules/proxy/source_admission.py +125 -0
  646. app/modules/proxy/source_dispatch.py +999 -0
  647. app/modules/proxy/sticky_repository.py +831 -0
  648. app/modules/proxy/thread_anchors.py +491 -0
  649. app/modules/proxy/tool_call_dedupe.py +802 -0
  650. app/modules/proxy/types.py +54 -0
  651. app/modules/proxy/work_admission.py +121 -0
  652. app/modules/quota_planner/__init__.py +1 -0
  653. app/modules/quota_planner/api.py +276 -0
  654. app/modules/quota_planner/logic.py +743 -0
  655. app/modules/quota_planner/repository.py +722 -0
  656. app/modules/quota_planner/scheduler.py +212 -0
  657. app/modules/quota_planner/schemas.py +102 -0
  658. app/modules/quota_planner/warmup.py +693 -0
  659. app/modules/rate_limit_reset_credits/__init__.py +0 -0
  660. app/modules/rate_limit_reset_credits/api.py +601 -0
  661. app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
  662. app/modules/rate_limit_reset_credits/store.py +114 -0
  663. app/modules/reports/__init__.py +1 -0
  664. app/modules/reports/api.py +134 -0
  665. app/modules/reports/cache.py +74 -0
  666. app/modules/reports/filters.py +27 -0
  667. app/modules/reports/repository.py +525 -0
  668. app/modules/reports/rollup.py +150 -0
  669. app/modules/reports/rollup_read.py +148 -0
  670. app/modules/reports/schemas.py +123 -0
  671. app/modules/reports/service.py +287 -0
  672. app/modules/reports/thread_identity.py +256 -0
  673. app/modules/request_logs/__init__.py +0 -0
  674. app/modules/request_logs/api.py +224 -0
  675. app/modules/request_logs/cost_backfill.py +154 -0
  676. app/modules/request_logs/mappers.py +97 -0
  677. app/modules/request_logs/repository.py +1765 -0
  678. app/modules/request_logs/schemas.py +145 -0
  679. app/modules/request_logs/service.py +348 -0
  680. app/modules/role_mappings/__init__.py +0 -0
  681. app/modules/role_mappings/api.py +206 -0
  682. app/modules/role_mappings/matching.py +55 -0
  683. app/modules/role_mappings/repository.py +106 -0
  684. app/modules/role_mappings/schemas.py +67 -0
  685. app/modules/role_mappings/service.py +334 -0
  686. app/modules/runtime/__init__.py +1 -0
  687. app/modules/runtime/api.py +18 -0
  688. app/modules/runtime/schemas.py +14 -0
  689. app/modules/runtime/service.py +218 -0
  690. app/modules/scim/__init__.py +1 -0
  691. app/modules/scim/api.py +194 -0
  692. app/modules/scim/dependencies.py +112 -0
  693. app/modules/scim/errors.py +77 -0
  694. app/modules/scim/management_api.py +156 -0
  695. app/modules/scim/repository.py +212 -0
  696. app/modules/scim/schemas.py +145 -0
  697. app/modules/scim/service.py +786 -0
  698. app/modules/scim/tokens.py +43 -0
  699. app/modules/settings/__init__.py +1 -0
  700. app/modules/settings/api.py +1677 -0
  701. app/modules/settings/repository.py +606 -0
  702. app/modules/settings/schemas.py +496 -0
  703. app/modules/settings/service.py +769 -0
  704. app/modules/shared/__init__.py +0 -0
  705. app/modules/shared/schemas.py +22 -0
  706. app/modules/sticky_sessions/__init__.py +1 -0
  707. app/modules/sticky_sessions/api.py +118 -0
  708. app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
  709. app/modules/sticky_sessions/schemas.py +78 -0
  710. app/modules/sticky_sessions/service.py +196 -0
  711. app/modules/telemetry/__init__.py +5 -0
  712. app/modules/telemetry/api.py +115 -0
  713. app/modules/telemetry/clients.py +62 -0
  714. app/modules/telemetry/consent.py +115 -0
  715. app/modules/telemetry/scheduler.py +85 -0
  716. app/modules/telemetry/schemas.py +163 -0
  717. app/modules/telemetry/sender.py +221 -0
  718. app/modules/telemetry/snapshot.py +490 -0
  719. app/modules/usage/__init__.py +0 -0
  720. app/modules/usage/additional_quota_keys.py +379 -0
  721. app/modules/usage/api.py +45 -0
  722. app/modules/usage/background_repository.py +147 -0
  723. app/modules/usage/builders.py +559 -0
  724. app/modules/usage/depletion_service.py +277 -0
  725. app/modules/usage/live_ingest.py +356 -0
  726. app/modules/usage/mappers.py +29 -0
  727. app/modules/usage/plan_downgrade_observations.py +488 -0
  728. app/modules/usage/repository.py +1890 -0
  729. app/modules/usage/schemas.py +68 -0
  730. app/modules/usage/service.py +102 -0
  731. app/modules/usage/updater.py +1488 -0
  732. app/static/assets/Trans-Cj-J9mVA.js +1 -0
  733. app/static/assets/access-page-DHzZiXZi.js +1 -0
  734. app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
  735. app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
  736. app/static/assets/account-status-DIsU5sUS.js +1 -0
  737. app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
  738. app/static/assets/accounts-page-Bbai8ift.js +3 -0
  739. app/static/assets/api-Dwa7XxPe.js +1 -0
  740. app/static/assets/api-client-dugX1Pti.js +55 -0
  741. app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
  742. app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
  743. app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
  744. app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
  745. app/static/assets/apis-page-BvXz9c6i.js +2 -0
  746. app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
  747. app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
  748. app/static/assets/automations-page-DQgNAJOR.js +3 -0
  749. app/static/assets/badge-DqMb9es0.js +1 -0
  750. app/static/assets/blur-email-HEvKagG2.js +1 -0
  751. app/static/assets/button-BjVmmiGW.js +1 -0
  752. app/static/assets/checkbox-apsl9vzk.js +1 -0
  753. app/static/assets/chevron-left-1d_pHVa5.js +1 -0
  754. app/static/assets/circle-alert-BS0xd_o1.js +1 -0
  755. app/static/assets/clsx-DB0hHKMi.js +1 -0
  756. app/static/assets/colors-BBy8GEtr.js +1 -0
  757. app/static/assets/copy-button-D7nAvOKR.js +1 -0
  758. app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
  759. app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
  760. app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
  761. app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
  762. app/static/assets/donut-chart-Bxc69cra.js +1 -0
  763. app/static/assets/download-kQteIcsB.js +1 -0
  764. app/static/assets/ellipsis-B_5OgIre.js +1 -0
  765. app/static/assets/empty-state-JcA7tc8m.js +1 -0
  766. app/static/assets/es2015-bMH8YKZT.js +41 -0
  767. app/static/assets/es6-B264ni6W.js +69 -0
  768. app/static/assets/external-link-UVAOgMQ0.js +1 -0
  769. app/static/assets/i18n-BWqZ9Ua2.js +1 -0
  770. app/static/assets/inbox-B_RM2s1J.js +1 -0
  771. app/static/assets/index-DIpd264Z.js +23 -0
  772. app/static/assets/index-DMeLKRJx.css +2 -0
  773. app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
  774. app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
  775. app/static/assets/iso-Bpe5Ihbb.js +1 -0
  776. app/static/assets/label-CGaLmLNf.js +1 -0
  777. app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
  778. app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
  779. app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
  780. app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
  781. app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
  782. app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
  783. app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
  784. app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
  785. app/static/assets/payload-CIYnGZz9.js +1 -0
  786. app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
  787. app/static/assets/report-chart-card-BXDliwcc.js +1 -0
  788. app/static/assets/reports-page-CGCJeMHG.js +3 -0
  789. app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
  790. app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
  791. app/static/assets/search-HfhqwERt.js +1 -0
  792. app/static/assets/select-Cl33bz-g.js +1 -0
  793. app/static/assets/settings-page-mZj01HYl.js +2 -0
  794. app/static/assets/shim-CZwyM4Q7.js +1 -0
  795. app/static/assets/skeleton-C_EUc4Rp.js +1 -0
  796. app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
  797. app/static/assets/status-badge-CHHA5X6c.js +1 -0
  798. app/static/assets/switch-CzxgokcM.js +1 -0
  799. app/static/assets/table-DXaNSQ0G.js +1 -0
  800. app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
  801. app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
  802. app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
  803. app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
  804. app/static/assets/trash-D3KyWr1h.js +1 -0
  805. app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
  806. app/static/assets/use-accounts-PWlVGw3z.js +1 -0
  807. app/static/assets/use-auth-CuDlW3iC.js +1 -0
  808. app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
  809. app/static/assets/use-model-sources-Bul24JBu.js +1 -0
  810. app/static/assets/use-models-DUCPUuZg.js +1 -0
  811. app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
  812. app/static/assets/useTranslation--56nHhlH.js +1 -0
  813. app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
  814. app/static/assets/users-uIC3TIlE.js +1 -0
  815. app/static/assets/vendor-query-_bHZuxEx.js +1 -0
  816. app/static/assets/vendor-react-BVL0YZPW.js +9 -0
  817. app/static/favicon.svg +10 -0
  818. app/static/fonts/GeistSans-Variable.woff2 +0 -0
  819. app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
  820. app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
  821. app/static/index.html +29 -0
  822. codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
  823. codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
  824. codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
  825. codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
  826. codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
  827. config/additional_quota_registry.json +11 -0
@@ -0,0 +1,4538 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import logging
5
+ from collections.abc import Awaitable, Callable, Iterable
6
+ from dataclasses import replace
7
+ from typing import Any, TypeVar, cast
8
+
9
+ import anyio
10
+
11
+ from app.core.balancer.types import UpstreamError
12
+ from app.core.clients.files import create_file as core_create_file # noqa: F401
13
+ from app.core.clients.files import finalize_file as core_finalize_file # noqa: F401
14
+ from app.core.clients.proxy import CodexControlResponse as CodexControlResponse
15
+ from app.core.clients.proxy import ( # noqa: F401
16
+ ImageFetchSession,
17
+ ProxyResponseError,
18
+ UpstreamProxyRouteTrace,
19
+ _as_image_fetch_session,
20
+ _inline_content_images,
21
+ _inline_input_image_urls,
22
+ _ws_transport_payload_budget_bytes,
23
+ filter_inbound_headers,
24
+ pop_compact_timeout_overrides,
25
+ pop_stream_timeout_overrides,
26
+ pop_transcribe_timeout_overrides,
27
+ push_compact_timeout_overrides,
28
+ push_stream_timeout_overrides,
29
+ push_transcribe_timeout_overrides,
30
+ )
31
+ from app.core.clients.proxy import codex_control_request as core_codex_control_request # noqa: F401
32
+ from app.core.clients.proxy import compact_responses as core_compact_responses # noqa: F401
33
+ from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401
34
+ from app.core.clients.proxy_websocket import (
35
+ UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE,
36
+ UpstreamWebSocketMessage,
37
+ UpstreamWebSocketTransportError,
38
+ is_account_neutral_websocket_error_code,
39
+ )
40
+ from app.core.clock import REAL_CLOCK, REAL_SCHEDULER, Clock, Scheduler, clock_for, scheduler_for
41
+ from app.core.errors import response_failed_event
42
+ from app.core.openai.models import OpenAIEvent
43
+ from app.core.openai.parsing import (
44
+ _LIFECYCLE_EVENT_TYPES,
45
+ classify_event_type,
46
+ parse_sse_event_payload,
47
+ )
48
+ from app.core.types import JsonValue
49
+ from app.core.usage.live_hub import publish_live_usage
50
+ from app.core.usage.live_snapshots import EVENT_MARKER, parse_rate_limit_event_text
51
+ from app.core.utils.request_id import reset_request_id, set_request_id
52
+ from app.core.utils.shared_future import wait_on_shared_future
53
+ from app.core.utils.sse import format_sse_event, format_sse_event_from_text, parse_sse_data_json_text
54
+ from app.db.models import Account
55
+ from app.modules.proxy._service.api_key_usage import (
56
+ _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS,
57
+ )
58
+ from app.modules.proxy._service.compact import (
59
+ _sticky_key_for_compact_request as _sticky_key_for_compact_request,
60
+ )
61
+ from app.modules.proxy._service.compact import (
62
+ _sticky_key_from_compact_payload as _sticky_key_from_compact_payload,
63
+ )
64
+ from app.modules.proxy._service.http_bridge import helpers as _http_bridge_helpers
65
+ from app.modules.proxy._service.http_bridge.accepted_replay import (
66
+ _http_bridge_accepted_anchored_replay_candidate,
67
+ _http_bridge_accepted_capacity_retry_allowed,
68
+ _stage_websocket_request_state_for_replay,
69
+ )
70
+ from app.modules.proxy._service.http_bridge.helpers import (
71
+ _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
72
+ _await_task_deferring_cancellation,
73
+ _forget_http_bridge_denied_anchor_fence,
74
+ _http_bridge_abandonment_may_settle_circuit,
75
+ _http_bridge_denied_anchor_fence_current_map,
76
+ _http_bridge_denied_anchor_fence_entry,
77
+ _http_bridge_durable_lease_ttl_seconds,
78
+ _http_bridge_event_proves_upstream_liveness,
79
+ _http_bridge_eventless_precreated_deadline,
80
+ _http_bridge_request_budget_seconds,
81
+ _http_bridge_request_counts_against_queue,
82
+ _http_bridge_request_state_holds_safe_replay,
83
+ _http_bridge_retry_circuit_attempt_selection_for_pending_requests,
84
+ _log_http_bridge_event,
85
+ _normalize_http_bridge_error_event,
86
+ _record_http_bridge_denied_anchor_fence,
87
+ _record_http_bridge_stuck_retire,
88
+ _record_http_bridge_unmatched_upstream_liveness,
89
+ _schedule_http_bridge_background_cleanup,
90
+ )
91
+ from app.modules.proxy._service.http_bridge.quarantine import (
92
+ _clear_http_bridge_quarantine,
93
+ _http_bridge_quarantine_clear_fence,
94
+ _record_http_bridge_quarantine_eventless_timeout,
95
+ _record_http_bridge_quarantine_wedged_pending,
96
+ )
97
+ from app.modules.proxy._service.http_bridge.retry_circuit import (
98
+ _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
99
+ _POISON_ANCHOR_CAPTURE_UNAVAILABLE,
100
+ _http_bridge_anchor_poison_detail,
101
+ )
102
+ from app.modules.proxy._service.http_bridge.service_stubs import (
103
+ _assign_websocket_response_id,
104
+ _await_cancelled_task,
105
+ _build_stream_incomplete_terminal_event_for_request,
106
+ _classify_upstream_close,
107
+ _find_websocket_request_state_by_response_id,
108
+ _http_error_status_from_payload,
109
+ _is_account_neutral_transport_drop,
110
+ _is_missing_tool_output_error,
111
+ _is_previous_response_not_found_error,
112
+ _is_security_work_authorization_required_error,
113
+ _match_websocket_request_state_for_anonymous_event,
114
+ _matching_websocket_request_states_for_missing_tool_output_error,
115
+ _matching_websocket_request_states_for_previous_response_error,
116
+ _maybe_rewrite_websocket_previous_response_not_found_event,
117
+ _pop_matching_websocket_request_states,
118
+ _pop_terminal_websocket_request_state,
119
+ _prepare_websocket_request_state_for_account_switch,
120
+ _previous_response_id_from_not_found_message,
121
+ _release_websocket_response_create_gate,
122
+ _response_output_item_done_tool_call,
123
+ _rewrite_websocket_continuity_corruption_event,
124
+ _rewrite_websocket_downstream_response_id,
125
+ _rewrite_websocket_previous_response_owner_unavailable_event,
126
+ _rewrite_websocket_suppressed_duplicate_tool_call_completion_event,
127
+ _security_work_advisory_event,
128
+ _service_get_settings,
129
+ _service_tier_from_event_payload,
130
+ _upstream_websocket_disconnect_message,
131
+ _websocket_auth_request_can_switch_account,
132
+ _websocket_downstream_response_id,
133
+ _websocket_event_error_code,
134
+ _websocket_event_error_message,
135
+ _websocket_event_error_param,
136
+ _websocket_event_error_type,
137
+ _websocket_owner_pinned_quota_error_code,
138
+ _websocket_precreated_auth_error_code,
139
+ _websocket_precreated_retry_error_code,
140
+ _websocket_response_id,
141
+ )
142
+ from app.modules.proxy._service.observability import (
143
+ _hash_identifier as _hash_identifier,
144
+ )
145
+ from app.modules.proxy._service.observability import (
146
+ _hash_identifier_or_none as _hash_identifier_or_none,
147
+ )
148
+ from app.modules.proxy._service.observability import (
149
+ _interesting_header_keys as _interesting_header_keys,
150
+ )
151
+ from app.modules.proxy._service.observability import (
152
+ _tools_hash as _tools_hash,
153
+ )
154
+ from app.modules.proxy._service.observability import (
155
+ _truncate_identifier as _truncate_identifier,
156
+ )
157
+ from app.modules.proxy._service.support import (
158
+ _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE,
159
+ _ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS,
160
+ _ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS,
161
+ _HARD_HTTP_BRIDGE_AFFINITY_KINDS, # noqa: F401
162
+ _MODEL_OUTPUT_EVENT_TYPES,
163
+ _PENDING_TOOL_CALL_ITEM_TYPES,
164
+ _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401
165
+ _account_capacity_wait_payload,
166
+ _clear_websocket_deferred_reasoning_downstream_texts,
167
+ _clear_websocket_precreated_replay_fallback,
168
+ _clear_websocket_request_error_overrides,
169
+ _DeferredKeyedStreamHealthPenalty,
170
+ _HTTPBridgeCompletedDeliveryScope,
171
+ _HTTPBridgeRetryCircuitAttemptSelection,
172
+ _HTTPBridgeSession,
173
+ _mark_response_create_attempt_observed,
174
+ _pop_websocket_deferred_reasoning_downstream_texts,
175
+ _record_response_event,
176
+ _signal_propagated_capacity_startup_ready,
177
+ _signal_propagated_capacity_startup_wait,
178
+ _websocket_request_can_replay_before_visible_output,
179
+ _websocket_should_defer_reasoning_prelude,
180
+ _WebSocketReceiveTimeout,
181
+ _WebSocketRequestState,
182
+ )
183
+ from app.modules.proxy._service.support import (
184
+ _websocket_route_log_kwargs as _websocket_route_log_kwargs,
185
+ )
186
+ from app.modules.proxy._service.warmup import (
187
+ WarmupExecutionData as WarmupExecutionData,
188
+ )
189
+ from app.modules.proxy._service.warmup import (
190
+ WarmupFailedAccountData as WarmupFailedAccountData,
191
+ )
192
+ from app.modules.proxy._service.warmup import (
193
+ WarmupSkippedAccountData as WarmupSkippedAccountData,
194
+ )
195
+ from app.modules.proxy._service.warmup import (
196
+ WarmupSubmittedAccountData as WarmupSubmittedAccountData,
197
+ )
198
+ from app.modules.proxy._service.warmup import (
199
+ _is_warmup_usage_eligible as _is_warmup_usage_eligible,
200
+ )
201
+ from app.modules.proxy._service.warmup import (
202
+ _materialize_warmup_account as _materialize_warmup_account,
203
+ )
204
+ from app.modules.proxy._service.warmup import (
205
+ _snapshot_warmup_account as _snapshot_warmup_account,
206
+ )
207
+ from app.modules.proxy._service.warmup import (
208
+ _WarmupAccountSnapshot as _WarmupAccountSnapshot,
209
+ )
210
+ from app.modules.proxy._service.warmup import (
211
+ _WarmupSubmitResult as _WarmupSubmitResult,
212
+ )
213
+ from app.modules.proxy._service.warmup import (
214
+ _WarmupUsageSnapshot as _WarmupUsageSnapshot,
215
+ )
216
+ from app.modules.proxy.affinity import (
217
+ _extract_model_class,
218
+ )
219
+ from app.modules.proxy.continuity import is_http_bridge_account_neutral_replay
220
+ from app.modules.proxy.helpers import (
221
+ _normalize_error_code,
222
+ is_upstream_model_capacity_error,
223
+ )
224
+ from app.modules.proxy.tool_call_dedupe import (
225
+ mark_duplicate_tool_call_downstream_event,
226
+ rewrite_parallel_tool_call_text,
227
+ )
228
+ from app.modules.proxy.tool_call_dedupe import (
229
+ response_id_from_payload as tool_call_response_id_from_payload,
230
+ )
231
+
232
+ logger = logging.getLogger("app.modules.proxy.service")
233
+
234
+ _HTTP_BRIDGE_RECOVERY_SETTLEMENT_RETRY_DELAYS = (
235
+ 0.25,
236
+ 0.5,
237
+ 1.0,
238
+ 2.0,
239
+ 4.0,
240
+ 8.0,
241
+ 15.0,
242
+ 30.0,
243
+ 60.0,
244
+ 120.0,
245
+ )
246
+ _HTTP_BRIDGE_RECOVERY_SETTLEMENT_LEASE_REFRESH_INTERVAL_SECONDS = 10.0
247
+ _HTTP_BRIDGE_DENIED_ANCHOR_CLEAR_RETRY_DELAYS = (
248
+ 0.25,
249
+ 0.5,
250
+ 1.0,
251
+ 2.0,
252
+ 4.0,
253
+ 8.0,
254
+ 15.0,
255
+ 30.0,
256
+ 60.0,
257
+ )
258
+ # A single missing response.created is not proof that an account is bad: the
259
+ # upstream may have accepted the request while the transport was silent. Only
260
+ # repeated failures on separate bridge retirements are allowed to influence
261
+ # account routing, and the signal expires quickly so a transient upstream
262
+ # incident does not permanently drain an account.
263
+ _HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS = 300.0
264
+ _HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD = 3
265
+
266
+
267
+ async def _record_http_bridge_account_timeout_signal(
268
+ service: Any,
269
+ session: "_HTTPBridgeSession",
270
+ *,
271
+ detail: str = _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
272
+ ) -> None:
273
+ """Drain an account after repeated eventless upstream failures.
274
+
275
+ This is deliberately separate from the per-session retry circuit. A
276
+ timeout or abrupt eventless transport drop cannot be replayed safely for a
277
+ continuity-bound turn, but three independent eventless failures are enough
278
+ evidence to keep *new* turns away from that account until its normal
279
+ health probe succeeds.
280
+ """
281
+
282
+ account_id = session.account.id
283
+ now = clock_for(service).monotonic()
284
+ async with service._http_bridge_account_timeout_lock:
285
+ failures = service._http_bridge_account_timeout_failures.setdefault(account_id, [])
286
+ failures[:] = [
287
+ timestamp for timestamp in failures if now - timestamp < _HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS
288
+ ]
289
+ failures.append(now)
290
+ if len(failures) < _HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD:
291
+ return
292
+ # Start a fresh evidence window after applying one health penalty. A
293
+ # continuously failing account should be re-evaluated by normal
294
+ # health-tier logic, not receive an unbounded error-count increase from
295
+ # every pending request on one broken socket.
296
+ failures.clear()
297
+
298
+ try:
299
+ # Health-tier draining starts at two transient errors. Apply exactly
300
+ # that minimum penalty so one threshold event actually removes the
301
+ # account from normal routing without over-counting the incident.
302
+ await service._load_balancer.record_errors(session.account, 2)
303
+ except Exception:
304
+ logger.warning(
305
+ "Failed to record repeated HTTP bridge account timeout account_id=%s",
306
+ account_id,
307
+ exc_info=True,
308
+ )
309
+ else:
310
+ logger.warning(
311
+ "HTTP bridge account temporarily drained after repeated eventless upstream failures "
312
+ "account_id=%s detail=%s threshold=%s window_seconds=%.0f",
313
+ account_id,
314
+ detail,
315
+ _HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD,
316
+ _HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS,
317
+ )
318
+
319
+
320
+ async def _update_http_bridge_operation_state(
321
+ service: Any,
322
+ session: "_HTTPBridgeSession",
323
+ request_state: Any,
324
+ *,
325
+ state: str,
326
+ response_id: str | None = None,
327
+ ) -> None:
328
+ """Persist operation outcome without allowing journaling to break streaming."""
329
+ operation_id = getattr(request_state, "operation_id", None)
330
+ session_id = getattr(session, "durable_session_id", None)
331
+ owner_epoch = getattr(session, "durable_owner_epoch", None)
332
+ update_operation = getattr(getattr(service, "_durable_bridge", None), "update_operation", None)
333
+ if not operation_id or session_id is None or owner_epoch is None or not callable(update_operation):
334
+ return
335
+ try:
336
+ marked = await update_operation(
337
+ operation_id=operation_id,
338
+ session_id=session_id,
339
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
340
+ owner_epoch=owner_epoch,
341
+ state=state,
342
+ response_id=response_id,
343
+ )
344
+ if marked and response_id is not None:
345
+ request_state.operation_persisted_response_id = response_id
346
+ if not marked:
347
+ logger.info(
348
+ "HTTP bridge operation outcome owner fence rejected operation_id=%s state=%s",
349
+ operation_id,
350
+ state,
351
+ )
352
+ except Exception:
353
+ logger.warning(
354
+ "Failed to persist HTTP bridge operation outcome operation_id=%s state=%s",
355
+ operation_id,
356
+ state,
357
+ exc_info=True,
358
+ )
359
+
360
+
361
+ def _http_bridge_operation_state_for_event(event_type: str | None) -> str | None:
362
+ return {
363
+ "response.created": "acknowledged",
364
+ "response.completed": "completed",
365
+ "response.incomplete": "incomplete",
366
+ "response.failed": "failed",
367
+ "error": "failed",
368
+ }.get(event_type)
369
+
370
+
371
+ async def _persist_http_bridge_operation_event(
372
+ service: Any,
373
+ session: "_HTTPBridgeSession",
374
+ request_state: Any,
375
+ event_block: str,
376
+ *,
377
+ terminal: bool = False,
378
+ terminal_state: str | None = None,
379
+ terminal_event_queue: Any | None = None,
380
+ terminal_delivery_scope: _HTTPBridgeCompletedDeliveryScope | None = None,
381
+ terminal_append_barrier: Callable[[], Awaitable[None]] | None = None,
382
+ terminal_delivery_barrier: Callable[[], Awaitable[None]] | None = None,
383
+ ) -> bool:
384
+ """Spool one downstream-visible SSE block for reconnect replay.
385
+
386
+ Return whether terminal failure handling already queued the block.
387
+ """
388
+ operation_id = getattr(request_state, "operation_id", None)
389
+ session_id = getattr(session, "durable_session_id", None)
390
+ owner_epoch = getattr(session, "durable_owner_epoch", None)
391
+ batcher_enqueue = getattr(getattr(service, "_http_bridge_operation_event_batcher", None), "enqueue", None)
392
+ append_event = getattr(getattr(service, "_durable_bridge", None), "append_operation_event", None)
393
+ if not operation_id or session_id is None or owner_epoch is None:
394
+ return False
395
+ append_barrier_task: asyncio.Task[None] | None = None
396
+ delivery_barrier_task: asyncio.Task[None] | None = None
397
+ terminal_enqueued = False
398
+ deferred_cancellation: asyncio.CancelledError | None = None
399
+ scheduler = scheduler_for(service)
400
+
401
+ async def release_terminal_append_barrier() -> asyncio.CancelledError | None:
402
+ nonlocal append_barrier_task
403
+ if terminal_append_barrier is None:
404
+ return None
405
+ if append_barrier_task is None:
406
+
407
+ async def await_append_barrier() -> None:
408
+ await terminal_append_barrier()
409
+
410
+ append_barrier_task = scheduler.create_task(
411
+ await_append_barrier(),
412
+ name=f"http-bridge-terminal-append-barrier-{operation_id}",
413
+ )
414
+ _, cancellation = await _await_task_deferring_cancellation(append_barrier_task)
415
+ return cancellation
416
+
417
+ async def release_terminal_delivery_barrier() -> asyncio.CancelledError | None:
418
+ nonlocal delivery_barrier_task
419
+ if terminal_delivery_barrier is None:
420
+ return None
421
+ if delivery_barrier_task is None:
422
+
423
+ async def await_delivery_barrier() -> None:
424
+ await terminal_delivery_barrier()
425
+
426
+ delivery_barrier_task = scheduler.create_task(
427
+ await_delivery_barrier(),
428
+ name=f"http-bridge-terminal-delivery-barrier-{operation_id}",
429
+ )
430
+ _, cancellation = await _await_task_deferring_cancellation(delivery_barrier_task)
431
+ return cancellation
432
+
433
+ async def enqueue_terminal_delivery() -> bool:
434
+ if terminal_event_queue is None:
435
+ return False
436
+ await terminal_event_queue.put(event_block)
437
+ await terminal_event_queue.put(None)
438
+ if terminal_delivery_scope is not None:
439
+ async with session.pending_lock:
440
+ terminal_delivery_scope.terminal_enqueued = True
441
+ return True
442
+
443
+ async def enqueue_terminal_delivery_deferring_cancellation() -> tuple[bool, asyncio.CancelledError | None]:
444
+ delivery_task = scheduler.create_task(
445
+ enqueue_terminal_delivery(),
446
+ name=f"http-bridge-terminal-delivery-{operation_id}",
447
+ )
448
+ return await _await_task_deferring_cancellation(delivery_task)
449
+
450
+ try:
451
+ batcher = getattr(service, "_http_bridge_operation_event_batcher", None)
452
+ append_terminal_batch = getattr(batcher, "append_terminal_event", None)
453
+ if terminal and terminal_state is not None and callable(append_terminal_batch):
454
+ instance_id = _service_get_settings().http_responses_session_bridge_instance_id
455
+ expected_response_ids = tuple(
456
+ dict.fromkeys(
457
+ response_identity
458
+ for response_identity in (
459
+ request_state.response_id,
460
+ getattr(request_state, "operation_persisted_response_id", None),
461
+ request_state.replay_downstream_response_id,
462
+ )
463
+ if response_identity is not None
464
+ )
465
+ )
466
+ expected_response_id = expected_response_ids[0] if expected_response_ids else None
467
+ alternate_expected_response_id = expected_response_ids[1] if len(expected_response_ids) > 1 else None
468
+ response_id = _websocket_downstream_response_id(request_state)
469
+
470
+ async def append_terminal_batch_capturing_error() -> tuple[Any | None, Exception | None]:
471
+ try:
472
+ return (
473
+ await append_terminal_batch(
474
+ operation_id=operation_id,
475
+ session_id=session_id,
476
+ instance_id=instance_id,
477
+ owner_epoch=owner_epoch,
478
+ event_text=event_block,
479
+ max_bytes=int(
480
+ getattr(
481
+ _service_get_settings(),
482
+ "http_responses_session_bridge_operation_event_spool_max_bytes",
483
+ 2 * 1024 * 1024,
484
+ )
485
+ ),
486
+ state=terminal_state,
487
+ response_id=response_id,
488
+ ),
489
+ None,
490
+ )
491
+ except Exception as exc:
492
+ # Return the optional-spool failure as data so the canonical
493
+ # defer helper can also return a caller-cancellation marker.
494
+ return None, exc
495
+
496
+ append_task = scheduler.create_task(
497
+ append_terminal_batch_capturing_error(),
498
+ name=f"http-bridge-terminal-append-{operation_id}",
499
+ )
500
+ # Counted grouped siblings wait on this barrier; release it even when
501
+ # append raises so gather(..., return_exceptions=True) cannot strand them.
502
+ try:
503
+ (append_result, append_error), deferred_cancellation = await _await_task_deferring_cancellation(
504
+ append_task
505
+ )
506
+ finally:
507
+ barrier_cancellation = await release_terminal_append_barrier()
508
+ deferred_cancellation = deferred_cancellation or barrier_cancellation
509
+ if append_error is not None:
510
+ raise append_error
511
+ persisted = bool(append_result)
512
+ if not persisted:
513
+ logger.info("HTTP bridge terminal event spool became incomplete operation_id=%s", operation_id)
514
+ settlement_required = bool(getattr(append_result, "settlement_required", False))
515
+ if settlement_required:
516
+ terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation()
517
+ deferred_cancellation = deferred_cancellation or delivery_cancellation
518
+ if terminal_delivery_barrier is not None:
519
+ if not terminal_enqueued:
520
+ terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation()
521
+ deferred_cancellation = deferred_cancellation or delivery_cancellation
522
+ barrier_cancellation = await release_terminal_delivery_barrier()
523
+ deferred_cancellation = deferred_cancellation or barrier_cancellation
524
+ if settlement_required:
525
+ settle_terminal_batch = getattr(batcher, "settle_terminal_event", None)
526
+
527
+ async def settle_terminal_append_failure() -> None:
528
+ if callable(settle_terminal_batch):
529
+ await settle_terminal_batch(
530
+ operation_id=operation_id,
531
+ session_id=session_id,
532
+ instance_id=instance_id,
533
+ owner_epoch=owner_epoch,
534
+ state=terminal_state,
535
+ expected_response_id=expected_response_id,
536
+ alternate_expected_response_id=alternate_expected_response_id,
537
+ response_id=response_id,
538
+ )
539
+ else:
540
+ await _update_http_bridge_operation_state(
541
+ service,
542
+ session,
543
+ request_state,
544
+ state=terminal_state,
545
+ response_id=response_id,
546
+ )
547
+
548
+ settlement_task = scheduler.create_task(
549
+ settle_terminal_append_failure(),
550
+ name=f"http-bridge-terminal-settlement-{operation_id}",
551
+ )
552
+ _, settlement_cancellation = await _await_task_deferring_cancellation(settlement_task)
553
+ deferred_cancellation = deferred_cancellation or settlement_cancellation
554
+ if deferred_cancellation is not None:
555
+ if not terminal_enqueued:
556
+ await enqueue_terminal_delivery_deferring_cancellation()
557
+ raise deferred_cancellation
558
+ return terminal_enqueued
559
+ if callable(batcher_enqueue):
560
+ await batcher_enqueue(
561
+ operation_id=operation_id,
562
+ session_id=session_id,
563
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
564
+ owner_epoch=owner_epoch,
565
+ event_text=event_block,
566
+ terminal=terminal,
567
+ )
568
+ return False
569
+ if not callable(append_event):
570
+ return False
571
+ persisted = await append_event(
572
+ operation_id=operation_id,
573
+ session_id=session_id,
574
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
575
+ owner_epoch=owner_epoch,
576
+ event_text=event_block,
577
+ max_bytes=int(
578
+ getattr(
579
+ _service_get_settings(),
580
+ "http_responses_session_bridge_operation_event_spool_max_bytes",
581
+ 2 * 1024 * 1024,
582
+ )
583
+ ),
584
+ )
585
+ if not persisted:
586
+ logger.info("HTTP bridge operation event spool became incomplete operation_id=%s", operation_id)
587
+ if terminal and terminal_state is not None:
588
+ await _update_http_bridge_operation_state(
589
+ service,
590
+ session,
591
+ request_state,
592
+ state=terminal_state,
593
+ response_id=_websocket_downstream_response_id(request_state),
594
+ )
595
+ return False
596
+ except Exception:
597
+ # The upstream result is still delivered. A reconnect can only replay
598
+ # when every event was durably persisted, so never fail a live stream
599
+ # because the optional spool is unavailable.
600
+ logger.warning("Failed to persist HTTP bridge operation event operation_id=%s", operation_id, exc_info=True)
601
+ barrier_cancellation = await release_terminal_append_barrier()
602
+ deferred_cancellation = deferred_cancellation or barrier_cancellation
603
+ if terminal_event_queue is not None and terminal and not terminal_enqueued:
604
+ try:
605
+ terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation()
606
+ deferred_cancellation = deferred_cancellation or delivery_cancellation
607
+ except Exception:
608
+ logger.debug(
609
+ "Failed to enqueue HTTP bridge terminal after spool error operation_id=%s",
610
+ operation_id,
611
+ exc_info=True,
612
+ )
613
+ barrier_cancellation = await release_terminal_delivery_barrier()
614
+ deferred_cancellation = deferred_cancellation or barrier_cancellation
615
+ if deferred_cancellation is not None:
616
+ raise deferred_cancellation
617
+ return terminal_enqueued
618
+
619
+
620
+ async def _wait_for_http_bridge_recovery_settlement_retry(
621
+ service: Any,
622
+ *,
623
+ session_id: str,
624
+ owner_epoch: int,
625
+ api_key_id: str | None,
626
+ delay_seconds: float,
627
+ ) -> None:
628
+ remaining = max(0.0, delay_seconds)
629
+ while remaining > 0:
630
+ await scheduler_for(service).sleep(
631
+ min(remaining, _HTTP_BRIDGE_RECOVERY_SETTLEMENT_LEASE_REFRESH_INTERVAL_SECONDS)
632
+ )
633
+ remaining -= _HTTP_BRIDGE_RECOVERY_SETTLEMENT_LEASE_REFRESH_INTERVAL_SECONDS
634
+ try:
635
+ await service._durable_bridge.renew_live_session(
636
+ session_id=session_id,
637
+ api_key_id=api_key_id,
638
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
639
+ owner_epoch=owner_epoch,
640
+ lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(),
641
+ )
642
+ except Exception:
643
+ logger.debug("Failed to refresh HTTP bridge lease during settlement backoff", exc_info=True)
644
+
645
+
646
+ async def _retry_http_bridge_recovery_settlement(
647
+ service: Any,
648
+ session: Any,
649
+ *,
650
+ session_id: str,
651
+ api_key_id: str | None,
652
+ instance_id: str,
653
+ owner_epoch: int,
654
+ request_fingerprint: str,
655
+ response_id: str | None,
656
+ release_origin_lease: bool,
657
+ ) -> None:
658
+ """Keep a response-observed journal row fenced until durable settlement succeeds."""
659
+
660
+ for delay_seconds in _HTTP_BRIDGE_RECOVERY_SETTLEMENT_RETRY_DELAYS:
661
+ await _wait_for_http_bridge_recovery_settlement_retry(
662
+ service,
663
+ session_id=session_id,
664
+ owner_epoch=owner_epoch,
665
+ api_key_id=api_key_id,
666
+ delay_seconds=delay_seconds,
667
+ )
668
+ try:
669
+ marked = await service._durable_bridge.mark_recovery_attempt_replayed(
670
+ session_id=session_id,
671
+ api_key_id=api_key_id,
672
+ instance_id=instance_id,
673
+ owner_epoch=owner_epoch,
674
+ request_fingerprint=request_fingerprint,
675
+ response_id=response_id,
676
+ )
677
+ if marked and (release_origin_lease or getattr(session, "closed", False)):
678
+ try:
679
+ await service._durable_bridge.release_live_session(
680
+ session_id=session_id,
681
+ instance_id=instance_id,
682
+ owner_epoch=owner_epoch,
683
+ draining=False,
684
+ )
685
+ except Exception:
686
+ logger.debug("Failed to release HTTP bridge recovery origin lease", exc_info=True)
687
+ if marked:
688
+ return
689
+ logger.warning("HTTP bridge recovery settlement owner fence rejected; retrying")
690
+ except Exception:
691
+ continue
692
+ logger.error(
693
+ "HTTP bridge recovery settlement retry budget exhausted session_id=%s fingerprint=%s",
694
+ _hash_identifier(session_id),
695
+ _hash_identifier(request_fingerprint),
696
+ )
697
+
698
+
699
+ def _schedule_http_bridge_recovery_settlement_retry(
700
+ service: Any,
701
+ session: Any,
702
+ **kwargs: Any,
703
+ ) -> None:
704
+ _schedule_http_bridge_background_cleanup(
705
+ service,
706
+ _retry_http_bridge_recovery_settlement(service, session, **kwargs),
707
+ name=f"http-bridge-recovery-settlement-{_hash_identifier(kwargs['request_fingerprint'])}",
708
+ error_message="HTTP bridge recovery settlement retry failed",
709
+ attribute=("_http_bridge_recovery_session_id", kwargs["session_id"]),
710
+ )
711
+
712
+
713
+ async def _retry_denied_http_bridge_anchor_clear(
714
+ service: Any,
715
+ session: Any,
716
+ *,
717
+ session_id: str | None,
718
+ api_key_id: str | None,
719
+ instance_id: str,
720
+ owner_epoch: int | None,
721
+ response_id: str,
722
+ durable_cleared: bool = False,
723
+ ) -> None:
724
+ """Retry a transient durable clear under the original owner fence."""
725
+ if session_id is None or owner_epoch is None:
726
+ await _retry_denied_http_bridge_anchor_local_cleanup(
727
+ service,
728
+ session,
729
+ response_id=response_id,
730
+ owner_key=f"local:{id(session)}",
731
+ )
732
+ return
733
+ for delay_seconds in _HTTP_BRIDGE_DENIED_ANCHOR_CLEAR_RETRY_DELAYS:
734
+ if session.durable_session_id != session_id or session.durable_owner_epoch != owner_epoch:
735
+ return
736
+ await _wait_for_http_bridge_recovery_settlement_retry(
737
+ service,
738
+ session_id=session_id,
739
+ owner_epoch=owner_epoch,
740
+ api_key_id=api_key_id,
741
+ delay_seconds=delay_seconds,
742
+ )
743
+ if not durable_cleared:
744
+ try:
745
+ cleared = await service._durable_bridge.clear_live_session_response_anchor_if_matches(
746
+ session_id=session_id,
747
+ api_key_id=api_key_id,
748
+ instance_id=instance_id,
749
+ owner_epoch=owner_epoch,
750
+ response_id=response_id,
751
+ )
752
+ except asyncio.CancelledError:
753
+ raise
754
+ except Exception:
755
+ logger.warning("Retrying denied HTTP bridge response-anchor clear after durable failure", exc_info=True)
756
+ continue
757
+ if cleared is None:
758
+ # A clean no-match means the captured owner fence or denied
759
+ # response is already gone. It is a terminal bookkeeping
760
+ # outcome, not a transient durable failure; keep the local
761
+ # alias and denial fence because ownership may have advanced,
762
+ # but do not spend the remaining retry budget on it.
763
+ return
764
+ durable_cleared = True
765
+ # Owner rebinding can happen while the lease-renewal backoff is
766
+ # sleeping. Serialize the final ownership check with rebinders before
767
+ # touching the alias or the process-local denial fence.
768
+ async with session.lifecycle_lock:
769
+ if session.durable_session_id != session_id or session.durable_owner_epoch != owner_epoch:
770
+ return
771
+ try:
772
+ unregister_succeeded = await service._unregister_http_bridge_previous_response_id(
773
+ session,
774
+ response_id,
775
+ expected_durable_session_id=session_id,
776
+ expected_durable_owner_epoch=owner_epoch,
777
+ )
778
+ except asyncio.CancelledError:
779
+ raise
780
+ except Exception:
781
+ logger.warning(
782
+ "Retrying denied HTTP bridge response-alias unregister after cleanup failure",
783
+ exc_info=True,
784
+ )
785
+ continue
786
+ if unregister_succeeded is False:
787
+ return
788
+ if not _forget_http_bridge_denied_anchor_fence(
789
+ service,
790
+ response_id,
791
+ owner_key=session_id,
792
+ owner_epoch=owner_epoch,
793
+ ):
794
+ return
795
+ session.denied_proxy_injected_anchor_ids.discard(response_id)
796
+ session.denied_proxy_injected_anchor_cleanup_pending.discard(response_id)
797
+ session.denied_proxy_injected_anchor_generation += 1
798
+ return
799
+ logger.error(
800
+ "Denied HTTP bridge response-anchor clear retry budget exhausted session_id=%s response_id=%s",
801
+ _hash_identifier(session_id),
802
+ _hash_identifier(response_id),
803
+ )
804
+
805
+
806
+ async def _retry_denied_http_bridge_anchor_local_cleanup(
807
+ service: Any,
808
+ session: Any,
809
+ *,
810
+ response_id: str,
811
+ owner_key: str,
812
+ ) -> None:
813
+ """Retry process-local alias cleanup for a session without a durable owner."""
814
+ for delay_seconds in _HTTP_BRIDGE_DENIED_ANCHOR_CLEAR_RETRY_DELAYS:
815
+ if delay_seconds:
816
+ await scheduler_for(service).sleep(delay_seconds)
817
+ async with session.lifecycle_lock:
818
+ if session.durable_session_id is not None or session.durable_owner_epoch is not None:
819
+ return
820
+ entry = _http_bridge_denied_anchor_fence_entry(service, response_id)
821
+ try:
822
+ unregister_succeeded = await service._unregister_http_bridge_previous_response_id(
823
+ session,
824
+ response_id,
825
+ )
826
+ except asyncio.CancelledError:
827
+ raise
828
+ except Exception:
829
+ logger.warning(
830
+ "Retrying local denied HTTP bridge response-alias unregister after cleanup failure",
831
+ exc_info=True,
832
+ )
833
+ continue
834
+ if unregister_succeeded is False:
835
+ continue
836
+ # The local session can lose its durable identity after a
837
+ # successor has already claimed the same response id. Remove
838
+ # this session's alias even when the process-local fence now
839
+ # belongs to that durable successor; only forget the fence when
840
+ # this local owner still owns it.
841
+ if entry is not None and entry.owner_key == owner_key:
842
+ _forget_http_bridge_denied_anchor_fence(
843
+ service,
844
+ response_id,
845
+ owner_key=owner_key,
846
+ owner_epoch=None,
847
+ )
848
+ session.denied_proxy_injected_anchor_ids.discard(response_id)
849
+ session.denied_proxy_injected_anchor_cleanup_pending.discard(response_id)
850
+ session.denied_proxy_injected_anchor_generation += 1
851
+ return
852
+ logger.error(
853
+ "Denied HTTP bridge local response-anchor cleanup retry budget exhausted response_id=%s",
854
+ _hash_identifier(response_id),
855
+ )
856
+
857
+
858
+ def _schedule_denied_http_bridge_anchor_clear_retry(
859
+ service: Any,
860
+ session: Any,
861
+ *,
862
+ response_id: str,
863
+ session_id: str | None = None,
864
+ api_key_id: str | None = None,
865
+ instance_id: str | None = None,
866
+ owner_epoch: int | None = None,
867
+ durable_cleared: bool = False,
868
+ ) -> None:
869
+ session_id = session.durable_session_id if session_id is None else session_id
870
+ owner_epoch = session.durable_owner_epoch if owner_epoch is None else owner_epoch
871
+ api_key_id = session.key.api_key_id if api_key_id is None else api_key_id
872
+ instance_id = (
873
+ _service_get_settings().http_responses_session_bridge_instance_id if instance_id is None else instance_id
874
+ )
875
+ if instance_id is None:
876
+ return
877
+ _schedule_http_bridge_background_cleanup(
878
+ service,
879
+ _retry_denied_http_bridge_anchor_clear(
880
+ service,
881
+ session,
882
+ session_id=session_id,
883
+ api_key_id=api_key_id,
884
+ instance_id=instance_id,
885
+ owner_epoch=owner_epoch,
886
+ response_id=response_id,
887
+ durable_cleared=durable_cleared,
888
+ ),
889
+ name=f"http-bridge-denied-anchor-clear-{_hash_identifier(response_id)}",
890
+ error_message="HTTP bridge denied-anchor clear retry failed",
891
+ )
892
+
893
+
894
+ T = TypeVar("T")
895
+ _TEXT_DELTA_EVENT_TYPES = frozenset({"response.output_text.delta", "response.refusal.delta"})
896
+ _UNSUPPORTED_DURABLE_TOOL_CALL_ITEM_TYPES = frozenset(
897
+ {
898
+ "computer_call",
899
+ "mcp_approval_request",
900
+ }
901
+ )
902
+
903
+
904
+ def _record_http_bridge_tool_call_lifecycle(
905
+ request_state: _WebSocketRequestState,
906
+ *,
907
+ event_type: str | None,
908
+ payload: dict[str, JsonValue] | None,
909
+ ) -> None:
910
+ if event_type not in {"response.output_item.added", "response.output_item.done"}:
911
+ return
912
+ item = payload.get("item") if isinstance(payload, dict) else None
913
+ if not isinstance(item, dict):
914
+ request_state.tool_call_manifest_invalid = True
915
+ return
916
+ item_type = item.get("type")
917
+ if not isinstance(item_type, str):
918
+ request_state.tool_call_manifest_invalid = True
919
+ return
920
+ if item_type in _UNSUPPORTED_DURABLE_TOOL_CALL_ITEM_TYPES:
921
+ # These calls require client-provided continuation state but are not
922
+ # representable by the direct function/custom/apply-patch replay proof.
923
+ # Persisting only a parallel supported call would make a partial suffix
924
+ # look complete, so keep the whole durable manifest unknown.
925
+ request_state.tool_call_manifest_invalid = True
926
+ return
927
+ if item_type not in _PENDING_TOOL_CALL_ITEM_TYPES:
928
+ return
929
+ call_id = item.get("call_id")
930
+ if not isinstance(call_id, str) or not call_id:
931
+ request_state.tool_call_manifest_invalid = True
932
+ return
933
+ target = (
934
+ request_state.added_tool_call_types
935
+ if event_type == "response.output_item.added"
936
+ else request_state.pending_tool_call_types
937
+ )
938
+ existing = target.get(call_id)
939
+ if existing is not None:
940
+ request_state.tool_call_manifest_invalid = True
941
+ return
942
+ target[call_id] = item_type
943
+
944
+
945
+ def _response_completed_tool_call_types(payload: dict[str, JsonValue] | None) -> dict[str, str] | None:
946
+ response = payload.get("response") if isinstance(payload, dict) else None
947
+ output = response.get("output") if isinstance(response, dict) else None
948
+ if not isinstance(output, list):
949
+ return None
950
+ result: dict[str, str] = {}
951
+ for item in output:
952
+ if not isinstance(item, dict):
953
+ return None
954
+ item_type = item.get("type")
955
+ if not isinstance(item_type, str):
956
+ return None
957
+ if item_type in _UNSUPPORTED_DURABLE_TOOL_CALL_ITEM_TYPES:
958
+ return None
959
+ if item_type not in _PENDING_TOOL_CALL_ITEM_TYPES:
960
+ continue
961
+ call_id = item.get("call_id")
962
+ if not isinstance(call_id, str) or not call_id:
963
+ return None
964
+ existing = result.get(call_id)
965
+ if existing is not None:
966
+ return None
967
+ result[call_id] = item_type
968
+ return result
969
+
970
+
971
+ def _durable_pending_tool_call_manifest(
972
+ request_state: _WebSocketRequestState,
973
+ payload: dict[str, JsonValue] | None,
974
+ ) -> dict[str, str] | None:
975
+ terminal_calls = _response_completed_tool_call_types(payload)
976
+ if request_state.tool_call_manifest_invalid or terminal_calls is None:
977
+ return None
978
+ if request_state.added_tool_call_types != request_state.pending_tool_call_types:
979
+ return None
980
+ if terminal_calls and terminal_calls != request_state.pending_tool_call_types:
981
+ return None
982
+ return dict(request_state.pending_tool_call_types)
983
+
984
+
985
+ _SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE = "security_work_authorization_required"
986
+ _SECURITY_WORK_RETRY_MESSAGE = (
987
+ "Upstream flagged this request as possible cybersecurity work. "
988
+ "codex-lb is retrying on an account marked as authorized for security work."
989
+ )
990
+
991
+
992
+ def _relinquish_http_bridge_capacity_wait_ownership(
993
+ session: "_HTTPBridgeSession",
994
+ request_state: _WebSocketRequestState,
995
+ claimed_terminal_request_states: list[_WebSocketRequestState],
996
+ ) -> None:
997
+ """Pop a capacity-wait request from pending ownership and record the settlement claim.
998
+
999
+ The selected-model capacity branch reserves its request *in* pending
1000
+ ownership while it waits (a younger submit must not take its queue slot),
1001
+ so the terminal pop that normally records the ``"claimed"`` marker never
1002
+ ran for it. Once the branch gives that ownership up -- the replay was
1003
+ refused or failed, or the request never had a consumer -- this bookkeeping
1004
+ continuation is the request's sole settlement owner exactly like a popped
1005
+ terminal, and an abort between here and ``_finalize_terminal_settlement``
1006
+ must reach the shielded abort settlement (issue #1594) instead of leaking
1007
+ the API-key reservation, its heartbeat, and the re-claimed create gate. The
1008
+ caller holds ``session.pending_lock``.
1009
+ """
1010
+ if request_state not in session.pending_requests:
1011
+ return
1012
+ session.pending_requests.remove(request_state)
1013
+ if _http_bridge_request_counts_against_queue(request_state):
1014
+ session.queued_request_count = max(0, session.queued_request_count - 1)
1015
+ request_state.terminal_settlement_phase = "claimed"
1016
+ if request_state not in claimed_terminal_request_states:
1017
+ claimed_terminal_request_states.append(request_state)
1018
+
1019
+
1020
+ async def _wait_before_http_bridge_model_capacity_retry(
1021
+ request_state: _WebSocketRequestState | None,
1022
+ *,
1023
+ emit_keepalives: bool,
1024
+ error_message: str | None,
1025
+ cancel_when_detached: bool = False,
1026
+ signal_startup_wait: bool = True,
1027
+ scheduler: Scheduler = REAL_SCHEDULER,
1028
+ clock: Clock = REAL_CLOCK,
1029
+ ) -> bool:
1030
+ """Sleep the bounded selected-model capacity delay before a bridge replay.
1031
+
1032
+ A propagated-error stream (``emit_keepalives=False``) normally signals the
1033
+ pre-response startup wait so its startup probe keeps waiting instead of
1034
+ timing out. An accepted request already committed its response start, so
1035
+ the wait branch passes ``signal_startup_wait=False`` for it: re-signalling
1036
+ would re-arm the startup wait behind a stream that is already open (spec:
1037
+ "Accepted public streams are replayed without keepalives").
1038
+ """
1039
+ if request_state is None or not is_upstream_model_capacity_error(error_message):
1040
+ return True
1041
+
1042
+ deadline = request_state.bridge_request_deadline
1043
+ if deadline is None:
1044
+ deadline = request_state.started_at + _http_bridge_request_budget_seconds(_service_get_settings())
1045
+ remaining_budget_seconds = max(0.0, deadline - clock.monotonic())
1046
+ if remaining_budget_seconds <= 0:
1047
+ return False
1048
+
1049
+ sleep_seconds = min(_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS, remaining_budget_seconds)
1050
+ request_state.account_capacity_waiting = True
1051
+ request_state.account_capacity_wait_reason = error_message
1052
+ request_state.account_capacity_wait_started_at = request_state.account_capacity_wait_started_at or clock.monotonic()
1053
+ request_state.account_capacity_wait_retry_after_seconds = sleep_seconds
1054
+ request_state.account_capacity_wait_suppress_keepalive = not emit_keepalives
1055
+ if not emit_keepalives and signal_startup_wait:
1056
+ _signal_http_bridge_capacity_startup_wait(request_state)
1057
+ try:
1058
+ remaining_sleep_seconds = sleep_seconds
1059
+ keepalive_countdown_seconds = 0.0
1060
+ while remaining_sleep_seconds > 0:
1061
+ if cancel_when_detached and request_state.event_queue is None:
1062
+ return False
1063
+ if emit_keepalives and keepalive_countdown_seconds <= 0 and request_state.event_queue is not None:
1064
+ await request_state.event_queue.put(
1065
+ format_sse_event(
1066
+ _account_capacity_wait_payload(
1067
+ request_state,
1068
+ request_id=request_state.request_log_id or request_state.request_id,
1069
+ reason=error_message,
1070
+ retry_after_seconds=remaining_sleep_seconds,
1071
+ now=clock.monotonic(),
1072
+ )
1073
+ )
1074
+ )
1075
+ keepalive_countdown_seconds = _ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS
1076
+ chunk_seconds = min(
1077
+ remaining_sleep_seconds,
1078
+ (
1079
+ _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS
1080
+ if cancel_when_detached
1081
+ else _ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS
1082
+ ),
1083
+ )
1084
+ await scheduler.sleep(chunk_seconds)
1085
+ remaining_sleep_seconds -= chunk_seconds
1086
+ keepalive_countdown_seconds -= chunk_seconds
1087
+ return (not cancel_when_detached or request_state.event_queue is not None) and clock.monotonic() < deadline
1088
+ finally:
1089
+ request_state.account_capacity_waiting = False
1090
+ if emit_keepalives:
1091
+ request_state.account_capacity_wait_suppress_keepalive = False
1092
+ request_state.account_capacity_wait_reason = None
1093
+ request_state.account_capacity_wait_retry_after_seconds = None
1094
+
1095
+
1096
+ async def _release_http_bridge_model_capacity_retry_admission(
1097
+ request_state: _WebSocketRequestState,
1098
+ ) -> None:
1099
+ """Release shared capacity while retaining the session create gate."""
1100
+ if request_state.response_create_admission is not None:
1101
+ request_state.response_create_admission.release()
1102
+ request_state.response_create_admission = None
1103
+ request_state.response_create_admission_reacquire_required = True
1104
+ account_response_create_lease = request_state.account_response_create_lease
1105
+ account_response_create_release = request_state.account_response_create_release
1106
+ request_state.account_response_create_lease = None
1107
+ request_state.account_response_create_release = None
1108
+ if account_response_create_lease is not None and account_response_create_release is not None:
1109
+ await account_response_create_release(account_response_create_lease)
1110
+
1111
+
1112
+ def _signal_http_bridge_model_capacity_retry_ready(
1113
+ request_state: _WebSocketRequestState,
1114
+ *,
1115
+ waited_for_model_capacity_retry: bool,
1116
+ retried: bool,
1117
+ ) -> None:
1118
+ if waited_for_model_capacity_retry and retried and request_state.propagate_http_errors:
1119
+ if request_state.capacity_startup_wait_event is not None:
1120
+ request_state.capacity_startup_wait_event.clear()
1121
+ if request_state.capacity_startup_ready_event is not None:
1122
+ request_state.capacity_startup_ready_event.set()
1123
+ _signal_propagated_capacity_startup_ready()
1124
+
1125
+
1126
+ def _signal_http_bridge_capacity_startup_wait(request_state: _WebSocketRequestState) -> None:
1127
+ if request_state.capacity_startup_ready_event is not None:
1128
+ request_state.capacity_startup_ready_event.clear()
1129
+ if request_state.capacity_startup_wait_event is not None:
1130
+ request_state.capacity_startup_wait_event.set()
1131
+ _signal_propagated_capacity_startup_wait()
1132
+
1133
+
1134
+ def _archive_http_bridge_upstream_text(
1135
+ session: "_HTTPBridgeSession",
1136
+ text: str,
1137
+ request_state: "_WebSocketRequestState | None",
1138
+ ) -> None:
1139
+ _archive_http_bridge_upstream_message(
1140
+ session,
1141
+ UpstreamWebSocketMessage(kind="text", text=text),
1142
+ request_state,
1143
+ )
1144
+
1145
+
1146
+ def _archive_http_bridge_upstream_message(
1147
+ session: "_HTTPBridgeSession",
1148
+ message: UpstreamWebSocketMessage,
1149
+ request_state: "_WebSocketRequestState | None",
1150
+ ) -> None:
1151
+ if request_state is None or request_state.archive_request_id is None:
1152
+ archive_request_id = None
1153
+ else:
1154
+ archive_request_id = request_state.archive_request_id
1155
+ archive_received = getattr(session.upstream, "archive_received", None)
1156
+ if not callable(archive_received):
1157
+ return
1158
+ token = set_request_id(archive_request_id)
1159
+ try:
1160
+ archive_received(message)
1161
+ finally:
1162
+ reset_request_id(token)
1163
+
1164
+
1165
+ async def _http_bridge_receive_timeout_with_eventless_deadline(
1166
+ session: "_HTTPBridgeSession",
1167
+ receive_timeout: _WebSocketReceiveTimeout | None,
1168
+ *,
1169
+ now: float,
1170
+ stuck_gate_retire_after_seconds: float,
1171
+ ) -> _WebSocketReceiveTimeout | None:
1172
+ if session.closed:
1173
+ return receive_timeout
1174
+ async with session.pending_lock:
1175
+ deadlines = [
1176
+ deadline
1177
+ for request_state in session.pending_requests
1178
+ if (
1179
+ deadline := _http_bridge_eventless_precreated_deadline(
1180
+ request_state,
1181
+ stuck_gate_retire_after_seconds=stuck_gate_retire_after_seconds,
1182
+ )
1183
+ )
1184
+ is not None
1185
+ ]
1186
+ if not deadlines:
1187
+ return receive_timeout
1188
+ eventless_timeout = _WebSocketReceiveTimeout(
1189
+ timeout_seconds=max(0.0, min(deadlines) - now),
1190
+ error_code=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
1191
+ error_message="Upstream did not acknowledge response.create before the client-safe deadline",
1192
+ fail_all_pending=True,
1193
+ )
1194
+ if receive_timeout is None or eventless_timeout.timeout_seconds <= receive_timeout.timeout_seconds:
1195
+ return eventless_timeout
1196
+ return receive_timeout
1197
+
1198
+
1199
+ async def _cancel_http_bridge_reader_child(
1200
+ task: asyncio.Task[Any] | None,
1201
+ *,
1202
+ label: str,
1203
+ scheduler_owner: Any,
1204
+ cleanup_tasks: set[asyncio.Task[None]] | None = None,
1205
+ ) -> bool:
1206
+ if task is None:
1207
+ return True
1208
+ if task.done():
1209
+ try:
1210
+ task.result()
1211
+ except asyncio.CancelledError:
1212
+ pass
1213
+ except Exception:
1214
+ logger.debug("HTTP bridge reader child already failed during cleanup label=%s", label, exc_info=True)
1215
+ return True
1216
+ try:
1217
+ return bool(
1218
+ await _await_cancelled_task(
1219
+ task,
1220
+ label=label,
1221
+ cleanup_tasks=cleanup_tasks,
1222
+ scheduler=scheduler_for(scheduler_owner),
1223
+ )
1224
+ )
1225
+ except Exception:
1226
+ logger.debug("Failed to cancel HTTP bridge reader child label=%s", label, exc_info=True)
1227
+ return task.done()
1228
+
1229
+
1230
+ async def _clear_durable_http_bridge_response_anchor(
1231
+ service: Any,
1232
+ session: "_HTTPBridgeSession",
1233
+ ) -> None:
1234
+ """Invalidate a durable proxy-injected anchor that proved eventless.
1235
+
1236
+ Runs while ``session`` still owns the durable row (before retirement
1237
+ releases the lease), so the fenced write lands under the session's own
1238
+ owner epoch instead of silently losing the fence to a released owner.
1239
+ """
1240
+ if session.durable_session_id is None or session.durable_owner_epoch is None:
1241
+ return
1242
+ try:
1243
+ lookup = await service._durable_bridge.clear_live_session_response_anchor(
1244
+ session_id=session.durable_session_id,
1245
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
1246
+ owner_epoch=session.durable_owner_epoch,
1247
+ )
1248
+ except Exception:
1249
+ logger.warning("Failed to clear durable HTTP bridge response anchor after stuck timeout", exc_info=True)
1250
+ return
1251
+ if lookup is None or lookup.owner_epoch != session.durable_owner_epoch or lookup.latest_response_id is not None:
1252
+ # None means the durable row is gone entirely (e.g. purged); an
1253
+ # epoch or anchor mismatch means a newer owner already claimed the
1254
+ # session before this fenced write executed. Either way, the anchor
1255
+ # was never actually cleared, so do not report an invalidation that
1256
+ # did not happen.
1257
+ return
1258
+ _log_http_bridge_event(
1259
+ "durable_anchor_invalidated",
1260
+ session.key,
1261
+ account_id=session.account.id,
1262
+ model=session.request_model,
1263
+ detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
1264
+ cache_key_family=session.key.affinity_kind,
1265
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1266
+ )
1267
+
1268
+
1269
+ async def _abandon_durable_http_bridge_continuity(
1270
+ service: Any,
1271
+ session: "_HTTPBridgeSession",
1272
+ *,
1273
+ detail: str = "repeated_zero_event_idle_timeout",
1274
+ settle_circuit: bool = False,
1275
+ expected_continuity: object = _POISON_ANCHOR_CAPTURE_UNAVAILABLE,
1276
+ authorized_episode: Any = None,
1277
+ ) -> bool:
1278
+ """Clear durable continuity before retiring a repeatedly poisoned bridge.
1279
+
1280
+ ``rebind_session_account(clear_continuity=True)`` is an existing fenced
1281
+ write that clears the durable response/turn anchor and its alias rows while
1282
+ this worker still owns the session. The ordinary retirement path then
1283
+ closes the row and removes the process-local registrations.
1284
+ """
1285
+ if session.durable_session_id is None or session.durable_owner_epoch is None:
1286
+ return False
1287
+ # Captured BEFORE the continuity clear's await: this is the closest
1288
+ # snapshot to the episode the poison consult validated, and a sibling
1289
+ # replacing the circuit episode while the rebind is in flight must not
1290
+ # be mistaken for the authorizing one — the settle below clears only
1291
+ # the episode this abandonment invalidated.
1292
+ async with service._http_bridge_retry_circuit_lock:
1293
+ # The consulted episode outranks a registry re-read: a sibling
1294
+ # settle can remove the entry between the consult and this lock,
1295
+ # and a None capture would run the settle unfenced — free to clear
1296
+ # a replacement episode opened during the awaits below. The
1297
+ # registry read remains only for legacy callers with no consult.
1298
+ authorizing_state = (
1299
+ authorized_episode
1300
+ if authorized_episode is not None
1301
+ else service._http_bridge_retry_circuits.get(session.key)
1302
+ )
1303
+ abandonment_expected_episode = (
1304
+ (
1305
+ authorizing_state.persisted_updated_at_epoch,
1306
+ authorizing_state.consecutive_failures,
1307
+ authorizing_state.persisted_admission_generation,
1308
+ )
1309
+ if authorizing_state is not None
1310
+ else None
1311
+ )
1312
+ rebind_fence_kwargs: dict[str, Any] = {}
1313
+ if expected_continuity is not _POISON_ANCHOR_CAPTURE_UNAVAILABLE:
1314
+ # Fence the continuity clear on both anchors captured when the
1315
+ # episode was validated: a completion registering a fresh response
1316
+ # anchor or a turn-state write landing in between changes a column
1317
+ # and the fenced write matches nothing.
1318
+ expected_response_id, expected_turn_state = cast("tuple[str | None, str | None]", expected_continuity)
1319
+ rebind_fence_kwargs["expected_latest_response_id"] = expected_response_id
1320
+ rebind_fence_kwargs["expected_latest_turn_state"] = expected_turn_state
1321
+ try:
1322
+ cleared = await service._durable_bridge.rebind_session_account(
1323
+ session_id=session.durable_session_id,
1324
+ api_key_id=session.key.api_key_id,
1325
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
1326
+ owner_epoch=session.durable_owner_epoch,
1327
+ account_id=session.account.id,
1328
+ clear_continuity=True,
1329
+ **rebind_fence_kwargs,
1330
+ )
1331
+ except Exception:
1332
+ logger.warning("Failed to abandon poisoned HTTP bridge continuity", exc_info=True)
1333
+ return False
1334
+ if not cleared:
1335
+ logger.warning(
1336
+ "Durable bridge continuity clear was fenced before poisoned anchor retirement",
1337
+ extra={
1338
+ "session_id": session.durable_session_id,
1339
+ "account_id": session.account.id,
1340
+ },
1341
+ )
1342
+ return False
1343
+ _log_http_bridge_event(
1344
+ "durable_anchor_poisoned",
1345
+ session.key,
1346
+ account_id=session.account.id,
1347
+ model=session.request_model,
1348
+ detail=detail,
1349
+ cache_key_family=session.key.affinity_kind,
1350
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1351
+ )
1352
+ # Settle only when the requests this abandonment covers are actually
1353
+ # stranded. A stale-anchor rejection that still holds a verified full
1354
+ # resend is about to be replayed, and that replay claims the circuit
1355
+ # generation at dispatch (#1863); clearing the circuit under it removes the
1356
+ # fence it depends on, which is why `response.completed` also skips its
1357
+ # clear for such a replay. Settling unconditionally broke five variants of
1358
+ # the stale-owner replay suite, and settling for no funnel caller left the
1359
+ # production wedge cooling for 60s after its anchor was already gone.
1360
+ if not settle_circuit:
1361
+ return True
1362
+ # The circuit was opened by failures against the anchor this call just
1363
+ # removed, so its cooldown is now backing off a cause that no longer
1364
+ # exists. Leaving it running refuses requests that carry no anchor at all:
1365
+ # observed live as a burst of ~25 rejections logged
1366
+ # `reason=retry_circuit_cooldown_continuity_bound previous_response_id=None`
1367
+ # in the 60s after a successful clear, every one of which would have gone
1368
+ # upstream cleanly. A confirmed abandonment is proof the next attempt
1369
+ # cannot repeat that failure, which is the same evidence a completed
1370
+ # response carries, so settle the circuit the same way. A genuinely new
1371
+ # failure re-opens it at the usual threshold.
1372
+ # A sibling completion can race this settle after the continuity clear:
1373
+ # it registers a FRESH anchor and erases its own transitional tombstone,
1374
+ # and writing the tombstone here again would durably 404 every valid
1375
+ # follow-up riding that fresh anchor, with no registration left to erase
1376
+ # it. Re-read the continuity: an anchor that is present and different
1377
+ # from the one this abandonment removed is fresh evidence, and the
1378
+ # settle then writes a plain reset instead of the tombstone.
1379
+ abandonment_settled_detail: str | None = _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
1380
+ abandoned_anchor, abandoned_turn_state = (
1381
+ (expected_continuity[0], expected_continuity[1])
1382
+ if isinstance(expected_continuity, tuple) and len(expected_continuity) == 2
1383
+ else (None, None)
1384
+ )
1385
+
1386
+ def _continuity_is_fresh(observed: tuple[str | None, str | None] | None) -> bool:
1387
+ # Fresh evidence in EITHER continuity column: a delta can resolve
1388
+ # through the turn state alone, so a turn-state advance without a
1389
+ # response id is still continuity a tombstone must not fail closed.
1390
+ return observed is not None and (
1391
+ (observed[0] is not None and observed[0] != abandoned_anchor)
1392
+ or (observed[1] is not None and observed[1] != abandoned_turn_state)
1393
+ )
1394
+
1395
+ if session.durable_session_id:
1396
+ try:
1397
+ post_clear_continuity = await service._durable_bridge.session_latest_continuity(
1398
+ session_id=session.durable_session_id
1399
+ )
1400
+ except Exception:
1401
+ # Unknown continuity keeps the tombstone: the stale-tombstone
1402
+ # cost is self-healing (the next completion settles and erases
1403
+ # it), while a missing tombstone silently drops delta context.
1404
+ post_clear_continuity = None
1405
+ if _continuity_is_fresh(post_clear_continuity):
1406
+ abandonment_settled_detail = None
1407
+ circuit_settled = await service._clear_http_bridge_retry_circuit(
1408
+ session,
1409
+ settled_detail=abandonment_settled_detail,
1410
+ settled_detail_authoritative=True,
1411
+ expected_episode=abandonment_expected_episode,
1412
+ )
1413
+ if not circuit_settled:
1414
+ # The clear reloads fresh state on every call, so one immediate
1415
+ # retry covers a transient durable failure or a double CAS miss
1416
+ # before the surviving cooldown is left to expire on its own. The
1417
+ # abandonment itself still reports success: the anchor is gone, the
1418
+ # marker must record that so the episode cannot clear now-empty
1419
+ # continuity again, and only the settlement is owed.
1420
+ circuit_settled = await service._clear_http_bridge_retry_circuit(
1421
+ session,
1422
+ settled_detail=abandonment_settled_detail,
1423
+ settled_detail_authoritative=True,
1424
+ expected_episode=abandonment_expected_episode,
1425
+ )
1426
+ if not circuit_settled:
1427
+ _log_http_bridge_event(
1428
+ "durable_circuit_settle_failed",
1429
+ session.key,
1430
+ account_id=session.account.id,
1431
+ model=session.request_model,
1432
+ detail=detail,
1433
+ cache_key_family=session.key.affinity_kind,
1434
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1435
+ )
1436
+ if circuit_settled and abandonment_settled_detail is not None and session.durable_session_id:
1437
+ # The pre-settle re-read closes most of the window, but a sibling
1438
+ # registration can still land between that read and the settle
1439
+ # write — and its own tombstone cleanup ran BEFORE this settle
1440
+ # wrote a new one. Re-read once more after the write and erase the
1441
+ # tombstone this settle left when fresh continuity now exists; the
1442
+ # fenced detail-only rewrite defers to any newer write, and both
1443
+ # sides reconciling after their own writes makes every
1444
+ # interleaving converge.
1445
+ try:
1446
+ reconcile_continuity = await service._durable_bridge.session_latest_continuity(
1447
+ session_id=session.durable_session_id
1448
+ )
1449
+ except Exception:
1450
+ reconcile_continuity = None
1451
+ if _continuity_is_fresh(reconcile_continuity):
1452
+ await service._http_bridge_reconcile_transitional_tombstone(session)
1453
+ return True
1454
+
1455
+
1456
+ async def _invalidate_denied_http_bridge_anchor(
1457
+ service: Any,
1458
+ session: "_HTTPBridgeSession",
1459
+ *,
1460
+ denied_response_id: str | None,
1461
+ ) -> bool:
1462
+ """Retire an anchor upstream has explicitly denied.
1463
+
1464
+ ``previous_response_not_found`` against an anchor the proxy injected is a
1465
+ verdict, not a symptom: the id came from this proxy's own durable record,
1466
+ no client asked for it, and upstream says it does not exist. The poison
1467
+ counter cannot act on that verdict because it only scores reader failures
1468
+ (see ``_HTTP_BRIDGE_ANCHOR_POISON_DETAILS``), so the dead id survives at
1469
+ any threshold and is re-injected into the following turn, where the
1470
+ store-context trim strips the resent history against it and upstream never
1471
+ emits ``response.created``.
1472
+
1473
+ Clearing costs nothing that is not already lost. The next turn simply
1474
+ dispatches unanchored with the history the client sends, which is the
1475
+ client's own replay rather than a server-side one, so no forked child
1476
+ response can be created against a parent this proxy cannot see.
1477
+
1478
+ The durable clear is conditional on the denied id still being the durable
1479
+ latest response, and removes only that response alias. The in-memory clear
1480
+ is unconditional for the same id even when the durable write is fenced,
1481
+ because it strictly removes one way for the denied id to come back. A
1482
+ durable row that survives re-injects the id on a later turn, which is denied
1483
+ in turn and re-enters this path, so the clear is re-attempted rather than
1484
+ lost.
1485
+
1486
+ ``closed`` only fences new admissions. Requests admitted before a session
1487
+ closed can still deliver a terminal denial and must publish its provenance
1488
+ and finish the same cleanup.
1489
+ """
1490
+ if denied_response_id is None:
1491
+ return False
1492
+ sibling_advanced = False
1493
+ async with session.lifecycle_lock:
1494
+ # Serialize publication with the submitter's final tombstone check and
1495
+ # upstream send. A sibling completion can advance the current carrier
1496
+ # while an already-prepared request still holds the denied id; that
1497
+ # request must remain fenced even when there is no current anchor left
1498
+ # to clear.
1499
+ session.denied_proxy_injected_anchor_generation += 1
1500
+ # Retain denial provenance after the session-local tombstone is retired.
1501
+ # A request that began on an absent canonical session otherwise receives
1502
+ # a successor with no local generation and can redispatch this id.
1503
+ durable_session_id = session.durable_session_id
1504
+ durable_owner_epoch = session.durable_owner_epoch
1505
+ durable_api_key_id = session.key.api_key_id
1506
+ durable_instance_id = _service_get_settings().http_responses_session_bridge_instance_id
1507
+ owner_key = durable_session_id if durable_session_id is not None else f"local:{id(session)}"
1508
+ recorded_generation = _record_http_bridge_denied_anchor_fence(
1509
+ service,
1510
+ denied_response_id,
1511
+ owner_key=owner_key,
1512
+ owner_epoch=durable_owner_epoch,
1513
+ )
1514
+ current_fence = _http_bridge_denied_anchor_fence_current_map(service).get(owner_key)
1515
+ recorded_entry = _http_bridge_denied_anchor_fence_entry(service, denied_response_id)
1516
+ record_won_owner_slot = (
1517
+ current_fence == denied_response_id
1518
+ and recorded_entry is not None
1519
+ and recorded_entry.owner_key == owner_key
1520
+ and recorded_entry.generation == recorded_generation
1521
+ )
1522
+ # Keep one current session-local tombstone. Displaced ids remain
1523
+ # fenced in the process ledger while any prepared request pins them;
1524
+ # retaining every historical id here would make the session carrier
1525
+ # grow without bound and duplicate that ledger's ownership fence. A
1526
+ # stale detached predecessor must not erase a successor's tombstone,
1527
+ # so only replace the set when this publication owns the current slot.
1528
+ if record_won_owner_slot:
1529
+ session.denied_proxy_injected_anchor_ids.clear()
1530
+ session.denied_proxy_injected_anchor_ids.add(denied_response_id)
1531
+ elif not session.denied_proxy_injected_anchor_ids:
1532
+ # A detached predecessor may still have a request pinned to its
1533
+ # own session object. Keep that one local tombstone without
1534
+ # growing a second historical slot beside a successor's entry.
1535
+ session.denied_proxy_injected_anchor_ids.add(denied_response_id)
1536
+ # Another request may have completed and advanced the anchor between
1537
+ # the denied dispatch and this frame. Only retire the id that was
1538
+ # refused.
1539
+ if session.last_completed_response_id != denied_response_id:
1540
+ sibling_advanced = True
1541
+ elif hasattr(session, "denied_proxy_injected_anchor_cleanup_pending") and recorded_entry is not None:
1542
+ # Only the current anchor needs durable/alias cleanup. A sibling
1543
+ # that already advanced the carrier leaves a historical fence for
1544
+ # any pinned request, but has no unresolved cleanup to preserve at
1545
+ # session close.
1546
+ session.denied_proxy_injected_anchor_cleanup_pending.add(denied_response_id)
1547
+ cleared = False
1548
+ no_durable_owner = durable_session_id is None or durable_owner_epoch is None
1549
+ if sibling_advanced:
1550
+ if no_durable_owner:
1551
+ _forget_http_bridge_denied_anchor_fence(
1552
+ service,
1553
+ denied_response_id,
1554
+ owner_key=owner_key,
1555
+ owner_epoch=durable_owner_epoch,
1556
+ )
1557
+ getattr(session, "denied_proxy_injected_anchor_cleanup_pending", set()).discard(denied_response_id)
1558
+ return False
1559
+ retry_durable_clear = False
1560
+ unregister_succeeded = False
1561
+ owner_matches_for_cleanup = False
1562
+ unregister_error: BaseException | None = None
1563
+ durable_error: BaseException | None = None
1564
+ try:
1565
+ try:
1566
+ if not no_durable_owner:
1567
+ lookup = await service._durable_bridge.clear_live_session_response_anchor_if_matches(
1568
+ session_id=durable_session_id,
1569
+ api_key_id=durable_api_key_id,
1570
+ instance_id=durable_instance_id,
1571
+ owner_epoch=durable_owner_epoch,
1572
+ response_id=denied_response_id,
1573
+ )
1574
+ cleared = lookup is not None
1575
+ # ``None`` is a clean fenced no-match (the owner epoch or
1576
+ # latest response changed), not a durable failure. Preserve
1577
+ # the local alias and tombstone for the newer owner, but do
1578
+ # not schedule background retries; raised failures below are
1579
+ # the retryable case.
1580
+ except Exception:
1581
+ retry_durable_clear = True
1582
+ logger.warning("Failed to clear denied HTTP bridge response anchor", exc_info=True)
1583
+ finally:
1584
+ try:
1585
+ if cleared or no_durable_owner:
1586
+ async with session.lifecycle_lock:
1587
+ owner_matches_for_cleanup = (
1588
+ session.durable_session_id == durable_session_id
1589
+ and session.durable_owner_epoch == durable_owner_epoch
1590
+ )
1591
+ if owner_matches_for_cleanup:
1592
+ try:
1593
+ unregister_result = await service._unregister_http_bridge_previous_response_id(
1594
+ session,
1595
+ denied_response_id,
1596
+ expected_durable_session_id=durable_session_id,
1597
+ expected_durable_owner_epoch=durable_owner_epoch,
1598
+ )
1599
+ unregister_succeeded = unregister_result is not False
1600
+ except asyncio.CancelledError as exc:
1601
+ unregister_error = exc
1602
+ except Exception as exc:
1603
+ unregister_error = exc
1604
+ retry_durable_clear = True
1605
+ finally:
1606
+ async with session.lifecycle_lock:
1607
+ if session.last_completed_response_id == denied_response_id:
1608
+ session.last_completed_response_id = None
1609
+ session.last_completed_response_account_id = None
1610
+ session.last_completed_input_count = 0
1611
+ session.last_completed_input_prefix_fingerprint = None
1612
+ session.last_pending_tool_calls.clear()
1613
+ if owner_matches_for_cleanup and (cleared or no_durable_owner) and unregister_succeeded:
1614
+ session.denied_proxy_injected_anchor_ids.discard(denied_response_id)
1615
+ session.denied_proxy_injected_anchor_cleanup_pending.discard(denied_response_id)
1616
+ session.denied_proxy_injected_anchor_generation += 1
1617
+ _forget_http_bridge_denied_anchor_fence(
1618
+ service,
1619
+ denied_response_id,
1620
+ owner_key=durable_session_id if durable_session_id is not None else f"local:{id(session)}",
1621
+ owner_epoch=durable_owner_epoch,
1622
+ )
1623
+ if retry_durable_clear:
1624
+ _schedule_denied_http_bridge_anchor_clear_retry(
1625
+ service,
1626
+ session,
1627
+ response_id=denied_response_id,
1628
+ session_id=durable_session_id,
1629
+ api_key_id=durable_api_key_id,
1630
+ instance_id=durable_instance_id,
1631
+ owner_epoch=durable_owner_epoch,
1632
+ durable_cleared=cleared,
1633
+ )
1634
+ except asyncio.CancelledError as exc:
1635
+ durable_error = exc
1636
+ if unregister_error is not None:
1637
+ raise unregister_error
1638
+ if durable_error is not None:
1639
+ raise durable_error
1640
+ return cleared
1641
+
1642
+
1643
+ def _denied_proxy_injected_anchor_id(
1644
+ request_states: Iterable["_WebSocketRequestState"],
1645
+ ) -> str | None:
1646
+ """Choose the anchor a denial may retire, if any.
1647
+
1648
+ Only an anchor shared exclusively by requests that codex-lb injected onto
1649
+ full-resend-shaped payloads can be retired. A client-supplied or delta-only
1650
+ sibling has no other way to convey prior context once its anchor is gone,
1651
+ which is the same rule the expired-anchor path applies before clearing
1652
+ durable continuity.
1653
+ """
1654
+ requests_by_anchor: dict[str, list[_WebSocketRequestState]] = {}
1655
+ for request_state in request_states:
1656
+ previous_response_id = request_state.previous_response_id
1657
+ if previous_response_id is not None:
1658
+ requests_by_anchor.setdefault(previous_response_id, []).append(request_state)
1659
+
1660
+ safely_retirable_anchors = [
1661
+ previous_response_id
1662
+ for previous_response_id, grouped_request_states in requests_by_anchor.items()
1663
+ if all(
1664
+ request_state.proxy_injected_previous_response_id
1665
+ and request_state.proxy_injected_anchor_had_full_resend_payload
1666
+ for request_state in grouped_request_states
1667
+ )
1668
+ ]
1669
+ if len(safely_retirable_anchors) == 1:
1670
+ return safely_retirable_anchors[0]
1671
+ return None
1672
+
1673
+
1674
+ async def _retire_denied_http_bridge_anchor(
1675
+ service: Any,
1676
+ session: "_HTTPBridgeSession",
1677
+ *,
1678
+ request_states: Iterable["_WebSocketRequestState"],
1679
+ ) -> None:
1680
+ """Best-effort retirement of an anchor upstream denied.
1681
+
1682
+ Retirement is bookkeeping. It must never change how the denial itself is
1683
+ delivered downstream, so a failure here is logged and swallowed rather than
1684
+ escaping into terminal-event handling.
1685
+ """
1686
+ denied_response_id = _denied_proxy_injected_anchor_id(request_states)
1687
+ if denied_response_id is None:
1688
+ return
1689
+ try:
1690
+ await _invalidate_denied_http_bridge_anchor(
1691
+ service,
1692
+ session,
1693
+ denied_response_id=denied_response_id,
1694
+ )
1695
+ except Exception:
1696
+ logger.warning("Failed to retire a denied proxy-injected HTTP bridge anchor", exc_info=True)
1697
+
1698
+
1699
+ class _HTTPBridgeUpstreamEventsMixin:
1700
+ def _spawn_http_bridge_upstream_reader(self: Any, session: "_HTTPBridgeSession") -> asyncio.Task[None]:
1701
+ """Start the per-session upstream reader on the owner's scheduler."""
1702
+
1703
+ return scheduler_for(self).create_task(self._relay_http_bridge_upstream_messages(session))
1704
+
1705
+ async def _cancel_http_bridge_previous_reader(self: Any, old_reader: asyncio.Task[Any]) -> bool:
1706
+ """Cancel and drain a replaced upstream reader before a reconnect."""
1707
+
1708
+ return await _await_cancelled_task(
1709
+ old_reader,
1710
+ label="http bridge upstream reader",
1711
+ cleanup_tasks=self._background_cleanup_tasks,
1712
+ scheduler=scheduler_for(self),
1713
+ )
1714
+
1715
+ async def _await_http_bridge_registry_wait(self: Any, future: asyncio.Future[T], *, timeout: float) -> T:
1716
+ """Await a shared inflight/capacity registry future with a bounded timeout.
1717
+
1718
+ Not ``wait_for(shield(...))``: shield attaches per-waiter callbacks to
1719
+ the shared registry future, which livelocks the event loop under mass
1720
+ timeout (see shared_future.py). The timed wait runs on the owner's
1721
+ scheduler so virtual time can expire it.
1722
+ """
1723
+
1724
+ return await wait_on_shared_future(future, timeout=timeout, scheduler=scheduler_for(self))
1725
+
1726
+ async def _fail_http_bridge_reader_and_maybe_retire(
1727
+ self: Any,
1728
+ session: "_HTTPBridgeSession",
1729
+ *,
1730
+ error_code: str,
1731
+ error_message: str,
1732
+ penalize_account: bool = True,
1733
+ retire_detail: str | None = None,
1734
+ force_retire: bool = False,
1735
+ upstream_close_code: int | None = None,
1736
+ response_events_seen: int | None = None,
1737
+ transport_classification: str | None = None,
1738
+ retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None,
1739
+ account_neutral_transport_drop: bool = False,
1740
+ ) -> bool:
1741
+ scheduler = scheduler_for(self)
1742
+ session.closed = True
1743
+ async with session.pending_lock:
1744
+ failed_pending_count = sum(
1745
+ 1
1746
+ for request_state in session.pending_requests
1747
+ if _http_bridge_request_counts_against_queue(request_state)
1748
+ )
1749
+ session.queued_request_count = max(0, session.queued_request_count - failed_pending_count)
1750
+ observed_response_events = max(
1751
+ (getattr(request_state, "response_event_count", 0) for request_state in session.pending_requests),
1752
+ default=0,
1753
+ )
1754
+ pending_request_states = list(session.pending_requests)
1755
+ if retry_circuit_attempt_selection is None:
1756
+ retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
1757
+ pending_request_states
1758
+ )
1759
+ retry_circuit_attempt_kwargs = {
1760
+ "retry_circuit_attempt_selection": retry_circuit_attempt_selection,
1761
+ }
1762
+ # The #1534 wedge shape: a reattached stream that streamed response
1763
+ # events whose ``response.created`` was never assigned. The eventless
1764
+ # watchdog and the durable-anchor clear both key on
1765
+ # ``response_event_count == 0`` and never trip on it, so quarantine
1766
+ # the session here so later requests stop re-attaching to it.
1767
+ _record_http_bridge_quarantine_wedged_pending(self, session, pending_request_states)
1768
+ observed_close_code = (
1769
+ upstream_close_code if upstream_close_code is not None else session.last_upstream_close_code
1770
+ )
1771
+ observed_response_events = (
1772
+ response_events_seen if response_events_seen is not None else observed_response_events
1773
+ )
1774
+ close_classification = (
1775
+ _classify_upstream_close(observed_close_code, response_events_seen=observed_response_events)
1776
+ if observed_close_code is not None
1777
+ else None
1778
+ )
1779
+ _log_http_bridge_event(
1780
+ "reader_failure",
1781
+ session.key,
1782
+ account_id=session.account.id,
1783
+ model=session.request_model,
1784
+ pending_count=failed_pending_count,
1785
+ detail=error_code,
1786
+ error_message=_truncate_identifier(error_message),
1787
+ upstream_close_code=observed_close_code,
1788
+ response_events_seen=observed_response_events,
1789
+ transport_classification=transport_classification
1790
+ or (
1791
+ f"websocket_close_{close_classification}"
1792
+ if close_classification is not None
1793
+ else "websocket_transport_error"
1794
+ ),
1795
+ cache_key_family=session.key.affinity_kind,
1796
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1797
+ )
1798
+ # Draining-only requests no longer count against the queue, but their
1799
+ # event-batcher contexts still belong to the disconnected operation
1800
+ # and must be discarded just like ordinary pending requests.
1801
+ operation_states: list[Any] = [
1802
+ request_state for request_state in pending_request_states if getattr(request_state, "operation_id", None)
1803
+ ]
1804
+ # Remove the disconnected attempt's in-memory spool before publishing
1805
+ # UNKNOWN/ACKNOWLEDGED state. A same-replica reconnect may reclaim the
1806
+ # operation as soon as that state is visible; discarding afterward
1807
+ # could then delete the replacement attempt's events.
1808
+ discard_operation = getattr(
1809
+ getattr(self, "_http_bridge_operation_event_batcher", None),
1810
+ "discard_operation",
1811
+ None,
1812
+ )
1813
+ if callable(discard_operation):
1814
+ for request_state in operation_states:
1815
+ operation_id = getattr(request_state, "operation_id", None)
1816
+ if operation_id:
1817
+ await discard_operation(operation_id=operation_id)
1818
+ for request_state in operation_states:
1819
+ # A shared websocket can carry several logical response.create
1820
+ # requests. Classify each operation from its own event count;
1821
+ # using the session-wide maximum would mark an eventless
1822
+ # sibling as safely retryable after another request streamed.
1823
+ operation_state = "unknown" if getattr(request_state, "response_event_count", 0) == 0 else "acknowledged"
1824
+ await _update_http_bridge_operation_state(
1825
+ self,
1826
+ session,
1827
+ request_state,
1828
+ state=operation_state,
1829
+ )
1830
+ if force_retire and retire_detail:
1831
+ _log_http_bridge_event(
1832
+ retire_detail,
1833
+ session.key,
1834
+ account_id=session.account.id,
1835
+ model=session.request_model,
1836
+ pending_count=failed_pending_count,
1837
+ detail=retire_detail,
1838
+ cache_key_family=session.key.affinity_kind,
1839
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1840
+ )
1841
+ try:
1842
+ reservations_settled = await self._fail_pending_websocket_requests(
1843
+ account=session.account,
1844
+ account_id_value=session.account.id,
1845
+ pending_requests=session.pending_requests,
1846
+ pending_lock=session.pending_lock,
1847
+ error_code=error_code,
1848
+ error_message=error_message,
1849
+ api_key=None,
1850
+ response_create_gate=session.response_create_gate,
1851
+ penalize_account=penalize_account,
1852
+ )
1853
+ if (
1854
+ failed_pending_count > 0
1855
+ and reservations_settled is not False
1856
+ and observed_response_events == 0
1857
+ and (
1858
+ retire_detail == _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL
1859
+ or account_neutral_transport_drop
1860
+ )
1861
+ ):
1862
+ # Only penalize the account after pending-request cleanup has
1863
+ # settled its API-key reservations. A failed release must not
1864
+ # be hidden behind an already-recorded timeout health signal.
1865
+ # Account-neutral abrupt drops share the same windowed signal:
1866
+ # one drop is infrastructure noise, but repeated eventless
1867
+ # drops on the same account remain evidence of an account-side
1868
+ # fault and must still drain it (issue #1754).
1869
+ await _record_http_bridge_account_timeout_signal(
1870
+ self,
1871
+ session,
1872
+ detail=(
1873
+ "eventless_transport_drop"
1874
+ if account_neutral_transport_drop
1875
+ else _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL
1876
+ ),
1877
+ )
1878
+ finally:
1879
+ if session.admission_waiter_count > 0 and not force_retire:
1880
+ retry_circuit_detail = None
1881
+ if close_classification == "clean":
1882
+ retry_circuit_detail = "clean_close"
1883
+ elif observed_response_events == 0:
1884
+ retry_circuit_detail = next(
1885
+ (
1886
+ detail
1887
+ for detail in (retire_detail, error_code)
1888
+ if detail in {"stream_incomplete", "stream_idle_timeout", "upstream_keepalive_timeout"}
1889
+ ),
1890
+ None,
1891
+ )
1892
+ # Mirror the terminal, grouped, and direct-retirement paths:
1893
+ # a failed request that still holds a verified safe replay is
1894
+ # about to re-dispatch and claim the circuit generation, so it
1895
+ # must not strike the circuit or reach the poison clear here
1896
+ # either. A pre-drain handoff with no states keeps striking.
1897
+ pending_states_present = [state for state in pending_request_states if state is not None]
1898
+ reader_strike_eligible = not pending_states_present or any(
1899
+ not _http_bridge_request_state_holds_safe_replay(state) for state in pending_states_present
1900
+ )
1901
+ reader_retired = False
1902
+ reader_cancellation: asyncio.CancelledError | None = None
1903
+ if failed_pending_count > 0 and retry_circuit_detail is not None and reader_strike_eligible:
1904
+ reader_strike_detail = retry_circuit_detail
1905
+
1906
+ async def _reader_strike_and_clear() -> bool:
1907
+ consecutive_failures = (
1908
+ await self._record_http_bridge_retry_circuit_failure_for_attempt_selection(
1909
+ session,
1910
+ detail=reader_strike_detail,
1911
+ selection=retry_circuit_attempt_selection,
1912
+ )
1913
+ )
1914
+ poison_candidate_detail = await self._http_bridge_effective_anchor_poison_detail(
1915
+ session, reader_strike_detail
1916
+ )
1917
+ poison_episode = None
1918
+ poison_expected_anchor: object = _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1919
+ if poison_candidate_detail is not None and observed_response_events == 0:
1920
+ # The consult returns the exact episode it
1921
+ # validated; the marker below scopes to it.
1922
+ poison_episode, poison_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
1923
+ session,
1924
+ consecutive_failures=consecutive_failures,
1925
+ )
1926
+ if poison_candidate_detail is None or poison_episode is None:
1927
+ return False
1928
+ durable_cleared = await _abandon_durable_http_bridge_continuity(
1929
+ self,
1930
+ session,
1931
+ detail=poison_candidate_detail,
1932
+ settle_circuit=_http_bridge_abandonment_may_settle_circuit(pending_request_states),
1933
+ expected_continuity=poison_expected_anchor,
1934
+ authorized_episode=poison_episode,
1935
+ )
1936
+ if not durable_cleared:
1937
+ _log_http_bridge_event(
1938
+ "durable_anchor_poison_clear_failed",
1939
+ session.key,
1940
+ account_id=session.account.id,
1941
+ model=session.request_model,
1942
+ pending_count=session.admission_waiter_count,
1943
+ detail=poison_candidate_detail,
1944
+ cache_key_family=session.key.affinity_kind,
1945
+ model_class=(
1946
+ _extract_model_class(session.request_model) if session.request_model else None
1947
+ ),
1948
+ )
1949
+ return False
1950
+ await self._http_bridge_mark_poison_anchor_cleared(session, episode=poison_episode)
1951
+ await self._retire_stale_pending_http_bridge_session(
1952
+ session,
1953
+ detail=poison_candidate_detail,
1954
+ response_events_seen=observed_response_events,
1955
+ # Reader-failure retirement must never revive: the
1956
+ # pending turns were already terminally failed and
1957
+ # this reader is condemned, so a post-suspension
1958
+ # liveness signal (which durable-anchor
1959
+ # rehydration can spoof without upstream evidence)
1960
+ # would only leave a readerless session registered.
1961
+ allow_liveness_revive=False,
1962
+ **retry_circuit_attempt_kwargs,
1963
+ )
1964
+ return True
1965
+
1966
+ # The failed requests are already drained and finalized,
1967
+ # so a cancellation escaping the strike, the episode
1968
+ # consult, or the rebind would leave an at-threshold
1969
+ # poisoned anchor stored with no retirement left to retry
1970
+ # the abandonment. Defer cancellation across the whole
1971
+ # settlement like the grouped path, then re-raise it.
1972
+ reader_settlement_task = scheduler.create_task(
1973
+ _reader_strike_and_clear(),
1974
+ name=f"http-bridge-reader-poison-settlement-{session.durable_session_id}",
1975
+ )
1976
+ reader_retired, reader_cancellation = await _await_task_deferring_cancellation(
1977
+ reader_settlement_task
1978
+ )
1979
+ if reader_retired is True:
1980
+ force_retire = True
1981
+ if reader_cancellation is not None:
1982
+ raise reader_cancellation
1983
+ else:
1984
+ if reader_cancellation is not None:
1985
+ raise reader_cancellation
1986
+ _log_http_bridge_event(
1987
+ "retire_deferred_for_admission_waiter",
1988
+ session.key,
1989
+ account_id=session.account.id,
1990
+ model=session.request_model,
1991
+ pending_count=session.admission_waiter_count,
1992
+ detail=retire_detail or error_code,
1993
+ cache_key_family=session.key.affinity_kind,
1994
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
1995
+ )
1996
+ else:
1997
+ if close_classification == "clean" and failed_pending_count > 0:
1998
+ waiterless_retirement = self._retire_stale_pending_http_bridge_session(
1999
+ session,
2000
+ detail=error_code,
2001
+ retry_circuit_detail="clean_close",
2002
+ response_events_seen=observed_response_events,
2003
+ retired_request_count=failed_pending_count,
2004
+ retired_request_states=pending_request_states,
2005
+ # See the poison branch above: reader-failure
2006
+ # retirement never revives a condemned session.
2007
+ allow_liveness_revive=False,
2008
+ **retry_circuit_attempt_kwargs,
2009
+ )
2010
+ else:
2011
+ waiterless_retirement = self._retire_stale_pending_http_bridge_session(
2012
+ session,
2013
+ detail=retire_detail or error_code,
2014
+ response_events_seen=observed_response_events,
2015
+ # ``_fail_pending_websocket_requests`` has already
2016
+ # claimed and drained these states. Carry the count
2017
+ # and the pre-drain state snapshot sampled under
2018
+ # ``pending_lock`` across that ownership transfer, so
2019
+ # normal reader failures still consume one strike and
2020
+ # a drained safe-replay holder still blocks the
2021
+ # settlement. The deferred/poison branch records its
2022
+ # own strike above and intentionally does not pass it.
2023
+ retired_request_count=failed_pending_count,
2024
+ retired_request_states=pending_request_states,
2025
+ # See the poison branch above: reader-failure
2026
+ # retirement never revives a condemned session.
2027
+ allow_liveness_revive=False,
2028
+ **retry_circuit_attempt_kwargs,
2029
+ )
2030
+ # The failed requests are already drained and finalized, so a
2031
+ # relay cancellation escaping this direct retirement would
2032
+ # skip the waiterless strike, the episode consult, the
2033
+ # poisoned-anchor abandonment, and the detach/close work with
2034
+ # no remaining request lifecycle to retry them. Defer
2035
+ # cancellation across the whole retirement like the
2036
+ # admission-waiter branch above, then re-raise it.
2037
+ waiterless_retirement_task = scheduler.create_task(
2038
+ waiterless_retirement,
2039
+ name=f"http-bridge-waiterless-retirement-{session.durable_session_id}",
2040
+ )
2041
+ _waiterless_result, waiterless_cancellation = await _await_task_deferring_cancellation(
2042
+ waiterless_retirement_task
2043
+ )
2044
+ if waiterless_cancellation is not None:
2045
+ raise waiterless_cancellation
2046
+ return force_retire or session.admission_waiter_count == 0
2047
+
2048
+ async def _relay_http_bridge_upstream_messages(
2049
+ self: Any,
2050
+ session: "_HTTPBridgeSession",
2051
+ ) -> None:
2052
+ scheduler = scheduler_for(self)
2053
+ clock = clock_for(self)
2054
+ runtime_settings = _service_get_settings()
2055
+ relay_upstream = session.upstream
2056
+ receive_task: asyncio.Task[UpstreamWebSocketMessage] | None = None
2057
+ wakeup_task: asyncio.Task[bool] | None = None
2058
+ reader_failure_retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None
2059
+ try:
2060
+ while True:
2061
+ reader_failure_retry_circuit_attempt_selection = None
2062
+ # Clear before taking the deadline snapshot. A send before the
2063
+ # clear is represented by its timestamp; a send after it leaves
2064
+ # the event set and wakes the persistent receive wait below.
2065
+ session.upstream_reader_wakeup.clear()
2066
+ # The wakeup waiter is reused across iterations while it is
2067
+ # still pending. A set() that landed while the previous
2068
+ # message was being processed completed it; that send is
2069
+ # already represented in the snapshot below, so consume the
2070
+ # fired waiter here without awaiting and re-arm it before the
2071
+ # next wait.
2072
+ if wakeup_task is not None and wakeup_task.done():
2073
+ wakeup_task.result()
2074
+ wakeup_task = None
2075
+ receive_timeout = await self._next_websocket_receive_timeout(
2076
+ session.pending_requests,
2077
+ pending_lock=session.pending_lock,
2078
+ proxy_request_budget_seconds=_http_bridge_request_budget_seconds(runtime_settings),
2079
+ stream_idle_timeout_seconds=runtime_settings.stream_idle_timeout_seconds,
2080
+ )
2081
+ stuck_gate_retire_after_seconds = float(
2082
+ _http_bridge_helpers.HTTP_BRIDGE_STUCK_GATE_RETIRE_AFTER_SECONDS
2083
+ )
2084
+ receive_timeout = await _http_bridge_receive_timeout_with_eventless_deadline(
2085
+ session,
2086
+ receive_timeout,
2087
+ now=clock.monotonic(),
2088
+ stuck_gate_retire_after_seconds=stuck_gate_retire_after_seconds,
2089
+ )
2090
+ if receive_task is None:
2091
+ receive_task = scheduler.create_task(session.upstream.receive())
2092
+
2093
+ message: UpstreamWebSocketMessage | None = None
2094
+ timed_out = False
2095
+ if receive_task.done():
2096
+ message = receive_task.result()
2097
+ receive_task = None
2098
+ elif receive_timeout is not None and receive_timeout.timeout_seconds <= 0:
2099
+ timed_out = True
2100
+ else:
2101
+ # Event.wait() waiters are level-triggered: a waiter
2102
+ # registered before clear() still fires on the next set(),
2103
+ # so a pending waiter stays valid across iterations and is
2104
+ # only re-created after it fires. Cancelling it per
2105
+ # message cost a create_task + sleep(0) + cancel + timed
2106
+ # asyncio.wait round trip; the finally below cancels the
2107
+ # long-lived waiter once at loop exit.
2108
+ if wakeup_task is None:
2109
+ wakeup_task = scheduler.create_task(session.upstream_reader_wakeup.wait())
2110
+ done, _pending = await scheduler.wait(
2111
+ (receive_task, wakeup_task),
2112
+ timeout=receive_timeout.timeout_seconds if receive_timeout is not None else None,
2113
+ return_when=asyncio.FIRST_COMPLETED,
2114
+ )
2115
+ if receive_task in done:
2116
+ message = receive_task.result()
2117
+ receive_task = None
2118
+ elif wakeup_task in done:
2119
+ wakeup_task.result()
2120
+ wakeup_task = None
2121
+ continue
2122
+ else:
2123
+ timed_out = True
2124
+
2125
+ if timed_out:
2126
+ if receive_timeout is None:
2127
+ raise RuntimeError("HTTP bridge reader timed out without a timeout contract")
2128
+ if receive_timeout.error_code == _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL:
2129
+ if receive_task is not None and receive_task.done():
2130
+ continue
2131
+ async with session.lifecycle_lock:
2132
+ # Send-failure cleanup marks the session closed and
2133
+ # disarms the timestamp while holding this lock.
2134
+ # Do not race that caller's terminal settlement.
2135
+ if session.closed:
2136
+ continue
2137
+ now = clock.monotonic()
2138
+ async with session.pending_lock:
2139
+ if receive_task is not None and receive_task.done():
2140
+ continue
2141
+ expired_request_states = [
2142
+ request_state
2143
+ for request_state in session.pending_requests
2144
+ if (
2145
+ deadline := _http_bridge_eventless_precreated_deadline(
2146
+ request_state,
2147
+ stuck_gate_retire_after_seconds=stuck_gate_retire_after_seconds,
2148
+ )
2149
+ )
2150
+ is not None
2151
+ and deadline <= now
2152
+ ]
2153
+ if not expired_request_states:
2154
+ continue
2155
+ expired_retry_circuit_attempt_selection = (
2156
+ _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
2157
+ expired_request_states
2158
+ )
2159
+ )
2160
+ reader_failure_retry_circuit_attempt_selection = expired_retry_circuit_attempt_selection
2161
+ pending_count = len(session.pending_requests)
2162
+ # A delta-only request has no other way to
2163
+ # convey prior context once its anchor is
2164
+ # cleared, so only clear anchors codex-lb
2165
+ # injected onto a full-resend-shaped payload.
2166
+ expired_proxy_injected_anchor = any(
2167
+ request_state.proxy_injected_previous_response_id
2168
+ and request_state.proxy_injected_anchor_had_full_resend_payload
2169
+ for request_state in expired_request_states
2170
+ )
2171
+
2172
+ # Do not mutate durable continuity while a receive
2173
+ # may still deliver the response event that proves
2174
+ # this timeout stale. The non-cancellable branch
2175
+ # remains fail-closed, so it clears immediately
2176
+ # before its forced retirement.
2177
+ force_retire = False
2178
+ if receive_task is not None:
2179
+ receive_cancelled = await _cancel_http_bridge_reader_child(
2180
+ receive_task,
2181
+ label="HTTP bridge upstream receive after missing response.created",
2182
+ cleanup_tasks=self._background_cleanup_tasks,
2183
+ scheduler_owner=self,
2184
+ )
2185
+ if receive_task.done() and not receive_task.cancelled():
2186
+ # A response (or a typed receive failure)
2187
+ # won the race with timeout handling. Let
2188
+ # the normal reader path settle it.
2189
+ continue
2190
+ force_retire = not receive_cancelled and not receive_task.cancelled()
2191
+ if not force_retire:
2192
+ receive_task = None
2193
+ async with session.pending_lock:
2194
+ for request_state in session.pending_requests:
2195
+ if request_state.failure_phase_override is None:
2196
+ request_state.failure_phase_override = "upstream"
2197
+ if request_state.failure_detail_override is None:
2198
+ request_state.failure_detail_override = (
2199
+ _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL
2200
+ )
2201
+ if force_retire:
2202
+ # Do not reconnect while the old receive task
2203
+ # still owns the superseded socket. The ordinary
2204
+ # timeout path takes the same fail-closed branch;
2205
+ # retain the explicit account-neutral timeout
2206
+ # classification rather than routing through the
2207
+ # generic reader-crash account penalty path.
2208
+ if expired_proxy_injected_anchor:
2209
+ await _clear_durable_http_bridge_response_anchor(self, session)
2210
+ _record_http_bridge_quarantine_eventless_timeout(self, session)
2211
+ session.closed = True
2212
+ await self._fail_http_bridge_reader_and_maybe_retire(
2213
+ session,
2214
+ error_code="upstream_request_timeout",
2215
+ error_message=receive_timeout.error_message,
2216
+ penalize_account=False,
2217
+ retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
2218
+ force_retire=True,
2219
+ retry_circuit_attempt_selection=expired_retry_circuit_attempt_selection,
2220
+ )
2221
+ break
2222
+ # A successfully cancelled receive cannot deliver
2223
+ # a late response event, so this is the safe point
2224
+ # to persist the durable-anchor invalidation.
2225
+ if expired_proxy_injected_anchor:
2226
+ await _clear_durable_http_bridge_response_anchor(self, session)
2227
+ # Count the eventless retire toward the repeated-
2228
+ # wedge quarantine; the first strike still goes
2229
+ # through the bounded pre-created recovery below.
2230
+ _record_http_bridge_quarantine_eventless_timeout(self, session)
2231
+ _record_http_bridge_stuck_retire(
2232
+ reason=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
2233
+ session=session,
2234
+ )
2235
+ _log_http_bridge_event(
2236
+ "missing_response_created_timeout",
2237
+ session.key,
2238
+ account_id=session.account.id,
2239
+ model=session.request_model,
2240
+ pending_count=pending_count,
2241
+ detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
2242
+ cache_key_family=session.key.affinity_kind,
2243
+ model_class=(
2244
+ _extract_model_class(session.request_model) if session.request_model else None
2245
+ ),
2246
+ )
2247
+ # A fresh, self-contained hard request can use the
2248
+ # same bounded pre-created recovery as the idle
2249
+ # timeout path. Keep the session open until the
2250
+ # recovery routine claims the handoff; otherwise
2251
+ # its retry gate would reject the request as
2252
+ # already retired. Continuity-bound requests still
2253
+ # fail closed in _retry_http_bridge_precreated_request.
2254
+ retried = await self._retry_http_bridge_precreated_request(session)
2255
+ if retried:
2256
+ continue
2257
+ session.closed = True
2258
+ await self._fail_http_bridge_reader_and_maybe_retire(
2259
+ session,
2260
+ error_code="upstream_request_timeout",
2261
+ error_message=receive_timeout.error_message,
2262
+ penalize_account=False,
2263
+ retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
2264
+ force_retire=True,
2265
+ retry_circuit_attempt_selection=expired_retry_circuit_attempt_selection,
2266
+ )
2267
+ break
2268
+
2269
+ async with session.pending_lock:
2270
+ retry_circuit_attempt_selection = (
2271
+ _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
2272
+ tuple(session.pending_requests)
2273
+ )
2274
+ )
2275
+ reader_failure_retry_circuit_attempt_selection = retry_circuit_attempt_selection
2276
+ if receive_task is not None:
2277
+ receive_cancelled = await _cancel_http_bridge_reader_child(
2278
+ receive_task,
2279
+ label="HTTP bridge upstream receive after timeout",
2280
+ cleanup_tasks=self._background_cleanup_tasks,
2281
+ scheduler_owner=self,
2282
+ )
2283
+ if not receive_cancelled:
2284
+ raise RuntimeError("HTTP bridge upstream receive did not cancel after timeout")
2285
+ receive_task = None
2286
+ retried = await self._retry_http_bridge_precreated_request(session)
2287
+ if retried:
2288
+ continue
2289
+ async with session.lifecycle_lock:
2290
+ await self._fail_http_bridge_reader_and_maybe_retire(
2291
+ session,
2292
+ error_code=receive_timeout.error_code,
2293
+ error_message=receive_timeout.error_message,
2294
+ retry_circuit_attempt_selection=retry_circuit_attempt_selection,
2295
+ )
2296
+ break
2297
+
2298
+ if message is None:
2299
+ raise RuntimeError("HTTP bridge upstream receive completed without a message")
2300
+ if message.kind == "text" and message.text is not None:
2301
+ session.last_upstream_close_code = None
2302
+ if EVENT_MARKER in message.text:
2303
+ publish_live_usage(
2304
+ parse_rate_limit_event_text(message.text),
2305
+ account_id=session.account.id,
2306
+ chatgpt_account_id=session.account.chatgpt_account_id,
2307
+ )
2308
+ await self._process_http_bridge_upstream_text(
2309
+ session,
2310
+ message.text,
2311
+ message=message,
2312
+ scheduler=scheduler,
2313
+ clock=clock,
2314
+ )
2315
+ if await self._retire_http_bridge_after_drain_if_ready(session):
2316
+ break
2317
+ continue
2318
+
2319
+ async with session.pending_lock:
2320
+ archive_request_state = session.pending_requests[0] if len(session.pending_requests) == 1 else None
2321
+ response_events_seen = max(
2322
+ (request_state.response_event_count for request_state in session.pending_requests),
2323
+ default=0,
2324
+ )
2325
+ # Buffered reasoning preludes are suppressed from
2326
+ # response_event_count on purpose, but they are still
2327
+ # application-layer output: a drop after one is not an
2328
+ # eventless drop for account-health purposes.
2329
+ upstream_output_observed = any(
2330
+ getattr(request_state, "upstream_model_output_seen", False)
2331
+ for request_state in session.pending_requests
2332
+ )
2333
+ accepted_response_pending = any(
2334
+ request_state.response_id is not None and not request_state.awaiting_response_created
2335
+ for request_state in session.pending_requests
2336
+ )
2337
+ reader_failure_retry_circuit_attempt_selection = (
2338
+ _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
2339
+ tuple(session.pending_requests)
2340
+ )
2341
+ )
2342
+ _archive_http_bridge_upstream_message(session, message, archive_request_state)
2343
+ session.last_upstream_close_generation += 1
2344
+ session.last_upstream_close_code = message.close_code
2345
+ retried = False
2346
+ # Account-neutral transport failures do not prove that the
2347
+ # upstream rejected response.create. The request may still be
2348
+ # executing, so replay could duplicate work, billing, or tool
2349
+ # side effects. Clean closes remain eligible for the bounded
2350
+ # pre-created retry circuit maintained by the session.
2351
+ account_neutral = is_account_neutral_websocket_error_code(message.error_code)
2352
+ # Only a terminal transport message (close or error) may replay
2353
+ # an accepted turn: a protocol-invalid binary frame did not end
2354
+ # the socket, so it keeps the pre-created retry semantics only.
2355
+ if not account_neutral and (message.kind in {"close", "error"} or not accepted_response_pending):
2356
+ retried = await self._retry_http_bridge_precreated_request(session)
2357
+ if retried:
2358
+ continue
2359
+ close_classification = (
2360
+ _classify_upstream_close(message.close_code, response_events_seen=response_events_seen)
2361
+ if message.close_code is not None
2362
+ else None
2363
+ )
2364
+ # An abrupt drop with no close frame and no response events is
2365
+ # weaker account-health evidence than a graceful pre-created
2366
+ # close, which is already exempted below. Keep the individual
2367
+ # drop account-neutral; repeated eventless drops still feed
2368
+ # the windowed account drain signal inside the failure path.
2369
+ # Only terminal transport messages qualify: a protocol-invalid
2370
+ # binary frame also carries no close code but did not end the
2371
+ # socket, so it keeps the existing penalty semantics.
2372
+ account_neutral_transport_drop = (
2373
+ message.kind in ("close", "error")
2374
+ and not account_neutral
2375
+ and not upstream_output_observed
2376
+ and _is_account_neutral_transport_drop(
2377
+ message.close_code, response_events_seen=response_events_seen
2378
+ )
2379
+ )
2380
+ async with session.lifecycle_lock:
2381
+ if (
2382
+ session.liveness_settlement_owner == "send"
2383
+ and message.error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE
2384
+ ):
2385
+ # A submitter publishes this dedicated claim beside the
2386
+ # failing send while holding lifecycle_lock. ``closed``
2387
+ # alone is only an admission/retirement state and must
2388
+ # never suppress settlement of still-pending siblings.
2389
+ break
2390
+ await self._fail_http_bridge_reader_and_maybe_retire(
2391
+ session,
2392
+ error_code=message.error_code or "stream_incomplete",
2393
+ error_message=_upstream_websocket_disconnect_message(message),
2394
+ upstream_close_code=message.close_code,
2395
+ response_events_seen=response_events_seen,
2396
+ transport_classification=(
2397
+ f"websocket_close_{close_classification}"
2398
+ if close_classification is not None
2399
+ else "websocket_transport_error"
2400
+ ),
2401
+ retry_circuit_attempt_selection=reader_failure_retry_circuit_attempt_selection,
2402
+ penalize_account=(
2403
+ not account_neutral
2404
+ and not account_neutral_transport_drop
2405
+ and not (message.kind == "close" and close_classification == "clean")
2406
+ ),
2407
+ account_neutral_transport_drop=account_neutral_transport_drop,
2408
+ **(
2409
+ # An admission waiter must not inherit a socket whose
2410
+ # heartbeat already proved it dead. Other failures
2411
+ # preserve the existing deferred-retirement handoff.
2412
+ {"force_retire": True}
2413
+ if message.error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE
2414
+ else {}
2415
+ ),
2416
+ )
2417
+ break
2418
+ except asyncio.CancelledError:
2419
+ raise
2420
+ except Exception as exc:
2421
+ if reader_failure_retry_circuit_attempt_selection is None:
2422
+ # A receive/processing exception can jump here before the
2423
+ # ordinary timeout or close branches publish their snapshot.
2424
+ # Capture before waiting for lifecycle ownership so a
2425
+ # concurrent recovery cannot replace the failed physical send.
2426
+ async with session.pending_lock:
2427
+ reader_failure_retry_circuit_attempt_selection = (
2428
+ _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
2429
+ tuple(session.pending_requests)
2430
+ )
2431
+ )
2432
+ logger.warning(
2433
+ "HTTP bridge upstream reader crashed account_id=%s bridge_kind=%s",
2434
+ session.account.id,
2435
+ session.key.affinity_kind,
2436
+ exc_info=True,
2437
+ )
2438
+ error_code = exc.error_code if isinstance(exc, UpstreamWebSocketTransportError) else "stream_incomplete"
2439
+ account_neutral = is_account_neutral_websocket_error_code(error_code)
2440
+ async with session.lifecycle_lock:
2441
+ if not (
2442
+ session.liveness_settlement_owner == "send"
2443
+ and error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE
2444
+ ):
2445
+ # Match the message path above when receive() raises while
2446
+ # a concurrent send failure already owns settlement.
2447
+ await self._fail_http_bridge_reader_and_maybe_retire(
2448
+ session,
2449
+ error_code=error_code,
2450
+ error_message=(
2451
+ str(exc)
2452
+ if isinstance(exc, UpstreamWebSocketTransportError)
2453
+ else "HTTP bridge upstream reader crashed before response.completed"
2454
+ ),
2455
+ penalize_account=not account_neutral,
2456
+ retry_circuit_attempt_selection=reader_failure_retry_circuit_attempt_selection,
2457
+ # Preserve ordinary crash handoff behavior, but never hand
2458
+ # a heartbeat-expired socket to an admission waiter.
2459
+ **({"force_retire": True} if error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE else {}),
2460
+ )
2461
+ finally:
2462
+ # Mark the socket this reader owned closed before the child
2463
+ # cancellations below suspend: the persistent wakeup waiter is
2464
+ # normally still pending here, and a recovery that replaces the
2465
+ # socket during that suspension clears the flag itself after
2466
+ # swapping ``session.upstream``.
2467
+ if session.upstream is relay_upstream:
2468
+ session.closed = True
2469
+ await _cancel_http_bridge_reader_child(
2470
+ wakeup_task,
2471
+ label="HTTP bridge reader wakeup wait",
2472
+ cleanup_tasks=self._background_cleanup_tasks,
2473
+ scheduler_owner=self,
2474
+ )
2475
+ await _cancel_http_bridge_reader_child(
2476
+ receive_task,
2477
+ label="HTTP bridge upstream receive",
2478
+ cleanup_tasks=self._background_cleanup_tasks,
2479
+ scheduler_owner=self,
2480
+ )
2481
+
2482
+ async def _process_http_bridge_upstream_text(
2483
+ self: Any,
2484
+ session: "_HTTPBridgeSession",
2485
+ text: str,
2486
+ *,
2487
+ message: UpstreamWebSocketMessage | None = None,
2488
+ scheduler: Scheduler | None = None,
2489
+ clock: Clock | None = None,
2490
+ ) -> None:
2491
+ # The relay loop resolves the collaborators once per session and passes
2492
+ # them down; the fallback only serves direct callers (tests).
2493
+ if scheduler is None:
2494
+ scheduler = scheduler_for(self)
2495
+ if clock is None:
2496
+ clock = clock_for(self)
2497
+ # One JSON document per websocket text frame: parse it directly instead
2498
+ # of framing it as SSE and running the line parser over it. The
2499
+ # data-only block is what unmatched events relay.
2500
+ event_block = f"data: {text}\n\n"
2501
+ if (
2502
+ message is not None
2503
+ and message.responses_interpreted
2504
+ and message.payload is not None
2505
+ and text.startswith("{")
2506
+ and "\n" not in text
2507
+ and "\r" not in text
2508
+ ):
2509
+ # Only this shape uses the direct JSON path in the legacy bridge.
2510
+ # Preserve its SSE-field semantics for whitespace/multiline text.
2511
+ payload = message.payload
2512
+ event_type = message.event_type
2513
+ routing = message.routing
2514
+ else:
2515
+ payload = parse_sse_data_json_text(text)
2516
+ event_type = classify_event_type(payload)
2517
+ routing = None
2518
+ event = parse_sse_event_payload(payload) if event_type in _LIFECYCLE_EVENT_TYPES else None
2519
+ completed_delivery_scope = _HTTPBridgeCompletedDeliveryScope() if event_type == "response.completed" else None
2520
+ claimed_terminal_request_states: list[_WebSocketRequestState] = []
2521
+ try:
2522
+ await self._process_parsed_http_bridge_upstream_event(
2523
+ session,
2524
+ text=text,
2525
+ event_block=event_block,
2526
+ payload=payload,
2527
+ event=event,
2528
+ event_type=event_type,
2529
+ response_id=_websocket_response_id(event, payload, routing=routing),
2530
+ completed_delivery_scope=completed_delivery_scope,
2531
+ claimed_terminal_request_states=claimed_terminal_request_states,
2532
+ scheduler=scheduler,
2533
+ clock=clock,
2534
+ )
2535
+ except BaseException:
2536
+ # Includes CancelledError. A terminal request popped from
2537
+ # session.pending_requests is owned exclusively by this
2538
+ # bookkeeping continuation: the reader's failure path and the
2539
+ # downstream detach backstop only settle requests still in
2540
+ # pending ownership, so an abort here would otherwise leak the
2541
+ # API-key reservation and its heartbeat forever (issue #1594).
2542
+ await self._settle_aborted_http_bridge_terminal_states(
2543
+ session,
2544
+ claimed_terminal_request_states,
2545
+ )
2546
+ raise
2547
+ else:
2548
+ for claimed_request_state in claimed_terminal_request_states:
2549
+ claimed_request_state.terminal_settlement_phase = None
2550
+ finally:
2551
+ if completed_delivery_scope is not None:
2552
+ completed_delivery_scope.active = False
2553
+
2554
+ async def _settle_aborted_http_bridge_terminal_states(
2555
+ self: Any,
2556
+ session: "_HTTPBridgeSession",
2557
+ request_states: list[_WebSocketRequestState],
2558
+ ) -> None:
2559
+ """Settle claimed-but-unfinalized terminal requests after an abort.
2560
+
2561
+ Once terminal-event bookkeeping pops a request from
2562
+ ``session.pending_requests`` it is the sole owner of that request's
2563
+ reservation settlement. If the bookkeeping continuation raises or is
2564
+ cancelled before ``_finalize_websocket_request_state`` transfers the
2565
+ settlement, nothing else can reach the request: the orphaned
2566
+ reservation heartbeat would keep refreshing ``updated_at`` and defeat
2567
+ the stale-reservation janitor permanently. Settle those requests here
2568
+ under a shielded scope. Requests a retry branch restored to pending
2569
+ ownership are skipped; they are reachable by ordinary cleanup again.
2570
+ Settlement is idempotent (release only flips ``status == "reserved"``),
2571
+ so overlapping with an already-transferred finalize is safe.
2572
+ """
2573
+ if not request_states:
2574
+ return
2575
+ with anyio.CancelScope(shield=True):
2576
+ for request_state in request_states:
2577
+ if request_state.terminal_settlement_phase is None:
2578
+ continue
2579
+ async with session.pending_lock:
2580
+ if request_state in session.pending_requests:
2581
+ request_state.terminal_settlement_phase = None
2582
+ continue
2583
+ request_state.terminal_settlement_phase = "abandoned"
2584
+ try:
2585
+ self._cancel_request_state_api_key_reservation_heartbeat(request_state)
2586
+ await self._release_websocket_request_state_reservation(request_state)
2587
+ request_state.api_key_reservation = None
2588
+ request_state.terminal_settlement_phase = None
2589
+ except Exception:
2590
+ # Leave the "abandoned" marker so the downstream detach
2591
+ # backstop can retry the settlement later.
2592
+ logger.warning(
2593
+ "Failed to settle aborted HTTP bridge terminal request request_id=%s account_id=%s",
2594
+ request_state.request_log_id or request_state.request_id,
2595
+ session.account.id,
2596
+ exc_info=True,
2597
+ )
2598
+ # Best effort: unblock the downstream waiter so it observes
2599
+ # end-of-stream instead of waiting for its idle timeout.
2600
+ event_queue = request_state.event_queue
2601
+ if event_queue is not None:
2602
+ try:
2603
+ event_queue.put_nowait(None)
2604
+ except asyncio.QueueFull:
2605
+ pass
2606
+
2607
+ async def _handle_or_defer_precreated_stream_health(
2608
+ self: Any,
2609
+ request_state: _WebSocketRequestState | None,
2610
+ account: Account,
2611
+ error: UpstreamError,
2612
+ code: str,
2613
+ ) -> None:
2614
+ """Write account health now, or defer it until reservation settlement.
2615
+
2616
+ Keyed pre-created retry branches run while the request's API-key
2617
+ reservation is still open. An immediate ``_handle_stream_error`` would
2618
+ mutate account health before the reservation settles, violating the
2619
+ settlement-ordering invariant (api-keys spec: the health write waits
2620
+ for settlement, and unconfirmed settlement leaves health unapplied).
2621
+ Queue the classified write on the request state instead;
2622
+ ``_drain_deferred_keyed_stream_health`` applies it after settlement or
2623
+ fallback release commits. Unkeyed requests keep the immediate write.
2624
+ ``account_health_error_handled`` is set either way so terminal
2625
+ finalization does not apply a duplicate penalty.
2626
+ """
2627
+ if request_state is not None and request_state.api_key_reservation is not None:
2628
+ request_state.deferred_keyed_stream_health.append(
2629
+ _DeferredKeyedStreamHealthPenalty(account=account, error=error, code=code)
2630
+ )
2631
+ else:
2632
+ await self._handle_stream_error(account, error, code)
2633
+ if request_state is not None:
2634
+ setattr(request_state, "account_health_error_handled", True)
2635
+
2636
+ async def _process_parsed_http_bridge_upstream_event(
2637
+ self: Any,
2638
+ session: "_HTTPBridgeSession",
2639
+ *,
2640
+ text: str,
2641
+ event_block: str,
2642
+ payload: dict[str, JsonValue] | None,
2643
+ event: OpenAIEvent | None,
2644
+ event_type: str | None,
2645
+ response_id: str | None,
2646
+ completed_delivery_scope: _HTTPBridgeCompletedDeliveryScope | None,
2647
+ claimed_terminal_request_states: list[_WebSocketRequestState],
2648
+ scheduler: Scheduler,
2649
+ clock: Clock,
2650
+ ) -> None:
2651
+ original_text = text
2652
+ error_message = _websocket_event_error_message(event_type, payload)
2653
+ is_typeless_error_event = (
2654
+ isinstance(payload, dict)
2655
+ and not isinstance(payload.get("type"), str)
2656
+ and isinstance(payload.get("error"), dict)
2657
+ )
2658
+ is_previous_response_not_found_event = _is_previous_response_not_found_error(
2659
+ code=_normalize_error_code(
2660
+ _websocket_event_error_code(event_type, payload),
2661
+ _websocket_event_error_type(event_type, payload),
2662
+ ),
2663
+ param=_websocket_event_error_param(event_type, payload),
2664
+ message=error_message,
2665
+ )
2666
+ is_missing_tool_output_event = _is_missing_tool_output_error(
2667
+ code=_normalize_error_code(
2668
+ _websocket_event_error_code(event_type, payload),
2669
+ _websocket_event_error_type(event_type, payload),
2670
+ ),
2671
+ param=_websocket_event_error_param(event_type, payload),
2672
+ message=error_message,
2673
+ )
2674
+ previous_response_id_hint = _previous_response_id_from_not_found_message(error_message)
2675
+ text, payload, event, event_type, event_block = rewrite_parallel_tool_call_text(
2676
+ text,
2677
+ payload,
2678
+ event_block=event_block,
2679
+ event=event,
2680
+ )
2681
+
2682
+ completed_event_queue: asyncio.Queue[str | None] | None = None
2683
+ completed_event_queue_claimed = False
2684
+ async with session.pending_lock:
2685
+ matched_request_state = None
2686
+ created_request_state = None
2687
+ suppress_downstream_event = False
2688
+ deferred_reasoning_prelude_event = False
2689
+ has_other_pending_requests = False
2690
+ grouped_previous_response_request_states: list[_WebSocketRequestState] = []
2691
+ anonymous_event_prefers_draining = event_type not in {"response.failed", "response.incomplete", "error"}
2692
+ if event_type == "response.created":
2693
+ matched_request_state = _assign_websocket_response_id(session.pending_requests, response_id)
2694
+ created_request_state = matched_request_state
2695
+ release_create_gate = matched_request_state is not None
2696
+ elif response_id is not None:
2697
+ matched_request_state = _find_websocket_request_state_by_response_id(
2698
+ session.pending_requests,
2699
+ response_id,
2700
+ )
2701
+ release_create_gate = False
2702
+ elif response_id is None:
2703
+ matched_request_state = _match_websocket_request_state_for_anonymous_event(
2704
+ session.pending_requests,
2705
+ prefer_previous_response_not_found=is_previous_response_not_found_event
2706
+ or is_missing_tool_output_event,
2707
+ previous_response_id_hint=previous_response_id_hint,
2708
+ error_message=error_message,
2709
+ allow_unanchored_previous_response_error=is_previous_response_not_found_event,
2710
+ prefer_draining_requests=anonymous_event_prefers_draining,
2711
+ event_type=event_type,
2712
+ )
2713
+ release_create_gate = False
2714
+ else:
2715
+ release_create_gate = False
2716
+
2717
+ _archive_http_bridge_upstream_text(session, original_text, matched_request_state)
2718
+ pending_request_count = len(session.pending_requests)
2719
+
2720
+ if matched_request_state is not None:
2721
+ # The deferred reasoning prelude intentionally skips ordinary
2722
+ # response-event accounting below, but it still proves that the
2723
+ # physical response.create received an upstream response. Publish
2724
+ # that attempt transition before any later recovery await can
2725
+ # classify the send as eventless.
2726
+ _mark_response_create_attempt_observed(matched_request_state, event_type)
2727
+ session.last_upstream_event_generation += 1
2728
+ now = clock.monotonic()
2729
+ if matched_request_state.latency_first_upstream_event_ms is None:
2730
+ matched_request_state.latency_first_upstream_event_ms = int(
2731
+ max(0.0, now - matched_request_state.started_at) * 1000
2732
+ )
2733
+ if event_type == "response.created" and matched_request_state.latency_response_created_ms is None:
2734
+ matched_request_state.latency_response_created_ms = int(
2735
+ max(0.0, now - matched_request_state.started_at) * 1000
2736
+ )
2737
+ actual_service_tier = _service_tier_from_event_payload(payload)
2738
+ if actual_service_tier is not None:
2739
+ matched_request_state.actual_service_tier = actual_service_tier
2740
+ matched_request_state.service_tier = actual_service_tier
2741
+ _record_http_bridge_tool_call_lifecycle(
2742
+ matched_request_state,
2743
+ event_type=event_type,
2744
+ payload=payload,
2745
+ )
2746
+ completed_tool_call = _response_output_item_done_tool_call(payload)
2747
+ if completed_tool_call is not None:
2748
+ completed_call_id, completed_call_type = completed_tool_call
2749
+ if completed_call_id not in matched_request_state.pending_function_call_ids:
2750
+ matched_request_state.pending_function_call_ids.append(completed_call_id)
2751
+ matched_request_state.pending_tool_call_types[completed_call_id] = completed_call_type
2752
+ if mark_duplicate_tool_call_downstream_event(
2753
+ payload,
2754
+ seen_tool_call_keys=matched_request_state.seen_tool_call_keys,
2755
+ response_id=tool_call_response_id_from_payload(payload) or matched_request_state.request_id,
2756
+ scope_side_effects_by_response_id=False,
2757
+ ):
2758
+ matched_request_state.suppressed_duplicate_tool_call = True
2759
+ return
2760
+ if event_type in _TEXT_DELTA_EVENT_TYPES:
2761
+ matched_request_state.downstream_visible = True
2762
+ if event_type == "response.created" and matched_request_state.suppress_next_created_downstream:
2763
+ matched_request_state.suppress_next_created_downstream = False
2764
+ suppress_downstream_event = True
2765
+ elif (
2766
+ event_type == "response.in_progress" and matched_request_state.suppress_next_in_progress_downstream
2767
+ ):
2768
+ matched_request_state.suppress_next_in_progress_downstream = False
2769
+ suppress_downstream_event = True
2770
+ if payload is not None:
2771
+ rewritten_payload = _rewrite_websocket_downstream_response_id(payload, matched_request_state)
2772
+ # ``text`` is the serialization ``payload`` was parsed from (or the
2773
+ # tool-call rewrite's compact re-dump). Relaying it verbatim is
2774
+ # only valid while nothing above mutated ``payload`` in place:
2775
+ # ``mark_duplicate_tool_call_downstream_event`` trims partially
2776
+ # duplicated ``multi_tool_use.parallel`` arguments on
2777
+ # ``response.output_item.done`` items without returning a new
2778
+ # object, so those (rare, per-item) events are always
2779
+ # re-serialized. Every other pre-framing step is read-only.
2780
+ if rewritten_payload is not payload or event_type == "response.output_item.done":
2781
+ payload = rewritten_payload
2782
+ event_block = format_sse_event(payload)
2783
+ else:
2784
+ # Identity fast path: nothing changed, so frame the
2785
+ # upstream JSON text instead of serializing the dict again.
2786
+ event_block = format_sse_event_from_text(payload, text)
2787
+ if _websocket_should_defer_reasoning_prelude(matched_request_state, event_type, payload):
2788
+ matched_request_state.deferred_reasoning_downstream_texts.append(event_block)
2789
+ matched_request_state.last_upstream_activity_at = now
2790
+ matched_request_state.upstream_model_output_seen = True
2791
+ suppress_downstream_event = True
2792
+ deferred_reasoning_prelude_event = True
2793
+ elif event_type in _MODEL_OUTPUT_EVENT_TYPES:
2794
+ matched_request_state.upstream_model_output_seen = True
2795
+ if not suppress_downstream_event:
2796
+ matched_request_state.downstream_visible = True
2797
+
2798
+ terminal_request_state = None
2799
+ if event_type in {"response.completed", "response.failed", "response.incomplete", "error"}:
2800
+ early_retry_error_code = _websocket_precreated_retry_error_code(
2801
+ matched_request_state,
2802
+ event_type=event_type,
2803
+ payload=payload,
2804
+ has_other_pending_requests=any(
2805
+ pending_request is not matched_request_state for pending_request in session.pending_requests
2806
+ ),
2807
+ )
2808
+ reserve_terminal_for_model_capacity_retry = bool(
2809
+ matched_request_state is not None
2810
+ and early_retry_error_code is not None
2811
+ and early_retry_error_code != _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
2812
+ and not is_previous_response_not_found_event
2813
+ and is_upstream_model_capacity_error(error_message)
2814
+ and _websocket_request_can_replay_before_visible_output(matched_request_state)
2815
+ and _http_bridge_accepted_capacity_retry_allowed(matched_request_state)
2816
+ )
2817
+ if reserve_terminal_for_model_capacity_retry:
2818
+ terminal_request_state = matched_request_state
2819
+ else:
2820
+ terminal_request_state = _pop_terminal_websocket_request_state(
2821
+ session.pending_requests,
2822
+ response_id=response_id,
2823
+ fallback_request_state=matched_request_state,
2824
+ prefer_previous_response_not_found=is_previous_response_not_found_event
2825
+ or is_missing_tool_output_event,
2826
+ previous_response_id_hint=previous_response_id_hint,
2827
+ error_message=error_message,
2828
+ allow_unanchored_previous_response_error=is_previous_response_not_found_event,
2829
+ allow_precreated_terminal_fallback=True,
2830
+ prefer_draining_requests=anonymous_event_prefers_draining,
2831
+ )
2832
+ if terminal_request_state is not None:
2833
+ # Upstream generation ends here; the durable alias, operation,
2834
+ # recovery and circuit-settlement writes below and the
2835
+ # finalizer's settlement are local and must not stretch the
2836
+ # throughput sample's span. A later terminal for the same turn
2837
+ # (capacity retry) replaces it; those rows are not sampled.
2838
+ terminal_request_state.upstream_terminal_at = clock.monotonic()
2839
+ if (
2840
+ matched_request_state is None
2841
+ and terminal_request_state is not None
2842
+ and response_id is not None
2843
+ and event_type == "response.completed"
2844
+ and terminal_request_state.response_id is None
2845
+ ):
2846
+ terminal_request_state.response_id = response_id
2847
+ matched_request_state = terminal_request_state
2848
+ elif (
2849
+ matched_request_state is None
2850
+ and terminal_request_state is not None
2851
+ and response_id is not None
2852
+ and terminal_request_state.response_id == response_id
2853
+ ):
2854
+ matched_request_state = terminal_request_state
2855
+ if (
2856
+ terminal_request_state is not None
2857
+ and not reserve_terminal_for_model_capacity_retry
2858
+ and _http_bridge_request_counts_against_queue(terminal_request_state)
2859
+ ):
2860
+ session.queued_request_count = max(0, session.queued_request_count - 1)
2861
+ elif is_previous_response_not_found_event or is_missing_tool_output_event:
2862
+ grouped_previous_response_request_states = _pop_matching_websocket_request_states(
2863
+ session.pending_requests,
2864
+ _matching_websocket_request_states_for_previous_response_error(
2865
+ session.pending_requests,
2866
+ previous_response_id_hint=previous_response_id_hint,
2867
+ error_message=error_message,
2868
+ allow_unanchored_previous_response_error=is_previous_response_not_found_event,
2869
+ ),
2870
+ )
2871
+ if not grouped_previous_response_request_states and is_missing_tool_output_event:
2872
+ grouped_previous_response_request_states = _pop_matching_websocket_request_states(
2873
+ session.pending_requests,
2874
+ _matching_websocket_request_states_for_missing_tool_output_error(
2875
+ session.pending_requests,
2876
+ ),
2877
+ )
2878
+ if grouped_previous_response_request_states:
2879
+ grouped_counted_requests = sum(
2880
+ 1
2881
+ for grouped_request_state in grouped_previous_response_request_states
2882
+ if _http_bridge_request_counts_against_queue(grouped_request_state)
2883
+ )
2884
+ session.queued_request_count = max(
2885
+ 0,
2886
+ session.queued_request_count - grouped_counted_requests,
2887
+ )
2888
+ if (
2889
+ terminal_request_state is None
2890
+ and event_type == "error"
2891
+ and is_typeless_error_event
2892
+ and not grouped_previous_response_request_states
2893
+ ):
2894
+ grouped_previous_response_request_states = list(session.pending_requests)
2895
+ session.pending_requests.clear()
2896
+ if grouped_previous_response_request_states:
2897
+ grouped_counted_requests = sum(
2898
+ 1
2899
+ for grouped_request_state in grouped_previous_response_request_states
2900
+ if _http_bridge_request_counts_against_queue(grouped_request_state)
2901
+ )
2902
+ session.queued_request_count = max(
2903
+ 0,
2904
+ session.queued_request_count - grouped_counted_requests,
2905
+ )
2906
+ has_other_pending_requests = any(
2907
+ pending_request is not terminal_request_state for pending_request in session.pending_requests
2908
+ )
2909
+ if (
2910
+ event_type == "response.completed"
2911
+ and terminal_request_state is not None
2912
+ and terminal_request_state not in session.pending_requests
2913
+ ):
2914
+ completed_event_queue = terminal_request_state.event_queue
2915
+ completed_event_queue_claimed = True
2916
+ if completed_event_queue is not None and completed_delivery_scope is not None:
2917
+ completed_delivery_scope.active = True
2918
+ terminal_request_state.completed_delivery_scope = completed_delivery_scope
2919
+
2920
+ # Every request popped from pending ownership above is now owned
2921
+ # exclusively by this bookkeeping continuation. Record the claim
2922
+ # (still under pending_lock) so an abort before finalization can
2923
+ # settle the reservation instead of leaking it (issue #1594).
2924
+ for claimed_request_state in (terminal_request_state, *grouped_previous_response_request_states):
2925
+ if claimed_request_state is not None and claimed_request_state not in session.pending_requests:
2926
+ claimed_request_state.terminal_settlement_phase = "claimed"
2927
+ if claimed_request_state not in claimed_terminal_request_states:
2928
+ claimed_terminal_request_states.append(claimed_request_state)
2929
+
2930
+ if len(grouped_previous_response_request_states) > 1:
2931
+ session.upstream_control.reconnect_requested = True
2932
+ if is_previous_response_not_found_event:
2933
+ # This branch settles every request that shared the denied
2934
+ # anchor and then returns, so the single-request retirement
2935
+ # below is never reached for a fan-out denial.
2936
+ await _retire_denied_http_bridge_anchor(
2937
+ self,
2938
+ session,
2939
+ request_states=grouped_previous_response_request_states,
2940
+ )
2941
+ grouped_error_reason = (
2942
+ "previous_response_not_found"
2943
+ if is_previous_response_not_found_event
2944
+ else "missing_tool_output"
2945
+ if is_missing_tool_output_event
2946
+ else "stream_incomplete"
2947
+ )
2948
+ grouped_terminal_events = []
2949
+ for grouped_request_state in grouped_previous_response_request_states:
2950
+ grouped_request_state.error_http_status_override = 502
2951
+ (
2952
+ _grouped_downstream_text,
2953
+ grouped_event_block,
2954
+ grouped_event,
2955
+ grouped_payload,
2956
+ grouped_event_type,
2957
+ ) = _build_stream_incomplete_terminal_event_for_request(
2958
+ grouped_request_state,
2959
+ reason=grouped_error_reason,
2960
+ )
2961
+ grouped_operation_state = _http_bridge_operation_state_for_event(grouped_event_type)
2962
+ grouped_terminal_events.append(
2963
+ (
2964
+ grouped_request_state,
2965
+ grouped_event_block,
2966
+ grouped_event,
2967
+ grouped_payload,
2968
+ grouped_event_type,
2969
+ grouped_operation_state,
2970
+ )
2971
+ )
2972
+
2973
+ # This grouped settlement returns before the single-request
2974
+ # settlement path below, so without recording here a multi-request
2975
+ # continuity failure fails every grouped request with a synthetic
2976
+ # terminal event and never advances the circuit — leaving the anchor
2977
+ # that failed them reusable. The detail is read back off the built
2978
+ # event so it matches whatever the single-request path would have
2979
+ # recorded for the same reason, and recording precedes the persist
2980
+ # and delivery machinery below for the same reason it does there: a
2981
+ # client resending on observed completion must not outrun it.
2982
+ grouped_poison_strike_failures = 0
2983
+ grouped_poison_detail: str | None = None
2984
+ for (
2985
+ grouped_request_state,
2986
+ _grouped_terminal_block,
2987
+ grouped_terminal_event,
2988
+ _grouped_terminal_payload,
2989
+ _grouped_terminal_event_type,
2990
+ _grouped_terminal_operation_state,
2991
+ ) in grouped_terminal_events:
2992
+ # Same admission test the single-request settlement path
2993
+ # applies: a request still holding a verified full resend is
2994
+ # about to be replayed and claims the circuit generation at
2995
+ # dispatch, so charging it here would let two safely
2996
+ # replayable requests open the circuit between them and clear
2997
+ # the anchor both of them could still have used.
2998
+ if grouped_request_state.response_event_count != 0 or _http_bridge_request_state_holds_safe_replay(
2999
+ grouped_request_state
3000
+ ):
3001
+ continue
3002
+ if grouped_request_state.request_kind == "prewarm" or grouped_request_state.skip_request_log:
3003
+ # Same exclusion the single-request terminal branch
3004
+ # applies: an internal warmup probe carries no anchor
3005
+ # and proves nothing about the key's continuity, and
3006
+ # one warmup plus one client request must not open the
3007
+ # circuit between them after a single real failure.
3008
+ continue
3009
+ grouped_terminal_error = (
3010
+ grouped_terminal_event.response.error
3011
+ if grouped_terminal_event is not None and grouped_terminal_event.response is not None
3012
+ else None
3013
+ )
3014
+ grouped_terminal_detail = _normalize_error_code(
3015
+ grouped_terminal_error.code if grouped_terminal_error else None,
3016
+ grouped_terminal_error.type if grouped_terminal_error else None,
3017
+ )
3018
+ if grouped_terminal_detail is None:
3019
+ continue
3020
+ grouped_strike_failures = await self._record_http_bridge_retry_circuit_failure(
3021
+ session,
3022
+ detail=grouped_terminal_detail,
3023
+ attempt=grouped_request_state.response_create_attempt,
3024
+ terminal_pre_response_frame=True,
3025
+ )
3026
+ # A fan-out carrying two or more eventless requests advances the
3027
+ # circuit through its threshold inside this loop, so the anchor
3028
+ # that failed all of them is proven dead here just as it is on
3029
+ # the single-request path. Keep the highest count and its poison
3030
+ # detail; discarding them left the grouped branch returning with
3031
+ # the durable anchor still stored.
3032
+ grouped_poison_candidate = await self._http_bridge_effective_anchor_poison_detail(
3033
+ session, grouped_terminal_detail
3034
+ )
3035
+ if grouped_poison_candidate is not None and grouped_strike_failures is not None:
3036
+ if grouped_strike_failures > grouped_poison_strike_failures:
3037
+ grouped_poison_strike_failures = grouped_strike_failures
3038
+ grouped_poison_detail = grouped_poison_candidate
3039
+
3040
+ append_terminal_batch = getattr(
3041
+ getattr(self, "_http_bridge_operation_event_batcher", None),
3042
+ "append_terminal_event",
3043
+ None,
3044
+ )
3045
+ append_participants = {
3046
+ id(grouped_request_state)
3047
+ for grouped_request_state, *_rest in grouped_terminal_events
3048
+ if grouped_request_state.operation_id
3049
+ and session.durable_session_id is not None
3050
+ and session.durable_owner_epoch is not None
3051
+ and callable(append_terminal_batch)
3052
+ }
3053
+ append_ready = asyncio.Event()
3054
+ append_lock = asyncio.Lock()
3055
+ append_arrivals = 0
3056
+ if not append_participants:
3057
+ append_ready.set()
3058
+
3059
+ async def await_all_grouped_appends() -> None:
3060
+ nonlocal append_arrivals
3061
+ async with append_lock:
3062
+ append_arrivals += 1
3063
+ if append_arrivals == len(append_participants):
3064
+ append_ready.set()
3065
+ await append_ready.wait()
3066
+
3067
+ delivery_ready = asyncio.Event()
3068
+ delivery_lock = asyncio.Lock()
3069
+ delivery_arrivals = 0
3070
+
3071
+ async def await_all_grouped_deliveries() -> None:
3072
+ nonlocal delivery_arrivals
3073
+ async with delivery_lock:
3074
+ delivery_arrivals += 1
3075
+ if delivery_arrivals == len(grouped_terminal_events):
3076
+ delivery_ready.set()
3077
+ await delivery_ready.wait()
3078
+
3079
+ async def persist_one_grouped_terminal_event(
3080
+ grouped_terminal_event: tuple[Any, str, OpenAIEvent | None, Any, str | None, str | None],
3081
+ ) -> None:
3082
+ (
3083
+ grouped_request_state,
3084
+ grouped_event_block,
3085
+ _grouped_event,
3086
+ _grouped_payload,
3087
+ _grouped_event_type,
3088
+ grouped_operation_state,
3089
+ ) = grouped_terminal_event
3090
+ if id(grouped_request_state) in append_participants:
3091
+ await _persist_http_bridge_operation_event(
3092
+ self,
3093
+ session,
3094
+ grouped_request_state,
3095
+ grouped_event_block,
3096
+ terminal=True,
3097
+ terminal_state=grouped_operation_state,
3098
+ terminal_event_queue=grouped_request_state.event_queue,
3099
+ terminal_append_barrier=await_all_grouped_appends,
3100
+ terminal_delivery_barrier=await_all_grouped_deliveries,
3101
+ )
3102
+ else:
3103
+ await append_ready.wait()
3104
+ if grouped_request_state.event_queue is not None:
3105
+ await grouped_request_state.event_queue.put(grouped_event_block)
3106
+ await grouped_request_state.event_queue.put(None)
3107
+ await await_all_grouped_deliveries()
3108
+ await _persist_http_bridge_operation_event(
3109
+ self,
3110
+ session,
3111
+ grouped_request_state,
3112
+ grouped_event_block,
3113
+ terminal=True,
3114
+ terminal_state=grouped_operation_state,
3115
+ )
3116
+ if grouped_operation_state is not None and grouped_operation_state != "failed":
3117
+ await _update_http_bridge_operation_state(
3118
+ self,
3119
+ session,
3120
+ grouped_request_state,
3121
+ state=grouped_operation_state,
3122
+ response_id=_websocket_downstream_response_id(grouped_request_state),
3123
+ )
3124
+
3125
+ async def persist_grouped_terminal_events() -> Exception | None:
3126
+ first_error: Exception | None = None
3127
+ # Spawned through the scheduler (``gather`` would ``ensure_future``
3128
+ # each coroutine into an unowned task) so a simulation owns the
3129
+ # per-request persistence work of a grouped terminal.
3130
+ persistence_results = await asyncio.gather(
3131
+ *(
3132
+ scheduler.create_task(
3133
+ persist_one_grouped_terminal_event(item),
3134
+ name=f"http-bridge-grouped-terminal-persist-{session.durable_session_id}",
3135
+ )
3136
+ for item in grouped_terminal_events
3137
+ ),
3138
+ return_exceptions=True,
3139
+ )
3140
+ for persistence_result in persistence_results:
3141
+ if isinstance(persistence_result, Exception) and first_error is None:
3142
+ first_error = persistence_result
3143
+ try:
3144
+ for (
3145
+ grouped_request_state,
3146
+ _grouped_event_block,
3147
+ grouped_event,
3148
+ grouped_payload,
3149
+ grouped_event_type,
3150
+ _grouped_operation_state,
3151
+ ) in grouped_terminal_events:
3152
+ try:
3153
+ await self._finalize_websocket_request_state(
3154
+ grouped_request_state,
3155
+ account=session.account,
3156
+ account_id_value=session.account.id,
3157
+ event=grouped_event,
3158
+ event_type=grouped_event_type,
3159
+ payload=grouped_payload,
3160
+ api_key=grouped_request_state.api_key,
3161
+ upstream_control=session.upstream_control,
3162
+ response_create_gate=session.response_create_gate,
3163
+ )
3164
+ except Exception as exc:
3165
+ if first_error is None:
3166
+ first_error = exc
3167
+ except Exception as exc:
3168
+ if first_error is None:
3169
+ first_error = exc
3170
+ try:
3171
+ # Grouped terminal errors settle detached/abandoned requests
3172
+ # (event_queue is None) with no downstream stream finalizer
3173
+ # left to run, so release the now-idle session's account
3174
+ # stream lease here just like the single terminal path below.
3175
+ await self._maybe_release_idle_http_bridge_session_lease(session)
3176
+ except Exception as exc:
3177
+ if first_error is None:
3178
+ first_error = exc
3179
+ return first_error
3180
+
3181
+ grouped_settlement_task = scheduler.create_task(
3182
+ persist_grouped_terminal_events(),
3183
+ name=f"http-bridge-grouped-terminal-settlement-{session.durable_session_id}",
3184
+ )
3185
+ grouped_error, grouped_cancellation = await _await_task_deferring_cancellation(grouped_settlement_task)
3186
+ if grouped_poison_detail is not None and grouped_poison_strike_failures > 0:
3187
+ # The grouped frames have been published by here, so this mirrors
3188
+ # the single-request ordering: the strikes precede delivery, the
3189
+ # durable clear follows it.
3190
+ #
3191
+ # This runs under cancellation too. `_await_task_deferring_
3192
+ # cancellation` has already waited for every grouped frame and
3193
+ # finalization, so the requests this anchor failed are gone and
3194
+ # no later retirement can retry the clear for them; skipping it
3195
+ # here left the poisoned anchor stored for another replica or
3196
+ # for reuse once the local quarantine expired. Defer the
3197
+ # cancellation across the write the same way, then re-raise it
3198
+ # below unchanged. The clear is internally exception-safe.
3199
+ grouped_clear_detail = grouped_poison_detail
3200
+ grouped_clear_strike_failures = grouped_poison_strike_failures
3201
+
3202
+ async def _consult_and_clear_grouped_anchor() -> None:
3203
+ # The episode consult is a durable await of its own: run
3204
+ # it inside the same cancellation-deferred task as the
3205
+ # clear, or a reader cancellation landing mid-consult
3206
+ # escapes with the grouped requests already finalized and
3207
+ # no retirement left to retry the abandonment. The
3208
+ # consult reads the live registered episode, so a
3209
+ # sibling settle during grouped publication vetoes the
3210
+ # clear.
3211
+ grouped_poison_episode, grouped_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
3212
+ session,
3213
+ consecutive_failures=grouped_clear_strike_failures,
3214
+ )
3215
+ if grouped_poison_episode is None:
3216
+ return
3217
+ durable_cleared = await _abandon_durable_http_bridge_continuity(
3218
+ self,
3219
+ session,
3220
+ detail=grouped_clear_detail,
3221
+ expected_continuity=grouped_expected_anchor,
3222
+ authorized_episode=grouped_poison_episode,
3223
+ # A mixed group can hold a member with a verified safe
3224
+ # replay whose dispatch claims the circuit generation;
3225
+ # settling under it would remove the fence it depends
3226
+ # on, same as the retirement funnels.
3227
+ settle_circuit=_http_bridge_abandonment_may_settle_circuit(
3228
+ grouped_previous_response_request_states
3229
+ ),
3230
+ )
3231
+ if not durable_cleared:
3232
+ return
3233
+ # A mixed group's abandonment deliberately leaves the
3234
+ # circuit alive for its safe member, so the episode
3235
+ # survives and must remember its anchor was already
3236
+ # cleared — one poisoned anchor is abandoned once. The
3237
+ # marker belongs inside this owned task: a cancellation
3238
+ # landing between the durable clear and the marker write
3239
+ # would otherwise leave the cleared episode unmarked, and
3240
+ # an ordinary load would re-arm quarantine from the
3241
+ # unchanged surviving row after the safe replay had
3242
+ # already recovered the conversation.
3243
+ await self._http_bridge_mark_poison_anchor_cleared(session, episode=grouped_poison_episode)
3244
+
3245
+ grouped_clear_task = scheduler.create_task(
3246
+ _consult_and_clear_grouped_anchor(),
3247
+ name=f"http-bridge-grouped-anchor-clear-{session.durable_session_id}",
3248
+ )
3249
+ _grouped_clear_result, grouped_clear_cancellation = await _await_task_deferring_cancellation(
3250
+ grouped_clear_task
3251
+ )
3252
+ if grouped_cancellation is None:
3253
+ grouped_cancellation = grouped_clear_cancellation
3254
+ if grouped_cancellation is not None:
3255
+ if grouped_error is not None:
3256
+ logger.warning(
3257
+ "Grouped HTTP bridge terminal finalization failed while preserving cancellation error=%r",
3258
+ grouped_error,
3259
+ )
3260
+ raise grouped_cancellation
3261
+ if grouped_error is not None:
3262
+ raise grouped_error
3263
+ return
3264
+
3265
+ if len(grouped_previous_response_request_states) == 1 and terminal_request_state is None:
3266
+ terminal_request_state = grouped_previous_response_request_states[0]
3267
+
3268
+ if not deferred_reasoning_prelude_event:
3269
+ if matched_request_state is terminal_request_state:
3270
+ _record_response_event(matched_request_state, event_type, now=clock.monotonic())
3271
+ else:
3272
+ _record_response_event(matched_request_state, event_type, now=clock.monotonic())
3273
+ _record_response_event(terminal_request_state, event_type, now=clock.monotonic())
3274
+
3275
+ status_request_state = terminal_request_state or matched_request_state
3276
+ if status_request_state is None and is_previous_response_not_found_event:
3277
+ session.upstream_control.reconnect_requested = True
3278
+ return
3279
+
3280
+ if status_request_state is None and pending_request_count:
3281
+ # The bridge multiplexes one upstream connection across pending
3282
+ # requests. An event that reaches none of them is dropped here, and
3283
+ # whatever was waiting for it waits until a timeout fires, so the
3284
+ # drop needs to be visible rather than inferred from a missing
3285
+ # downstream response.
3286
+ #
3287
+ # The frame still proves the upstream transport is alive. Nothing
3288
+ # resets the downstream pre-response silence clock from here (that
3289
+ # clock only sees matched queue items), so record the liveness as an
3290
+ # explicit marker: a later bridge_eventless_timeout with a non-zero
3291
+ # count is a local matching wedge, not a silent upstream.
3292
+ unmatched_liveness_count = _record_http_bridge_unmatched_upstream_liveness(
3293
+ session,
3294
+ event_type=event_type,
3295
+ )
3296
+ logger.warning(
3297
+ "HTTP bridge upstream event matched no pending request account_id=%s bridge_kind=%s "
3298
+ "event_type=%s has_response_id=%s pending_count=%d unmatched_upstream_liveness=%d",
3299
+ session.account.id,
3300
+ session.key.affinity_kind,
3301
+ event_type or "unknown",
3302
+ response_id is not None,
3303
+ pending_request_count,
3304
+ unmatched_liveness_count,
3305
+ )
3306
+ if _http_bridge_event_proves_upstream_liveness(event_type):
3307
+ _log_http_bridge_event(
3308
+ "unmatched_upstream_liveness",
3309
+ session.key,
3310
+ account_id=session.account.id,
3311
+ model=session.request_model,
3312
+ pending_count=pending_request_count,
3313
+ detail=(
3314
+ f"event_type={event_type or 'unknown'} unmatched_upstream_liveness={unmatched_liveness_count}"
3315
+ ),
3316
+ cache_key_family=session.key.affinity_kind,
3317
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
3318
+ )
3319
+
3320
+ if status_request_state is not None and event_type not in {
3321
+ "response.completed",
3322
+ "response.failed",
3323
+ "response.incomplete",
3324
+ "error",
3325
+ }:
3326
+ await self._maybe_touch_request_state_api_key_reservation(
3327
+ status_request_state,
3328
+ api_key=status_request_state.api_key,
3329
+ surface="http_bridge",
3330
+ )
3331
+
3332
+ continuity_persistence_failed_after_ack = False
3333
+ if (
3334
+ event_type == "response.completed"
3335
+ and terminal_request_state is not None
3336
+ and terminal_request_state.suppressed_duplicate_tool_call
3337
+ ):
3338
+ session.upstream_control.reconnect_requested = True
3339
+ session.closed = True
3340
+ try:
3341
+ await session.upstream.close()
3342
+ except Exception:
3343
+ logger.debug("Failed to close HTTP bridge upstream after suppressed duplicate tool call", exc_info=True)
3344
+ terminal_request_state.error_http_status_override = 502
3345
+ (
3346
+ event,
3347
+ payload,
3348
+ event_type,
3349
+ rewritten_text,
3350
+ ) = _rewrite_websocket_suppressed_duplicate_tool_call_completion_event(
3351
+ request_state=terminal_request_state,
3352
+ )
3353
+ event_block = f"data: {rewritten_text}\n\n"
3354
+
3355
+ if (
3356
+ status_request_state is not None
3357
+ and status_request_state.previous_response_id is not None
3358
+ and is_missing_tool_output_event
3359
+ ):
3360
+ status_request_state.error_http_status_override = 502
3361
+ event, payload, event_type, rewritten_text = _rewrite_websocket_continuity_corruption_event(
3362
+ request_state=status_request_state,
3363
+ upstream_control=session.upstream_control,
3364
+ reason="missing_tool_output",
3365
+ reconnect_requested=True,
3366
+ original_text=text,
3367
+ )
3368
+ event_block = f"data: {rewritten_text}\n\n"
3369
+
3370
+ if status_request_state is not None and is_previous_response_not_found_event:
3371
+ status_request_state.error_http_status_override = 502
3372
+ status_request_state.previous_response_not_found_rewritten = (
3373
+ response_id is None and not has_other_pending_requests
3374
+ )
3375
+ event, payload, event_type, rewritten_text = _maybe_rewrite_websocket_previous_response_not_found_event(
3376
+ request_state=status_request_state,
3377
+ event=event,
3378
+ payload=payload,
3379
+ event_type=event_type,
3380
+ upstream_control=session.upstream_control,
3381
+ original_text=text,
3382
+ )
3383
+ event_block = f"data: {rewritten_text}\n\n"
3384
+ await _retire_denied_http_bridge_anchor(
3385
+ self,
3386
+ session,
3387
+ request_states=(status_request_state,),
3388
+ )
3389
+
3390
+ retry_error_code = _websocket_precreated_retry_error_code(
3391
+ status_request_state,
3392
+ event_type=event_type,
3393
+ payload=payload,
3394
+ has_other_pending_requests=has_other_pending_requests,
3395
+ )
3396
+ auth_error_code = _websocket_precreated_auth_error_code(
3397
+ status_request_state,
3398
+ event_type=event_type,
3399
+ payload=payload,
3400
+ has_other_pending_requests=has_other_pending_requests,
3401
+ )
3402
+ owner_pinned_quota_error = _websocket_owner_pinned_quota_error_code(
3403
+ status_request_state,
3404
+ event_type=event_type,
3405
+ payload=payload,
3406
+ )
3407
+ retry_error_message = _websocket_event_error_message(event_type, payload)
3408
+ # An accepted anchored request waits only when the pre-created retry
3409
+ # can actually re-send it (proxy-injected anchor); a client-supplied
3410
+ # anchor falls through to the transparent-code branch, which forwards
3411
+ # the capacity terminal unchanged instead of staging a refused retry.
3412
+ wait_for_model_capacity_retry = bool(
3413
+ retry_error_code is not None
3414
+ and retry_error_code != _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
3415
+ and not is_previous_response_not_found_event
3416
+ and status_request_state is not None
3417
+ and is_upstream_model_capacity_error(retry_error_message)
3418
+ and _websocket_request_can_replay_before_visible_output(status_request_state)
3419
+ and _http_bridge_accepted_capacity_retry_allowed(status_request_state)
3420
+ )
3421
+ if (
3422
+ auth_error_code is not None
3423
+ and not is_previous_response_not_found_event
3424
+ and status_request_state is not None
3425
+ ):
3426
+ auth_retry_result = await self._retry_http_bridge_precreated_auth_request(
3427
+ session,
3428
+ status_request_state,
3429
+ error_message=_websocket_event_error_message(event_type, payload),
3430
+ )
3431
+ if auth_retry_result == "retried":
3432
+ return
3433
+ if auth_retry_result == "failed":
3434
+ async with session.pending_lock:
3435
+ if status_request_state in session.pending_requests:
3436
+ session.pending_requests.remove(status_request_state)
3437
+ session.queued_request_count = max(0, session.queued_request_count - 1)
3438
+ if is_http_bridge_account_neutral_replay(
3439
+ kind=session.key.affinity_kind,
3440
+ key=session.key.affinity_key,
3441
+ ):
3442
+ _clear_websocket_request_error_overrides(status_request_state)
3443
+ else:
3444
+ status_request_state.error_http_status_override = 502
3445
+ (
3446
+ _downstream_text,
3447
+ event_block,
3448
+ event,
3449
+ payload,
3450
+ event_type,
3451
+ ) = _build_stream_incomplete_terminal_event_for_request(status_request_state)
3452
+ elif wait_for_model_capacity_retry and status_request_state is not None and retry_error_code is not None:
3453
+ # Reserve the terminal request again before any await so a younger
3454
+ # submit cannot claim its queue slot while account health is being
3455
+ # updated. A concurrent detach will mark this state as draining.
3456
+ retry_consumer_attached = False
3457
+ # An accepted request has already committed its response start,
3458
+ # so the pre-response startup-wait signals must not fire for it.
3459
+ accepted_lifecycle_replay = (
3460
+ status_request_state.response_id is not None and not status_request_state.awaiting_response_created
3461
+ )
3462
+ async with session.pending_lock:
3463
+ if status_request_state.event_queue is not None:
3464
+ retry_consumer_attached = True
3465
+ if status_request_state not in session.pending_requests:
3466
+ session.pending_requests.appendleft(status_request_state)
3467
+ session.queued_request_count += 1
3468
+ if not await _stage_websocket_request_state_for_replay(
3469
+ status_request_state,
3470
+ create_gate=session.response_create_gate,
3471
+ surface="http_bridge",
3472
+ trigger="capacity_error",
3473
+ ):
3474
+ retry_consumer_attached = False
3475
+ if status_request_state.propagate_http_errors and not accepted_lifecycle_replay:
3476
+ _signal_http_bridge_capacity_startup_wait(status_request_state)
3477
+ await self._handle_or_defer_precreated_stream_health(
3478
+ status_request_state,
3479
+ session.account,
3480
+ {"message": retry_error_message or "Upstream error"},
3481
+ retry_error_code,
3482
+ )
3483
+ retry_consumer_attached = (
3484
+ retry_consumer_attached
3485
+ and status_request_state.event_queue is not None
3486
+ and not status_request_state.draining_until_terminal
3487
+ )
3488
+ if retry_consumer_attached:
3489
+ wait_request_had_event_queue = True
3490
+ if (
3491
+ status_request_state.response_create_admission is not None
3492
+ or status_request_state.account_response_create_lease is not None
3493
+ ):
3494
+ await _release_http_bridge_model_capacity_retry_admission(status_request_state)
3495
+ status_request_state.awaiting_response_created = True
3496
+ retry_after_wait = await _wait_before_http_bridge_model_capacity_retry(
3497
+ status_request_state,
3498
+ emit_keepalives=not status_request_state.propagate_http_errors,
3499
+ error_message=retry_error_message,
3500
+ cancel_when_detached=True,
3501
+ signal_startup_wait=not accepted_lifecycle_replay,
3502
+ scheduler=scheduler,
3503
+ clock=clock,
3504
+ )
3505
+ if wait_request_had_event_queue and status_request_state.event_queue is None:
3506
+ retry_after_wait = False
3507
+ suppress_capacity_keepalives_until_retry_finishes = (
3508
+ status_request_state.account_capacity_wait_suppress_keepalive
3509
+ )
3510
+ try:
3511
+ retried = retry_after_wait and await self._retry_http_bridge_precreated_request(
3512
+ session,
3513
+ request_state=status_request_state,
3514
+ )
3515
+ if retried:
3516
+ if not accepted_lifecycle_replay:
3517
+ _signal_http_bridge_model_capacity_retry_ready(
3518
+ status_request_state,
3519
+ waited_for_model_capacity_retry=True,
3520
+ retried=True,
3521
+ )
3522
+ return
3523
+ finally:
3524
+ if suppress_capacity_keepalives_until_retry_finishes:
3525
+ status_request_state.account_capacity_wait_suppress_keepalive = False
3526
+ async with session.pending_lock:
3527
+ _relinquish_http_bridge_capacity_wait_ownership(
3528
+ session, status_request_state, claimed_terminal_request_states
3529
+ )
3530
+ if retry_after_wait or not status_request_state.propagate_http_errors:
3531
+ status_request_state.error_http_status_override = 502
3532
+ (
3533
+ _downstream_text,
3534
+ event_block,
3535
+ event,
3536
+ payload,
3537
+ event_type,
3538
+ ) = _build_stream_incomplete_terminal_event_for_request(status_request_state)
3539
+ else:
3540
+ async with session.pending_lock:
3541
+ _relinquish_http_bridge_capacity_wait_ownership(
3542
+ session, status_request_state, claimed_terminal_request_states
3543
+ )
3544
+ elif owner_pinned_quota_error is not None and not is_previous_response_not_found_event:
3545
+ await self._handle_or_defer_precreated_stream_health(
3546
+ status_request_state,
3547
+ session.account,
3548
+ {"message": retry_error_message or "Upstream error"},
3549
+ owner_pinned_quota_error,
3550
+ )
3551
+ if (
3552
+ status_request_state is not None
3553
+ and status_request_state.previous_response_id is not None
3554
+ and status_request_state.preferred_account_id is not None
3555
+ ):
3556
+ safe_request_text = _prepare_websocket_request_state_for_account_switch(status_request_state)
3557
+ if safe_request_text is not None:
3558
+ previous_upstream_turn_state = session.upstream_turn_state
3559
+ previous_downstream_turn_state = session.downstream_turn_state
3560
+ session.upstream_turn_state = None
3561
+ session.downstream_turn_state = None
3562
+ await self._release_request_state_account_response_create_lease(status_request_state)
3563
+ status_request_state.excluded_account_ids.add(session.account.id)
3564
+ status_request_state.affinity_policy = replace(
3565
+ status_request_state.affinity_policy,
3566
+ reallocate_sticky=True,
3567
+ )
3568
+ status_request_state.request_text = safe_request_text
3569
+ async with session.pending_lock:
3570
+ if status_request_state not in session.pending_requests:
3571
+ session.pending_requests.appendleft(status_request_state)
3572
+ session.queued_request_count += 1
3573
+ status_request_state.awaiting_response_created = True
3574
+ status_request_state.response_id = None
3575
+ retried = await self._retry_http_bridge_precreated_request(session)
3576
+ if retried:
3577
+ return
3578
+ session.upstream_turn_state = previous_upstream_turn_state
3579
+ session.downstream_turn_state = previous_downstream_turn_state
3580
+ async with session.pending_lock:
3581
+ if status_request_state in session.pending_requests:
3582
+ session.pending_requests.remove(status_request_state)
3583
+ session.queued_request_count = max(0, session.queued_request_count - 1)
3584
+ status_request_state.error_http_status_override = 502
3585
+ (
3586
+ _downstream_text,
3587
+ event_block,
3588
+ event,
3589
+ payload,
3590
+ event_type,
3591
+ ) = _build_stream_incomplete_terminal_event_for_request(status_request_state)
3592
+ else:
3593
+ status_request_state.error_http_status_override = 502
3594
+ session.upstream_control.reconnect_requested = True
3595
+ session.upstream_control.retire_after_drain = True
3596
+ event, payload, event_type, rewritten_text = (
3597
+ _rewrite_websocket_previous_response_owner_unavailable_event(
3598
+ request_state=status_request_state,
3599
+ )
3600
+ )
3601
+ event_block = f"data: {rewritten_text}\n\n"
3602
+ elif (
3603
+ retry_error_code == _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
3604
+ and not is_previous_response_not_found_event
3605
+ and status_request_state is not None
3606
+ and _websocket_auth_request_can_switch_account(status_request_state)
3607
+ ):
3608
+ rejected_account_id = session.account.id
3609
+ status_request_state.precreated_replay_reason = _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
3610
+ status_request_state.precreated_replay_account_id = rejected_account_id
3611
+ previous_upstream_turn_state = session.upstream_turn_state
3612
+ previous_downstream_turn_state = session.downstream_turn_state
3613
+ previous_headers = session.headers
3614
+ await self._release_request_state_account_response_create_lease(status_request_state)
3615
+ async with session.pending_lock:
3616
+ if status_request_state not in session.pending_requests:
3617
+ session.pending_requests.appendleft(status_request_state)
3618
+ session.queued_request_count += 1
3619
+ status_request_state.awaiting_response_created = True
3620
+ status_request_state.response_id = None
3621
+ retried = await self._retry_http_bridge_precreated_request(session)
3622
+ if retried:
3623
+ logger.info(
3624
+ "Retried HTTP bridge request after account/model rejection "
3625
+ "request_id=%s rejected_account_id=%s model=%s",
3626
+ status_request_state.request_log_id or status_request_state.request_id,
3627
+ rejected_account_id,
3628
+ status_request_state.model,
3629
+ )
3630
+ return
3631
+ replacement_session_selected = session.account.id != rejected_account_id
3632
+ if not replacement_session_selected:
3633
+ session.upstream_turn_state = previous_upstream_turn_state
3634
+ session.downstream_turn_state = previous_downstream_turn_state
3635
+ session.headers = previous_headers
3636
+ async with session.pending_lock:
3637
+ if status_request_state in session.pending_requests:
3638
+ session.pending_requests.remove(status_request_state)
3639
+ session.queued_request_count = max(0, session.queued_request_count - 1)
3640
+ if replacement_session_selected:
3641
+ # Reconnect may have committed the session to a replacement
3642
+ # account before its replacement lease or send failed. Never
3643
+ # graft the rejected account's turn metadata back onto that
3644
+ # socket; retire the now-unused replacement session instead.
3645
+ session.upstream_control.reconnect_requested = True
3646
+ session.upstream_control.retire_after_drain = True
3647
+ await self._retire_http_bridge_after_drain_if_ready(session)
3648
+ payload = cast(
3649
+ dict[str, JsonValue],
3650
+ dict(
3651
+ response_failed_event(
3652
+ status_request_state.error_code_override or "upstream_unavailable",
3653
+ status_request_state.error_message_override or "HTTP bridge replacement retry failed",
3654
+ error_type=status_request_state.error_type_override or "server_error",
3655
+ response_id=status_request_state.request_id,
3656
+ error_param=status_request_state.error_param_override,
3657
+ )
3658
+ ),
3659
+ )
3660
+ event_block = format_sse_event(payload)
3661
+ event = parse_sse_event_payload(payload)
3662
+ event_type = "response.failed"
3663
+ if status_request_state.precreated_replay_reason == _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE:
3664
+ _clear_websocket_precreated_replay_fallback(status_request_state)
3665
+ elif (
3666
+ retry_error_code is not None
3667
+ and retry_error_code != _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
3668
+ and not is_previous_response_not_found_event
3669
+ ):
3670
+ await self._handle_or_defer_precreated_stream_health(
3671
+ status_request_state,
3672
+ session.account,
3673
+ {"message": retry_error_message or "Upstream error"},
3674
+ retry_error_code,
3675
+ )
3676
+ # Pre-created anchored requests belong to the owner-pinned branch
3677
+ # above; an accepted anchored follow-up with a proxy-injected
3678
+ # anchor and a retry-safe fresh body is replayed here exactly as
3679
+ # the capacity-message wait branch replays it.
3680
+ if status_request_state is not None and (
3681
+ status_request_state.previous_response_id is None
3682
+ or _http_bridge_accepted_anchored_replay_candidate(status_request_state)
3683
+ ):
3684
+ async with session.pending_lock:
3685
+ if status_request_state not in session.pending_requests:
3686
+ session.pending_requests.appendleft(status_request_state)
3687
+ session.queued_request_count += 1
3688
+ staged = await _stage_websocket_request_state_for_replay(
3689
+ status_request_state,
3690
+ create_gate=session.response_create_gate,
3691
+ surface="http_bridge",
3692
+ trigger="capacity_error",
3693
+ )
3694
+ retried = staged and await self._retry_http_bridge_precreated_request(session)
3695
+ if retried:
3696
+ return
3697
+ async with session.pending_lock:
3698
+ if status_request_state in session.pending_requests:
3699
+ session.pending_requests.remove(status_request_state)
3700
+ session.queued_request_count = max(0, session.queued_request_count - 1)
3701
+ if staged:
3702
+ # A busy create gate forwards the upstream terminal as-is;
3703
+ # only a replay that was attempted and failed is rewritten.
3704
+ status_request_state.error_http_status_override = 502
3705
+ (
3706
+ _downstream_text,
3707
+ event_block,
3708
+ event,
3709
+ payload,
3710
+ event_type,
3711
+ ) = _build_stream_incomplete_terminal_event_for_request(status_request_state)
3712
+
3713
+ completed_usage = (
3714
+ event.response.usage if event_type == "response.completed" and event and event.response else None
3715
+ )
3716
+ completed_empty_prewarm = (
3717
+ event_type == "response.completed"
3718
+ and terminal_request_state is not None
3719
+ and terminal_request_state.request_kind == "prewarm"
3720
+ and completed_usage is not None
3721
+ and completed_usage.output_tokens == 0
3722
+ )
3723
+
3724
+ # The circuit settle and quarantine clear run BEFORE the fresh
3725
+ # anchor is registered. A poison abandonment fences its continuity
3726
+ # clear on the anchor captured when its episode was validated, and
3727
+ # settle-first ordering is what makes that fence airtight: any
3728
+ # consult that runs after this settle is vetoed by the reset row,
3729
+ # and any abandonment authorized before it fences on the old anchor
3730
+ # and cannot touch the one registered below.
3731
+ completion_circuit_settlement_failed = False
3732
+ completion_quarantine_clear_fence: int | None = None
3733
+ completion_pre_settle_poison_detail: str | None = None
3734
+ completion_settles_onto_tombstone = False
3735
+ if (
3736
+ event_type == "response.completed"
3737
+ and terminal_request_state is not None
3738
+ and not terminal_request_state.suppressed_duplicate_tool_call
3739
+ and terminal_request_state.request_kind != "prewarm"
3740
+ and not terminal_request_state.skip_request_log
3741
+ ):
3742
+ # Captured before the settle and registration awaits below: a
3743
+ # concurrent request failing during those awaits arms a NEW
3744
+ # same-key quarantine whose evidence this completion does not
3745
+ # disprove, and the clear at the bottom must fence on what was
3746
+ # armed when this response proved the session healthy, not on
3747
+ # whatever owns the key by then.
3748
+ # Adopt any durable-only poison row BEFORE the fences are
3749
+ # captured: the settle's internal load would otherwise arm the
3750
+ # quarantine after the fence read (leaving a healthy key
3751
+ # suppressed for the TTL because the clear cannot match), and a
3752
+ # failed registration would find no pre-settle poison detail to
3753
+ # re-seed.
3754
+ completion_pre_settle_load_succeeded = await self._load_http_bridge_retry_circuit(session)
3755
+ completion_quarantine_clear_fence = _http_bridge_quarantine_clear_fence(self, session.key)
3756
+ async with self._http_bridge_retry_circuit_lock:
3757
+ pre_settle_state = self._http_bridge_retry_circuits.get(session.key)
3758
+ if pre_settle_state is not None:
3759
+ completion_pre_settle_poison_detail = (
3760
+ pre_settle_state.last_detail
3761
+ if _http_bridge_anchor_poison_detail(pre_settle_state.last_detail) is not None
3762
+ else pre_settle_state.owed_poison_detail
3763
+ )
3764
+ # An episode already carrying the fail-closed tombstone
3765
+ # keeps it through the settle: erasing it before the
3766
+ # fresh anchor commits would drop the only durable
3767
+ # marker while the registration can still fail or never
3768
+ # run at all.
3769
+ completion_settles_onto_tombstone = (
3770
+ completion_pre_settle_poison_detail is not None
3771
+ or pre_settle_state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
3772
+ )
3773
+ completion_registration_possible = (
3774
+ response_id is not None and matched_request_state is not None and not completed_empty_prewarm
3775
+ )
3776
+ if (
3777
+ not terminal_request_state.verified_stale_anchor_replay
3778
+ and completion_settles_onto_tombstone
3779
+ and not completion_registration_possible
3780
+ ):
3781
+ # No usable response id or matched request: the registration
3782
+ # block below never runs, so settling here would replace the
3783
+ # poison row with a zero-count tombstone while the OLD
3784
+ # poisoned anchor stays stored — the next planning load
3785
+ # reads the zero count as a disproved episode, revokes the
3786
+ # quarantine, and a full resend gets the dead anchor
3787
+ # injected. Keep the episode unsettled; a completion that
3788
+ # can actually register a fresh anchor settles it.
3789
+ completion_circuit_settlement_failed = True
3790
+ elif not terminal_request_state.verified_stale_anchor_replay:
3791
+ # A completion replacing a poison episode settles with the
3792
+ # transitional tombstone: a crash or takeover between this
3793
+ # settle and the registration below would otherwise leave
3794
+ # the old poisoned anchor stored while the reset row reads
3795
+ # as disproved. The tombstone is erased once the fresh
3796
+ # anchor commits.
3797
+ circuit_settled = await self._clear_http_bridge_retry_circuit(
3798
+ session,
3799
+ settled_detail=(
3800
+ _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
3801
+ if completion_settles_onto_tombstone
3802
+ else None
3803
+ ),
3804
+ )
3805
+ if not completion_pre_settle_load_succeeded:
3806
+ # The fence above was captured off a failed durable read;
3807
+ # the settle's own successful inner load may have armed
3808
+ # the poison quarantine AFTER that capture, and the final
3809
+ # fenced clear would then refuse to remove it — a healthy
3810
+ # key stuck for the whole poison window despite the fresh
3811
+ # anchor about to register. Recapture after the settle:
3812
+ # this still precedes the registration awaits, so a
3813
+ # quarantine armed by a genuinely concurrent strike
3814
+ # during those awaits stays outside the fence.
3815
+ completion_quarantine_clear_fence = _http_bridge_quarantine_clear_fence(self, session.key)
3816
+ if not circuit_settled:
3817
+ # The old poison episode was restored. Its owed clear is
3818
+ # suppressed only once the fresh anchor actually
3819
+ # persists below: suppressing here and then failing the
3820
+ # registration would leave the old poisoned anchor
3821
+ # stored with no funnel willing to clear it after the
3822
+ # process-local quarantine expires.
3823
+ completion_circuit_settlement_failed = True
3824
+
3825
+ # False until a fresh durable anchor actually confirms: a completed
3826
+ # event without a usable response id (or with no matched request)
3827
+ # skips the registration block entirely, and clearing the quarantine
3828
+ # then would leave the old poisoned anchor as the stored one for the
3829
+ # next reattach.
3830
+ completion_anchor_registration_confirmed = False
3831
+ if (
3832
+ response_id is not None
3833
+ and matched_request_state is not None
3834
+ and event_type == "response.completed"
3835
+ and not completed_empty_prewarm
3836
+ ):
3837
+ anchor_advance_supersession = None
3838
+ if completion_circuit_settlement_failed:
3839
+ # Applied BEFORE the fresh anchor is published: a concurrent
3840
+ # funnel's consult whose continuity read lands between the
3841
+ # publication and a later suppression would validate the old
3842
+ # poison row and fence its rebind on the anchor just
3843
+ # registered, deleting it. The suppression is transitional —
3844
+ # rolled back below when the registration fails, so the old
3845
+ # anchor never becomes uncleareable.
3846
+ anchor_advance_supersession = await self._http_bridge_suppress_poison_clear_after_anchor_advance(
3847
+ session
3848
+ )
3849
+ alias_registered = await self._register_http_bridge_previous_response_id(
3850
+ session,
3851
+ response_id,
3852
+ input_item_count=(
3853
+ matched_request_state.input_item_count if matched_request_state.input_item_count > 0 else None
3854
+ ),
3855
+ input_full_fingerprint=(
3856
+ matched_request_state.input_full_fingerprint if matched_request_state.input_item_count > 0 else None
3857
+ ),
3858
+ pending_tool_calls=_durable_pending_tool_call_manifest(matched_request_state, payload),
3859
+ )
3860
+ completion_anchor_registration_confirmed = alias_registered
3861
+ if not alias_registered and anchor_advance_supersession is not None:
3862
+ await self._http_bridge_restore_poison_clear_after_failed_anchor_advance(
3863
+ session, anchor_advance_supersession
3864
+ )
3865
+ if alias_registered and anchor_advance_supersession is not None:
3866
+ await self._http_bridge_promote_transitional_supersession(session, anchor_advance_supersession)
3867
+ if alias_registered and not completion_circuit_settlement_failed and completion_settles_onto_tombstone:
3868
+ # The fresh anchor committed: erase the transitional
3869
+ # tombstone the settle left, fenced on the row's own
3870
+ # values, with one reconcile round covering both a
3871
+ # transient durable blip and a CAS miss under a sticky
3872
+ # concurrent strike.
3873
+ await self._http_bridge_reconcile_transitional_tombstone(session)
3874
+ if (
3875
+ not alias_registered
3876
+ and not completion_circuit_settlement_failed
3877
+ and completion_pre_settle_poison_detail is not None
3878
+ ):
3879
+ # The settle-before-registration ordering already zeroed the
3880
+ # durable circuit, but the registration failure means the
3881
+ # OLD poisoned anchor is still the stored one. Without
3882
+ # durable evidence the next planning load revokes the kept
3883
+ # local quarantine as a disproved episode and other replicas
3884
+ # never arm; re-seed the row so the poison protection holds
3885
+ # everywhere until a real recovery lands.
3886
+ await self._http_bridge_restore_poison_row_after_failed_registration(
3887
+ session, completion_pre_settle_poison_detail
3888
+ )
3889
+ if not alias_registered and is_http_bridge_account_neutral_replay(
3890
+ kind=session.key.affinity_kind,
3891
+ key=session.key.affinity_key,
3892
+ ):
3893
+ session.upstream_control.reconnect_requested = True
3894
+ session.upstream_control.retire_after_drain = True
3895
+ matched_request_state.error_http_status_override = 502
3896
+ payload = cast(
3897
+ dict[str, JsonValue],
3898
+ dict(
3899
+ response_failed_event(
3900
+ "bridge_continuity_persistence_failed",
3901
+ "Recovered response continuity could not be persisted; retry the request.",
3902
+ response_id=_websocket_downstream_response_id(matched_request_state),
3903
+ )
3904
+ ),
3905
+ )
3906
+ event_block = format_sse_event(payload)
3907
+ event = parse_sse_event_payload(payload)
3908
+ event_type = "response.failed"
3909
+ # The upstream response was already acknowledged. The local
3910
+ # alias write failed, so expose a terminal error downstream
3911
+ # but keep the durable operation acknowledged/ambiguous to
3912
+ # prevent an identical retry from dispatching it again.
3913
+ continuity_persistence_failed_after_ack = True
3914
+ completed_usage = None
3915
+ completed_empty_prewarm = False
3916
+
3917
+ if (
3918
+ event_type == "response.completed"
3919
+ and terminal_request_state is not None
3920
+ and not terminal_request_state.suppressed_duplicate_tool_call
3921
+ and terminal_request_state.request_kind != "prewarm"
3922
+ and not terminal_request_state.skip_request_log
3923
+ # A failed circuit settlement leaves the at-threshold poison row
3924
+ # standing; the quarantine keeps covering it so the surviving
3925
+ # cooldown cannot hand the next attach a poisoned injection while
3926
+ # the settle retries at the next opportunity.
3927
+ and not completion_circuit_settlement_failed
3928
+ # A durable alias write that failed leaves the OLD anchor as the
3929
+ # stored one even though this worker's completion succeeded: a
3930
+ # replica or a restart would re-inject it, and the quarantine is
3931
+ # the only protection left once the circuit was settled.
3932
+ and completion_anchor_registration_confirmed
3933
+ ):
3934
+ # The quarantine clears only after the fresh anchor persisted:
3935
+ # a failed alias write rewrites the event to response.failed
3936
+ # above, this guard then skips, and the quarantine keeps
3937
+ # covering the old anchor that is still stored. The circuit
3938
+ # settle deliberately ran before registration for the
3939
+ # abandonment fence; the quarantine surviving here is what
3940
+ # protects the partial-failure window that ordering leaves.
3941
+ _clear_http_bridge_quarantine(
3942
+ self,
3943
+ session,
3944
+ key_generation=completion_quarantine_clear_fence,
3945
+ additional_key=terminal_request_state.verified_stale_anchor_retry_circuit_key,
3946
+ additional_key_generation=terminal_request_state.verified_stale_anchor_quarantine_generation,
3947
+ )
3948
+
3949
+ operation_state = _http_bridge_operation_state_for_event(event_type)
3950
+ if operation_state is not None:
3951
+ operation_request_states: list[Any] = []
3952
+ for candidate in (matched_request_state, terminal_request_state):
3953
+ if candidate is not None and candidate not in operation_request_states:
3954
+ operation_request_states.append(candidate)
3955
+ for operation_request_state in operation_request_states:
3956
+ request_operation_state = operation_state
3957
+ if continuity_persistence_failed_after_ack and operation_request_state is matched_request_state:
3958
+ request_operation_state = "acknowledged"
3959
+ if request_operation_state == "failed":
3960
+ # Failure rows are exposed only by the terminal-event
3961
+ # persistence path below, which appends the terminal SSE
3962
+ # block and flips the operation state atomically.
3963
+ continue
3964
+ await _update_http_bridge_operation_state(
3965
+ self,
3966
+ session,
3967
+ operation_request_state,
3968
+ state=request_operation_state,
3969
+ response_id=response_id,
3970
+ )
3971
+
3972
+ recovery_attempt_session_id = (
3973
+ matched_request_state.recovery_attempt_session_id
3974
+ if matched_request_state is not None and matched_request_state.recovery_attempt_session_id is not None
3975
+ else session.durable_session_id
3976
+ )
3977
+ recovery_attempt_owner_epoch = (
3978
+ matched_request_state.recovery_attempt_owner_epoch
3979
+ if matched_request_state is not None and matched_request_state.recovery_attempt_owner_epoch is not None
3980
+ else session.durable_owner_epoch
3981
+ )
3982
+
3983
+ if (
3984
+ isinstance(event_type, str)
3985
+ and (event_type.startswith("response.") or event_type == "error")
3986
+ and matched_request_state is not None
3987
+ and matched_request_state.recovery_attempt_fingerprint is not None
3988
+ and recovery_attempt_session_id is not None
3989
+ and recovery_attempt_owner_epoch is not None
3990
+ and (
3991
+ event_type in {"response.completed", "response.failed", "response.incomplete", "error"}
3992
+ or not matched_request_state.recovery_attempt_event_observed
3993
+ )
3994
+ ):
3995
+ settlement_marked = False
3996
+ for settlement_attempt in range(3):
3997
+ try:
3998
+ marked = await self._durable_bridge.mark_recovery_attempt_replayed(
3999
+ session_id=recovery_attempt_session_id,
4000
+ api_key_id=session.key.api_key_id,
4001
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4002
+ owner_epoch=recovery_attempt_owner_epoch,
4003
+ request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
4004
+ response_id=response_id,
4005
+ )
4006
+ if marked:
4007
+ settlement_marked = True
4008
+ break
4009
+ if settlement_attempt == 2:
4010
+ _schedule_http_bridge_recovery_settlement_retry(
4011
+ self,
4012
+ session,
4013
+ session_id=recovery_attempt_session_id,
4014
+ api_key_id=session.key.api_key_id,
4015
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4016
+ owner_epoch=recovery_attempt_owner_epoch,
4017
+ request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
4018
+ response_id=response_id,
4019
+ release_origin_lease=(
4020
+ recovery_attempt_session_id != session.durable_session_id
4021
+ and event_type
4022
+ in {"response.completed", "response.failed", "response.incomplete", "error"}
4023
+ ),
4024
+ )
4025
+ except Exception:
4026
+ if settlement_attempt == 2:
4027
+ logger.warning("Failed to settle HTTP bridge recovery attempt", exc_info=True)
4028
+ _schedule_http_bridge_recovery_settlement_retry(
4029
+ self,
4030
+ session,
4031
+ session_id=recovery_attempt_session_id,
4032
+ api_key_id=session.key.api_key_id,
4033
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4034
+ owner_epoch=recovery_attempt_owner_epoch,
4035
+ request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
4036
+ response_id=response_id,
4037
+ release_origin_lease=(
4038
+ recovery_attempt_session_id != session.durable_session_id
4039
+ and event_type
4040
+ in {"response.completed", "response.failed", "response.incomplete", "error"}
4041
+ ),
4042
+ )
4043
+ else:
4044
+ await scheduler.sleep(0.05 * (settlement_attempt + 1))
4045
+ if (
4046
+ settlement_marked
4047
+ and event_type in {"response.completed", "response.failed", "response.incomplete", "error"}
4048
+ and recovery_attempt_session_id != session.durable_session_id
4049
+ ):
4050
+ try:
4051
+ await self._durable_bridge.release_live_session(
4052
+ session_id=recovery_attempt_session_id,
4053
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4054
+ owner_epoch=recovery_attempt_owner_epoch,
4055
+ draining=False,
4056
+ )
4057
+ except Exception:
4058
+ logger.debug("Failed to release HTTP bridge recovery origin lease", exc_info=True)
4059
+ matched_request_state.recovery_attempt_event_observed = True
4060
+
4061
+ if event_type == "response.completed" and terminal_request_state is not None and not completed_empty_prewarm:
4062
+ # Record the completed response id regardless of input shape so
4063
+ # subsequent turns (including ones that never populated
4064
+ # input_item_count, e.g. string inputs) can still reuse this
4065
+ # anchor for continuity lookups.
4066
+ if response_id is not None:
4067
+ session.last_completed_response_id = response_id
4068
+ # This response was completed on the session's current account, so
4069
+ # that account owns the anchor. Record it so the anchor is only
4070
+ # replayed on the same account (never after a cross-account failover).
4071
+ session.last_completed_response_account_id = session.account.id
4072
+ # Remember which tool-call items the completed response left
4073
+ # pending so an anchored follow-up that omits their outputs
4074
+ # (interrupted turn) can receive synthetic interrupted
4075
+ # outputs instead of an upstream missing-tool-output 400.
4076
+ session.last_pending_tool_calls = dict(terminal_request_state.pending_tool_call_types)
4077
+ # Prefix trimming is only meaningful for list-shaped inputs, so
4078
+ # keep the input-count / fingerprint update scoped to that path.
4079
+ if terminal_request_state.input_item_count > 0:
4080
+ session.last_completed_input_count = terminal_request_state.input_item_count
4081
+ session.last_completed_input_prefix_fingerprint = terminal_request_state.input_full_fingerprint
4082
+
4083
+ normalize_error_event = (
4084
+ terminal_request_state is None or terminal_request_state.enforce_openai_sdk_contract
4085
+ ) and (matched_request_state is None or matched_request_state.enforce_openai_sdk_contract)
4086
+ settlement_payload = payload
4087
+ settlement_event = event
4088
+ settlement_event_type = event_type
4089
+ if event_type == "error" and normalize_error_event:
4090
+ http_status = _http_error_status_from_payload(payload)
4091
+ if status_request_state is not None and status_request_state.error_http_status_override is None:
4092
+ status_request_state.error_http_status_override = http_status
4093
+ (
4094
+ event_block,
4095
+ payload,
4096
+ event,
4097
+ event_type,
4098
+ ) = _normalize_http_bridge_error_event(
4099
+ event=event,
4100
+ payload=payload,
4101
+ request_state=terminal_request_state or matched_request_state,
4102
+ )
4103
+ settlement_payload = payload
4104
+ settlement_event = event
4105
+ settlement_event_type = event_type
4106
+ elif event_type == "error":
4107
+ http_status = _http_error_status_from_payload(payload)
4108
+ if status_request_state is not None and status_request_state.error_http_status_override is None:
4109
+ status_request_state.error_http_status_override = http_status
4110
+ (
4111
+ _settlement_event_block,
4112
+ settlement_payload,
4113
+ settlement_event,
4114
+ settlement_event_type,
4115
+ ) = _normalize_http_bridge_error_event(
4116
+ event=event,
4117
+ payload=payload,
4118
+ request_state=terminal_request_state or matched_request_state,
4119
+ )
4120
+
4121
+ if (
4122
+ settlement_event_type in {"response.failed", "error"}
4123
+ and matched_request_state is not None
4124
+ and matched_request_state.recovery_attempt_fingerprint is not None
4125
+ and recovery_attempt_session_id is not None
4126
+ and recovery_attempt_owner_epoch is not None
4127
+ and not matched_request_state.recovery_attempt_event_observed
4128
+ ):
4129
+ # An explicit deterministic failure is terminal evidence for the
4130
+ # journaled request, not an ambiguous transport outcome. Consume
4131
+ # the UNKNOWN row after normalizing top-level errors so a later
4132
+ # identical retry cannot turn it into an account-neutral replay.
4133
+ deterministic_settlement_marked = False
4134
+ for settlement_attempt in range(3):
4135
+ try:
4136
+ marked = await self._durable_bridge.mark_recovery_attempt_replayed(
4137
+ session_id=recovery_attempt_session_id,
4138
+ api_key_id=session.key.api_key_id,
4139
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4140
+ owner_epoch=recovery_attempt_owner_epoch,
4141
+ request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
4142
+ response_id=response_id,
4143
+ )
4144
+ if marked:
4145
+ deterministic_settlement_marked = True
4146
+ break
4147
+ if settlement_attempt == 2:
4148
+ _schedule_http_bridge_recovery_settlement_retry(
4149
+ self,
4150
+ session,
4151
+ session_id=recovery_attempt_session_id,
4152
+ api_key_id=session.key.api_key_id,
4153
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4154
+ owner_epoch=recovery_attempt_owner_epoch,
4155
+ request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
4156
+ response_id=response_id,
4157
+ release_origin_lease=recovery_attempt_session_id != session.durable_session_id,
4158
+ )
4159
+ except Exception:
4160
+ if settlement_attempt == 2:
4161
+ logger.warning("Failed to settle deterministic HTTP bridge recovery attempt", exc_info=True)
4162
+ _schedule_http_bridge_recovery_settlement_retry(
4163
+ self,
4164
+ session,
4165
+ session_id=recovery_attempt_session_id,
4166
+ api_key_id=session.key.api_key_id,
4167
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4168
+ owner_epoch=recovery_attempt_owner_epoch,
4169
+ request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
4170
+ response_id=response_id,
4171
+ release_origin_lease=recovery_attempt_session_id != session.durable_session_id,
4172
+ )
4173
+ else:
4174
+ await scheduler.sleep(0.05 * (settlement_attempt + 1))
4175
+ if deterministic_settlement_marked and recovery_attempt_session_id != session.durable_session_id:
4176
+ try:
4177
+ await self._durable_bridge.release_live_session(
4178
+ session_id=recovery_attempt_session_id,
4179
+ instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
4180
+ owner_epoch=recovery_attempt_owner_epoch,
4181
+ draining=False,
4182
+ )
4183
+ except Exception:
4184
+ logger.debug("Failed to release HTTP bridge recovery origin lease", exc_info=True)
4185
+
4186
+ if event_type == "response.created" and release_create_gate and created_request_state is not None:
4187
+ await _release_websocket_response_create_gate(
4188
+ created_request_state, session.response_create_gate, scheduler=scheduler
4189
+ )
4190
+
4191
+ if terminal_request_state is not None and settlement_event_type in {"response.failed", "error"}:
4192
+ if settlement_event_type == "error":
4193
+ error = settlement_event.error if settlement_event else None
4194
+ else:
4195
+ error = settlement_event.response.error if settlement_event and settlement_event.response else None
4196
+ terminal_error_code = _normalize_error_code(
4197
+ error.code if error else None,
4198
+ error.type if error else None,
4199
+ )
4200
+ terminal_error_message = error.message if error else None
4201
+ if _is_security_work_authorization_required_error(terminal_error_code, terminal_error_message):
4202
+ can_retry_security_work = (
4203
+ not is_http_bridge_account_neutral_replay(
4204
+ kind=session.key.affinity_kind,
4205
+ key=session.key.affinity_key,
4206
+ )
4207
+ and not session.account.security_work_authorized
4208
+ and not has_other_pending_requests
4209
+ and terminal_request_state.response_id is None
4210
+ and terminal_request_state.replay_count < 1
4211
+ and bool(terminal_request_state.request_text)
4212
+ and terminal_request_state.preferred_account_id != session.account.id
4213
+ and _websocket_auth_request_can_switch_account(terminal_request_state)
4214
+ and _websocket_request_can_replay_before_visible_output(terminal_request_state)
4215
+ )
4216
+ _clear_websocket_deferred_reasoning_downstream_texts(terminal_request_state)
4217
+ if terminal_request_state.event_queue is not None:
4218
+ await terminal_request_state.event_queue.put(
4219
+ format_sse_event(
4220
+ _security_work_advisory_event(
4221
+ code=_SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
4222
+ message=(
4223
+ _SECURITY_WORK_RETRY_MESSAGE
4224
+ if can_retry_security_work
4225
+ else "Upstream flagged this request as possible cybersecurity work. "
4226
+ "codex-lb cannot safely switch accounts after this response has already started, "
4227
+ "so the original upstream error is being forwarded."
4228
+ ),
4229
+ request_id=terminal_request_state.request_log_id or terminal_request_state.request_id,
4230
+ action=(
4231
+ "retry_security_work_authorized"
4232
+ if can_retry_security_work
4233
+ else "forward_original_security_work_error"
4234
+ ),
4235
+ account_id=session.account.id,
4236
+ )
4237
+ )
4238
+ )
4239
+ if can_retry_security_work:
4240
+ retried = await self._retry_http_bridge_security_work_request(session, terminal_request_state)
4241
+ if retried:
4242
+ return
4243
+
4244
+ terminal_strike_failures: int | None = None
4245
+ terminal_poison_detail: str | None = None
4246
+
4247
+ async def _finalize_terminal_settlement(settled_request_state: _WebSocketRequestState) -> None:
4248
+ try:
4249
+ await self._finalize_websocket_request_state(
4250
+ settled_request_state,
4251
+ account=session.account,
4252
+ account_id_value=session.account.id,
4253
+ event=settlement_event,
4254
+ event_type=settlement_event_type,
4255
+ payload=settlement_payload,
4256
+ api_key=settled_request_state.api_key,
4257
+ upstream_control=session.upstream_control,
4258
+ response_create_gate=session.response_create_gate,
4259
+ )
4260
+ finally:
4261
+ await self._maybe_release_idle_http_bridge_session_lease(session)
4262
+
4263
+ if settlement_event_type in {"response.failed", "response.incomplete", "error"}:
4264
+ error_code = None
4265
+ if settlement_event_type == "error":
4266
+ error = settlement_event.error if settlement_event else None
4267
+ error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
4268
+ elif settlement_event and settlement_event.response:
4269
+ error = settlement_event.response.error
4270
+ error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
4271
+ _log_http_bridge_event(
4272
+ "terminal_error",
4273
+ session.key,
4274
+ account_id=session.account.id,
4275
+ model=session.request_model,
4276
+ detail=error_code,
4277
+ pending_count=await self._http_bridge_pending_count(session),
4278
+ cache_key_family=session.key.affinity_kind,
4279
+ model_class=_extract_model_class(session.request_model) if session.request_model else None,
4280
+ )
4281
+ if (
4282
+ error_code is not None
4283
+ and terminal_request_state is not None
4284
+ and terminal_request_state.response_event_count == 0
4285
+ # Internal warmup probes carry no anchor and prove nothing
4286
+ # about the key's continuity; charging them would open and
4287
+ # quarantine the hard key before any real turn, exactly the
4288
+ # states the completion settle already excludes.
4289
+ and terminal_request_state.request_kind != "prewarm"
4290
+ and not terminal_request_state.skip_request_log
4291
+ # Only a held safe replay excludes the strike: an unanchored
4292
+ # request with no replay is stranded like any other, and the
4293
+ # delta spec's no-response/no-safe-replay rule applies to it
4294
+ # the same way the retirement funnels apply it.
4295
+ and not _http_bridge_request_state_holds_safe_replay(terminal_request_state)
4296
+ ):
4297
+ # An upstream terminal frame that fails the request before any
4298
+ # response event is the same pre-response failure the circuit
4299
+ # measures on eventless retirements; it reaches this settlement
4300
+ # path instead of the retirement funnel, so it would otherwise
4301
+ # never count. Attempt-scoped recording keeps a later
4302
+ # retirement of the same lifecycle from double-counting, and
4303
+ # the recorder itself drops non-circuit details and soft keys.
4304
+ #
4305
+ # This runs before the terminal frame and its queue sentinel
4306
+ # reach the client: once completion is observable the client can
4307
+ # resend the same anchor immediately, and the cooldown and
4308
+ # quarantine have to already be visible to that resend rather
4309
+ # than still awaiting durable I/O. Recovery paths that retry
4310
+ # this request in place have all declined by here.
4311
+ terminal_strike_failures = await self._record_http_bridge_retry_circuit_failure(
4312
+ session,
4313
+ detail=error_code,
4314
+ attempt=terminal_request_state.response_create_attempt,
4315
+ terminal_pre_response_frame=True,
4316
+ )
4317
+ terminal_poison_detail = await self._http_bridge_effective_anchor_poison_detail(session, error_code)
4318
+
4319
+ matched_event_queue = (
4320
+ completed_event_queue
4321
+ if completed_event_queue_claimed and matched_request_state is terminal_request_state
4322
+ else matched_request_state.event_queue
4323
+ if matched_request_state is not None
4324
+ else None
4325
+ )
4326
+ matched_deferred_texts = (
4327
+ _pop_websocket_deferred_reasoning_downstream_texts(matched_request_state)
4328
+ if matched_request_state is not None and not suppress_downstream_event
4329
+ else []
4330
+ )
4331
+ matched_terminal_state = _http_bridge_operation_state_for_event(event_type)
4332
+ if continuity_persistence_failed_after_ack and matched_request_state is not None:
4333
+ # The upstream response was already accepted. The downstream
4334
+ # failure only reports that its durable alias could not be
4335
+ # persisted, so keep the operation fenced as acknowledged while
4336
+ # retaining the failure SSE for the client.
4337
+ matched_terminal_state = "acknowledged"
4338
+
4339
+ async def _publish_matched_and_terminal_frames() -> None:
4340
+ nonlocal matched_terminal_enqueued
4341
+ if matched_request_state is not None and not suppress_downstream_event:
4342
+ for deferred_text in matched_deferred_texts:
4343
+ await _persist_http_bridge_operation_event(
4344
+ self,
4345
+ session,
4346
+ matched_request_state,
4347
+ deferred_text,
4348
+ terminal=False,
4349
+ )
4350
+ if matched_request_state is not None and matched_event_queue is not None and not suppress_downstream_event:
4351
+ for deferred_text in matched_deferred_texts:
4352
+ await matched_event_queue.put(deferred_text)
4353
+ if matched_request_state is not None and not suppress_downstream_event:
4354
+ matched_terminal_enqueued = await _persist_http_bridge_operation_event(
4355
+ self,
4356
+ session,
4357
+ matched_request_state,
4358
+ event_block,
4359
+ terminal=event_type in {"response.completed", "response.failed", "response.incomplete", "error"},
4360
+ terminal_state=matched_terminal_state,
4361
+ terminal_event_queue=matched_event_queue,
4362
+ terminal_delivery_scope=(completed_delivery_scope if completed_event_queue_claimed else None),
4363
+ )
4364
+ if (
4365
+ matched_request_state is not None
4366
+ and matched_event_queue is not None
4367
+ and not suppress_downstream_event
4368
+ and matched_terminal_enqueued is not True
4369
+ ):
4370
+ await matched_event_queue.put(event_block)
4371
+
4372
+ if terminal_request_state is None:
4373
+ return
4374
+
4375
+ terminal_event_queue = (
4376
+ completed_event_queue if completed_event_queue_claimed else terminal_request_state.event_queue
4377
+ )
4378
+ terminal_enqueued = matched_terminal_enqueued if terminal_request_state is matched_request_state else False
4379
+ if terminal_request_state is not matched_request_state:
4380
+ deferred_texts = _pop_websocket_deferred_reasoning_downstream_texts(terminal_request_state)
4381
+ for deferred_text in deferred_texts:
4382
+ if not suppress_downstream_event:
4383
+ await _persist_http_bridge_operation_event(
4384
+ self,
4385
+ session,
4386
+ terminal_request_state,
4387
+ deferred_text,
4388
+ terminal=False,
4389
+ )
4390
+ if terminal_event_queue is not None:
4391
+ await terminal_event_queue.put(deferred_text)
4392
+ if not suppress_downstream_event:
4393
+ terminal_enqueued = await _persist_http_bridge_operation_event(
4394
+ self,
4395
+ session,
4396
+ terminal_request_state,
4397
+ event_block,
4398
+ terminal=True,
4399
+ terminal_state=(
4400
+ "acknowledged"
4401
+ if continuity_persistence_failed_after_ack
4402
+ and terminal_request_state is matched_request_state
4403
+ else _http_bridge_operation_state_for_event(event_type)
4404
+ ),
4405
+ terminal_event_queue=terminal_event_queue,
4406
+ terminal_delivery_scope=(completed_delivery_scope if completed_event_queue_claimed else None),
4407
+ )
4408
+ if terminal_event_queue is not None and terminal_enqueued is not True:
4409
+ await terminal_event_queue.put(event_block)
4410
+ if terminal_event_queue is not None:
4411
+ if terminal_enqueued is not True:
4412
+ await terminal_event_queue.put(None)
4413
+ if completed_event_queue_claimed and completed_delivery_scope is not None:
4414
+ async with session.pending_lock:
4415
+ # Keep the completed claim authoritative after its producer
4416
+ # returns. A concurrent timeout may still be finishing
4417
+ # awaited recovery work before it rechecks this scope.
4418
+ completed_delivery_scope.terminal_enqueued = True
4419
+
4420
+ matched_terminal_enqueued = False
4421
+ terminal_publication_cancellation: asyncio.CancelledError | None = None
4422
+ if terminal_poison_detail is not None and terminal_strike_failures is not None:
4423
+ # Publication and settlement share one deferral for a poison
4424
+ # terminal: a cancellation landing inside the operation
4425
+ # persistence, between the queued frame and its sentinel, or on
4426
+ # any other publication await used to escape before the owned
4427
+ # settlement task even existed — the outer abort finalized the
4428
+ # popped request and the consult, abandonment, and marker never
4429
+ # started, leaving the poisoned durable anchor reusable once the
4430
+ # process-local quarantine expired.
4431
+ terminal_publication_task = scheduler.create_task(
4432
+ _publish_matched_and_terminal_frames(),
4433
+ name=f"http-bridge-terminal-publication-{session.durable_session_id}",
4434
+ )
4435
+ _terminal_publication_result, terminal_publication_cancellation = await _await_task_deferring_cancellation(
4436
+ terminal_publication_task
4437
+ )
4438
+ else:
4439
+ await _publish_matched_and_terminal_frames()
4440
+
4441
+ if terminal_request_state is None:
4442
+ return
4443
+
4444
+ if terminal_poison_detail is not None and terminal_strike_failures is not None:
4445
+ # The strike above opens the circuit, but the durable anchor that
4446
+ # failed is still stored. Only the retirement and close funnels ever
4447
+ # reached the poison clear, and a terminal frame settles through
4448
+ # neither, so the dead anchor survived every cooldown and re-poisoned
4449
+ # the key on the next reattach. Quarantine suppresses the injection
4450
+ # in the meantime, but it is process-local and expires; the durable
4451
+ # row has to be cleared for the recovery to hold.
4452
+ #
4453
+ # This gates on the circuit's own threshold rather than the
4454
+ # configurable anchor-poison threshold, and fires with the same
4455
+ # evidence that already quarantines the key: one decision, the
4456
+ # in-memory half suppressing the next injection and the durable half
4457
+ # clearing the stored anchor. The configurable threshold governs the
4458
+ # retirement and close funnels, where no circuit gates first. Here it
4459
+ # is unreachable by construction, which is issue #1830/#1852 itself:
4460
+ # the circuit opens at two failures and then refuses the key for
4461
+ # 60-600s per strike, so its default of seven is tens of minutes of
4462
+ # dead conversation away.
4463
+ #
4464
+ # Unlike the strike, this runs after the terminal frame is published.
4465
+ # A resend arriving in between is already covered by the quarantine
4466
+ # armed with the strike, so there is no reason to put a fenced
4467
+ # durable write in front of the client's own failure.
4468
+ #
4469
+ # The terminal frame is already on its way to the client, so
4470
+ # these awaits must not be the last thing that runs. If the
4471
+ # reader is cancelled while the episode consult's durable lookup
4472
+ # or the fenced write is in contention, the cancellation would
4473
+ # otherwise escape before the finalization below is entered and
4474
+ # the request would never be finalized despite having been
4475
+ # answered. The consult reads the currently registered episode,
4476
+ # not the detached count captured with the strike: a multiplexed
4477
+ # sibling that completed during publication has settled the
4478
+ # circuit and persisted a fresh anchor, and this clear must not
4479
+ # delete it.
4480
+ terminal_settlement_cancellation: asyncio.CancelledError | None = None
4481
+ try:
4482
+ terminal_settlement_detail = terminal_poison_detail
4483
+
4484
+ async def _terminal_consult_and_clear() -> None:
4485
+ consult_episode, consult_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
4486
+ session,
4487
+ consecutive_failures=terminal_strike_failures,
4488
+ )
4489
+ if consult_episode is not None:
4490
+ # A multiplexed survivor holding a verified safe
4491
+ # replay claims the source circuit generation right
4492
+ # before dispatch; settling under it would strip
4493
+ # that fence, so survivors count alongside the
4494
+ # terminal request, snapshotted at decision time.
4495
+ async with session.pending_lock:
4496
+ terminal_surviving_states = list(session.pending_requests)
4497
+ durable_cleared = await _abandon_durable_http_bridge_continuity(
4498
+ self,
4499
+ session,
4500
+ detail=terminal_settlement_detail,
4501
+ settle_circuit=_http_bridge_abandonment_may_settle_circuit(
4502
+ [terminal_request_state, *terminal_surviving_states]
4503
+ ),
4504
+ expected_continuity=consult_expected_anchor,
4505
+ authorized_episode=consult_episode,
4506
+ )
4507
+ if durable_cleared:
4508
+ # The abandonment succeeded even when its circuit
4509
+ # settlement remains outstanding: without the
4510
+ # marker, the next strike finds empty continuity,
4511
+ # refuses another abandonment, and the restored
4512
+ # cooldown backs off an anchor that is already
4513
+ # gone.
4514
+ await self._http_bridge_mark_poison_anchor_cleared(session, episode=consult_episode)
4515
+
4516
+ # The terminal frame is already published and the request is
4517
+ # about to be finalized and removed: a cancellation escaping
4518
+ # the consult or the fenced rebind would leave the owed clear
4519
+ # with no lifecycle left to retry it, and once the
4520
+ # process-local quarantine expires the stored dead anchor is
4521
+ # reusable. Defer cancellation across the settlement like the
4522
+ # grouped and reader funnels, finalize regardless, then
4523
+ # re-raise.
4524
+ terminal_settlement_task = scheduler.create_task(
4525
+ _terminal_consult_and_clear(),
4526
+ name=f"http-bridge-terminal-poison-settlement-{session.durable_session_id}",
4527
+ )
4528
+ _terminal_result, terminal_settlement_cancellation = await _await_task_deferring_cancellation(
4529
+ terminal_settlement_task
4530
+ )
4531
+ finally:
4532
+ await _finalize_terminal_settlement(terminal_request_state)
4533
+ if terminal_settlement_cancellation is None:
4534
+ terminal_settlement_cancellation = terminal_publication_cancellation
4535
+ if terminal_settlement_cancellation is not None:
4536
+ raise terminal_settlement_cancellation
4537
+ else:
4538
+ await _finalize_terminal_settlement(terminal_request_state)