codexLB-up 1.25.0b10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (827) hide show
  1. app/__init__.py +10 -0
  2. app/admin_cli.py +494 -0
  3. app/cli.py +339 -0
  4. app/codex_sessions_retag.py +487 -0
  5. app/core/__init__.py +0 -0
  6. app/core/audit/__init__.py +0 -0
  7. app/core/audit/service.py +216 -0
  8. app/core/audit/types.py +94 -0
  9. app/core/auth/__init__.py +236 -0
  10. app/core/auth/api_key_cache.py +65 -0
  11. app/core/auth/dashboard_access.py +429 -0
  12. app/core/auth/dashboard_mode.py +171 -0
  13. app/core/auth/dashboard_session_ttl.py +74 -0
  14. app/core/auth/dashboard_users_cache.py +120 -0
  15. app/core/auth/dependencies.py +710 -0
  16. app/core/auth/external_identity.py +78 -0
  17. app/core/auth/guardian.py +374 -0
  18. app/core/auth/models.py +49 -0
  19. app/core/auth/providers/__init__.py +129 -0
  20. app/core/auth/providers/oidc.py +838 -0
  21. app/core/auth/providers/registry.py +118 -0
  22. app/core/auth/refresh.py +394 -0
  23. app/core/auth/step_up.py +117 -0
  24. app/core/auth/totp.py +81 -0
  25. app/core/balancer/__init__.py +71 -0
  26. app/core/balancer/logic.py +1612 -0
  27. app/core/balancer/rendezvous_hash.py +21 -0
  28. app/core/balancer/types.py +21 -0
  29. app/core/bootstrap.py +172 -0
  30. app/core/cache/invalidation.py +426 -0
  31. app/core/clients/__init__.py +0 -0
  32. app/core/clients/codex.py +996 -0
  33. app/core/clients/codex_version.py +187 -0
  34. app/core/clients/codex_version_snapshot.py +2 -0
  35. app/core/clients/files.py +534 -0
  36. app/core/clients/headers.py +29 -0
  37. app/core/clients/http.py +511 -0
  38. app/core/clients/model_fetcher.py +276 -0
  39. app/core/clients/native_egress.py +1304 -0
  40. app/core/clients/oauth.py +477 -0
  41. app/core/clients/proxy.py +6089 -0
  42. app/core/clients/proxy_websocket.py +1411 -0
  43. app/core/clients/rate_limit_reset_credits.py +412 -0
  44. app/core/clients/stream_errors.py +12 -0
  45. app/core/clients/thread_cache_identity.py +347 -0
  46. app/core/clients/usage.py +467 -0
  47. app/core/clock.py +170 -0
  48. app/core/config/__init__.py +0 -0
  49. app/core/config/background_jobs.py +73 -0
  50. app/core/config/context_window_overrides.py +166 -0
  51. app/core/config/dashboard_overrides.py +145 -0
  52. app/core/config/inheritable.py +47 -0
  53. app/core/config/key_fingerprint.py +115 -0
  54. app/core/config/settings.py +872 -0
  55. app/core/config/settings_cache.py +100 -0
  56. app/core/config/spool_retention.py +217 -0
  57. app/core/config/tiers.py +174 -0
  58. app/core/conversation_archive.py +568 -0
  59. app/core/crypto.py +37 -0
  60. app/core/draining.py +11 -0
  61. app/core/errors.py +365 -0
  62. app/core/exceptions.py +155 -0
  63. app/core/handlers/__init__.py +3 -0
  64. app/core/handlers/exceptions.py +478 -0
  65. app/core/http_protocol.py +176 -0
  66. app/core/http_protocol_httptools.py +204 -0
  67. app/core/ingress_limits.py +17 -0
  68. app/core/metadata_store.py +18 -0
  69. app/core/metrics/__init__.py +0 -0
  70. app/core/metrics/middleware.py +74 -0
  71. app/core/metrics/prometheus.py +604 -0
  72. app/core/middleware/__init__.py +25 -0
  73. app/core/middleware/api_firewall.py +111 -0
  74. app/core/middleware/app_version.py +37 -0
  75. app/core/middleware/dashboard_auth_proxy.py +55 -0
  76. app/core/middleware/dashboard_csrf.py +177 -0
  77. app/core/middleware/dashboard_gzip.py +41 -0
  78. app/core/middleware/dashboard_overrides.py +59 -0
  79. app/core/middleware/firewall_cache.py +91 -0
  80. app/core/middleware/inflight.py +126 -0
  81. app/core/middleware/multipart_content_encoding.py +129 -0
  82. app/core/middleware/path_rewrite.py +148 -0
  83. app/core/middleware/request_body_limit.py +196 -0
  84. app/core/middleware/request_decompression.py +209 -0
  85. app/core/middleware/request_id.py +72 -0
  86. app/core/middleware/required_capability_http.py +116 -0
  87. app/core/middleware/trusted_proxy_headers.py +36 -0
  88. app/core/multipart.py +237 -0
  89. app/core/multipart_fields.py +91 -0
  90. app/core/openai/__init__.py +0 -0
  91. app/core/openai/chat_requests.py +493 -0
  92. app/core/openai/chat_responses.py +931 -0
  93. app/core/openai/contracts.py +46 -0
  94. app/core/openai/exceptions.py +24 -0
  95. app/core/openai/host_models.py +14 -0
  96. app/core/openai/images.py +413 -0
  97. app/core/openai/message_coercion.py +376 -0
  98. app/core/openai/model_refresh_scheduler.py +423 -0
  99. app/core/openai/model_registry.py +997 -0
  100. app/core/openai/model_registry_store.py +548 -0
  101. app/core/openai/models.py +172 -0
  102. app/core/openai/parsing.py +97 -0
  103. app/core/openai/requests.py +1980 -0
  104. app/core/openai/strict_schema.py +199 -0
  105. app/core/openai/tool_call_safety.py +99 -0
  106. app/core/openai/v1_requests.py +187 -0
  107. app/core/plan_types.py +90 -0
  108. app/core/prestop.py +202 -0
  109. app/core/rate_limiter/__init__.py +0 -0
  110. app/core/rate_limiter/db_rate_limiter.py +215 -0
  111. app/core/request_locality.py +473 -0
  112. app/core/resilience/__init__.py +0 -0
  113. app/core/resilience/backpressure.py +57 -0
  114. app/core/resilience/bulkhead.py +202 -0
  115. app/core/resilience/circuit_breaker.py +150 -0
  116. app/core/resilience/degradation.py +55 -0
  117. app/core/resilience/loop_lag_monitor.py +60 -0
  118. app/core/resilience/memory_monitor.py +137 -0
  119. app/core/resilience/network_recovery.py +256 -0
  120. app/core/resilience/overload.py +98 -0
  121. app/core/resilience/toggles.py +114 -0
  122. app/core/retention/__init__.py +0 -0
  123. app/core/retention/job.py +240 -0
  124. app/core/retention/scheduler.py +69 -0
  125. app/core/runtime_logging.py +601 -0
  126. app/core/scheduling/__init__.py +0 -0
  127. app/core/scheduling/leader_election.py +993 -0
  128. app/core/scheduling/leader_election_handle.py +25 -0
  129. app/core/server.py +140 -0
  130. app/core/shutdown.py +326 -0
  131. app/core/socket_peer.py +22 -0
  132. app/core/startup.py +41 -0
  133. app/core/timeout_invariants.py +347 -0
  134. app/core/tracing/__init__.py +0 -0
  135. app/core/tracing/otel.py +207 -0
  136. app/core/types.py +6 -0
  137. app/core/upstream_proxy/__init__.py +13 -0
  138. app/core/upstream_proxy/cache.py +130 -0
  139. app/core/upstream_proxy/resolver.py +189 -0
  140. app/core/upstream_proxy/types.py +104 -0
  141. app/core/usage/__init__.py +407 -0
  142. app/core/usage/depletion.py +109 -0
  143. app/core/usage/live_hub.py +41 -0
  144. app/core/usage/live_snapshots.py +174 -0
  145. app/core/usage/logs.py +212 -0
  146. app/core/usage/metadata_scheduler.py +118 -0
  147. app/core/usage/models.py +54 -0
  148. app/core/usage/pricing.py +576 -0
  149. app/core/usage/pricing_catalog.py +241 -0
  150. app/core/usage/pricing_snapshot.json +821 -0
  151. app/core/usage/quota.py +133 -0
  152. app/core/usage/refresh_policy.py +21 -0
  153. app/core/usage/refresh_scheduler.py +713 -0
  154. app/core/usage/reset_credits_refresh_scheduler.py +435 -0
  155. app/core/usage/types.py +168 -0
  156. app/core/utils/__init__.py +0 -0
  157. app/core/utils/json_guards.py +18 -0
  158. app/core/utils/locks.py +33 -0
  159. app/core/utils/masking.py +20 -0
  160. app/core/utils/proxy_env.py +104 -0
  161. app/core/utils/request_id.py +65 -0
  162. app/core/utils/retry.py +77 -0
  163. app/core/utils/shared_future.py +172 -0
  164. app/core/utils/sse.py +286 -0
  165. app/core/utils/time.py +24 -0
  166. app/db/__init__.py +0 -0
  167. app/db/account_identity_lock.py +56 -0
  168. app/db/alembic/__init__.py +1 -0
  169. app/db/alembic/env.py +70 -0
  170. app/db/alembic/revision_ids.py +43 -0
  171. app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
  172. app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
  173. app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
  174. app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
  175. app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
  176. app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
  177. app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
  178. app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
  179. app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
  180. app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
  181. app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
  182. app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
  183. app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
  184. app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
  185. app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
  186. app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
  187. app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
  188. app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
  189. app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
  190. app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
  191. app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
  192. app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
  193. app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
  194. app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
  195. app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
  196. app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
  197. app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
  198. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
  199. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
  200. app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
  201. app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
  202. app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
  203. app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
  204. app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
  205. app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
  206. app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
  207. app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
  208. app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
  209. app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
  210. app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
  211. app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
  212. app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
  213. app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
  214. app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
  215. app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
  216. app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
  217. app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
  218. app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
  219. app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
  220. app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
  221. app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
  222. app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
  223. app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
  224. app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
  225. app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
  226. app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
  227. app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
  228. app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
  229. app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
  230. app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
  231. app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
  232. app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
  233. app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
  234. app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
  235. app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
  236. app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
  237. app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
  238. app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
  239. app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
  240. app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
  241. app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
  242. app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
  243. app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
  244. app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
  245. app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
  246. app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
  247. app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
  248. app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
  249. app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
  250. app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
  251. app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
  252. app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
  253. app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
  254. app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
  255. app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
  256. app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
  257. app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
  258. app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
  259. app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
  260. app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
  261. app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
  262. app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
  263. app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
  264. app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
  265. app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
  266. app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
  267. app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
  268. app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
  269. app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
  270. app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
  271. app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
  272. app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
  273. app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
  274. app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
  275. app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
  276. app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
  277. app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
  278. app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
  279. app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
  280. app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
  281. app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
  282. app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
  283. app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
  284. app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
  285. app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
  286. app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
  287. app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
  288. app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
  289. app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
  290. app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
  291. app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
  292. app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
  293. app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
  294. app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
  295. app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
  296. app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
  297. app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
  298. app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
  299. app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
  300. app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
  301. app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
  302. app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
  303. app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
  304. app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
  305. app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
  306. app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
  307. app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
  308. app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
  309. app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
  310. app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
  311. app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
  312. app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
  313. app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
  314. app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
  315. app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
  316. app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
  317. app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
  318. app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
  319. app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
  320. app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
  321. app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
  322. app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
  323. app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
  324. app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
  325. app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
  326. app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
  327. app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
  328. app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
  329. app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
  330. app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
  331. app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
  332. app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
  333. app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
  334. app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
  335. app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
  336. app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
  337. app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
  338. app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
  339. app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
  340. app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
  341. app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
  342. app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
  343. app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
  344. app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
  345. app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
  346. app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
  347. app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
  348. app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
  349. app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
  350. app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
  351. app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
  352. app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
  353. app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
  354. app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
  355. app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
  356. app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
  357. app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
  358. app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
  359. app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
  360. app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
  361. app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
  362. app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
  363. app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
  364. app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
  365. app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
  366. app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
  367. app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
  368. app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
  369. app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
  370. app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
  371. app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
  372. app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
  373. app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
  374. app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
  375. app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
  376. app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
  377. app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
  378. app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
  379. app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
  380. app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
  381. app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
  382. app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
  383. app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
  384. app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
  385. app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
  386. app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
  387. app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
  388. app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
  389. app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
  390. app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
  391. app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
  392. app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
  393. app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
  394. app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
  395. app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
  396. app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
  397. app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
  398. app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
  399. app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
  400. app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
  401. app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
  402. app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
  403. app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
  404. app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
  405. app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
  406. app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
  407. app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
  408. app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
  409. app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
  410. app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
  411. app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
  412. app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
  413. app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
  414. app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
  415. app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
  416. app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
  417. app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
  418. app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
  419. app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
  420. app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
  421. app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
  422. app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
  423. app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
  424. app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
  425. app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
  426. app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
  427. app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
  428. app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
  429. app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
  430. app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
  431. app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
  432. app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
  433. app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
  434. app/db/alembic/versions/__init__.py +1 -0
  435. app/db/backup.py +64 -0
  436. app/db/migrate.py +1249 -0
  437. app/db/migration_lock.py +160 -0
  438. app/db/migration_url.py +18 -0
  439. app/db/models.py +3100 -0
  440. app/db/recover.py +244 -0
  441. app/db/session.py +1295 -0
  442. app/db/snapshot.py +46 -0
  443. app/db/sqlite_lock_retry.py +190 -0
  444. app/db/sqlite_utils.py +549 -0
  445. app/dependencies.py +462 -0
  446. app/main.py +1240 -0
  447. app/modules/__init__.py +0 -0
  448. app/modules/accounts/__init__.py +0 -0
  449. app/modules/accounts/api.py +426 -0
  450. app/modules/accounts/auth_manager.py +1439 -0
  451. app/modules/accounts/background_repository.py +160 -0
  452. app/modules/accounts/deletion.py +540 -0
  453. app/modules/accounts/mappers.py +654 -0
  454. app/modules/accounts/refresh_claims.py +352 -0
  455. app/modules/accounts/repository.py +1679 -0
  456. app/modules/accounts/schemas.py +308 -0
  457. app/modules/accounts/service.py +809 -0
  458. app/modules/accounts/usage_rollup.py +486 -0
  459. app/modules/accounts/usage_rollup_scheduler.py +75 -0
  460. app/modules/accounts/usage_time_rollup.py +1037 -0
  461. app/modules/accounts/usage_time_rollup_read.py +522 -0
  462. app/modules/api_keys/__init__.py +1 -0
  463. app/modules/api_keys/api.py +344 -0
  464. app/modules/api_keys/last_used_coalescer.py +216 -0
  465. app/modules/api_keys/limit_windows.py +41 -0
  466. app/modules/api_keys/repository.py +1200 -0
  467. app/modules/api_keys/reset_scheduler.py +85 -0
  468. app/modules/api_keys/schemas.py +135 -0
  469. app/modules/api_keys/service.py +2123 -0
  470. app/modules/audit/__init__.py +0 -0
  471. app/modules/audit/api.py +88 -0
  472. app/modules/audit/repository.py +74 -0
  473. app/modules/audit/schemas.py +33 -0
  474. app/modules/audit/service.py +71 -0
  475. app/modules/auth_providers/__init__.py +0 -0
  476. app/modules/auth_providers/api.py +127 -0
  477. app/modules/auth_providers/config.py +343 -0
  478. app/modules/auth_providers/repository.py +101 -0
  479. app/modules/auth_providers/schemas.py +88 -0
  480. app/modules/auth_providers/seed.py +105 -0
  481. app/modules/auth_providers/service.py +289 -0
  482. app/modules/automations/__init__.py +1 -0
  483. app/modules/automations/api.py +369 -0
  484. app/modules/automations/repository.py +1896 -0
  485. app/modules/automations/scheduler.py +107 -0
  486. app/modules/automations/schemas.py +158 -0
  487. app/modules/automations/service.py +2548 -0
  488. app/modules/cache_isolation_probe/__init__.py +0 -0
  489. app/modules/cache_isolation_probe/api.py +188 -0
  490. app/modules/cache_isolation_probe/prefix.py +144 -0
  491. app/modules/cache_isolation_probe/schemas.py +94 -0
  492. app/modules/cache_isolation_probe/sender.py +229 -0
  493. app/modules/cache_isolation_probe/service.py +525 -0
  494. app/modules/conversation_archive/__init__.py +1 -0
  495. app/modules/conversation_archive/api.py +101 -0
  496. app/modules/conversation_archive/schemas.py +28 -0
  497. app/modules/conversation_archive/service.py +165 -0
  498. app/modules/dashboard/__init__.py +1 -0
  499. app/modules/dashboard/api.py +106 -0
  500. app/modules/dashboard/builders.py +107 -0
  501. app/modules/dashboard/repository.py +304 -0
  502. app/modules/dashboard/schemas.py +148 -0
  503. app/modules/dashboard/service.py +534 -0
  504. app/modules/dashboard/timeframes.py +19 -0
  505. app/modules/dashboard/weekly_pace.py +617 -0
  506. app/modules/dashboard_auth/__init__.py +1 -0
  507. app/modules/dashboard_auth/api.py +1344 -0
  508. app/modules/dashboard_auth/oidc_api.py +753 -0
  509. app/modules/dashboard_auth/oidc_flows.py +330 -0
  510. app/modules/dashboard_auth/repository.py +452 -0
  511. app/modules/dashboard_auth/schemas.py +232 -0
  512. app/modules/dashboard_auth/service.py +1384 -0
  513. app/modules/dashboard_roles/__init__.py +0 -0
  514. app/modules/dashboard_roles/api.py +64 -0
  515. app/modules/dashboard_roles/repository.py +54 -0
  516. app/modules/dashboard_roles/schemas.py +29 -0
  517. app/modules/dashboard_roles/seed.py +80 -0
  518. app/modules/dashboard_roles/service.py +142 -0
  519. app/modules/dashboard_users/__init__.py +0 -0
  520. app/modules/dashboard_users/api.py +278 -0
  521. app/modules/dashboard_users/break_glass.py +169 -0
  522. app/modules/dashboard_users/credentials.py +18 -0
  523. app/modules/dashboard_users/identity_resolver.py +665 -0
  524. app/modules/dashboard_users/repository.py +725 -0
  525. app/modules/dashboard_users/schemas.py +117 -0
  526. app/modules/dashboard_users/service.py +1054 -0
  527. app/modules/firewall/__init__.py +1 -0
  528. app/modules/firewall/api.py +78 -0
  529. app/modules/firewall/repository.py +51 -0
  530. app/modules/firewall/schemas.py +28 -0
  531. app/modules/firewall/service.py +93 -0
  532. app/modules/fleet/__init__.py +1 -0
  533. app/modules/fleet/api.py +177 -0
  534. app/modules/fleet/mappers.py +56 -0
  535. app/modules/fleet/observability.py +372 -0
  536. app/modules/fleet/schemas.py +119 -0
  537. app/modules/health/__init__.py +0 -0
  538. app/modules/health/api.py +246 -0
  539. app/modules/health/schemas.py +27 -0
  540. app/modules/limit_warmup/__init__.py +1 -0
  541. app/modules/limit_warmup/repository.py +159 -0
  542. app/modules/limit_warmup/service.py +1001 -0
  543. app/modules/model_sources/__init__.py +1 -0
  544. app/modules/model_sources/api.py +99 -0
  545. app/modules/model_sources/catalog.py +307 -0
  546. app/modules/model_sources/forwarding.py +1435 -0
  547. app/modules/model_sources/projection.py +55 -0
  548. app/modules/model_sources/repository.py +201 -0
  549. app/modules/model_sources/schemas.py +79 -0
  550. app/modules/model_sources/selection.py +166 -0
  551. app/modules/model_sources/service.py +236 -0
  552. app/modules/oauth/__init__.py +0 -0
  553. app/modules/oauth/api.py +101 -0
  554. app/modules/oauth/repository.py +238 -0
  555. app/modules/oauth/schemas.py +49 -0
  556. app/modules/oauth/service.py +1196 -0
  557. app/modules/oauth/templates/oauth_success.html +122 -0
  558. app/modules/proxy/__init__.py +0 -0
  559. app/modules/proxy/_load_balancer/__init__.py +1 -0
  560. app/modules/proxy/_load_balancer/error_rate.py +116 -0
  561. app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
  562. app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
  563. app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
  564. app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
  565. app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
  566. app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
  567. app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
  568. app/modules/proxy/_load_balancer/tunables.py +97 -0
  569. app/modules/proxy/_load_balancer/types.py +96 -0
  570. app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
  571. app/modules/proxy/_service/__init__.py +6 -0
  572. app/modules/proxy/_service/api_key_usage.py +745 -0
  573. app/modules/proxy/_service/clock_budget.py +38 -0
  574. app/modules/proxy/_service/codex_control.py +611 -0
  575. app/modules/proxy/_service/compact.py +2211 -0
  576. app/modules/proxy/_service/file_ops.py +749 -0
  577. app/modules/proxy/_service/http_bridge/__init__.py +3 -0
  578. app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
  579. app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
  580. app/modules/proxy/_service/http_bridge/activity.py +178 -0
  581. app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
  582. app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
  583. app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
  584. app/modules/proxy/_service/http_bridge/protocol.py +167 -0
  585. app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
  586. app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
  587. app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
  588. app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
  589. app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
  590. app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
  591. app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
  592. app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
  593. app/modules/proxy/_service/observability.py +410 -0
  594. app/modules/proxy/_service/rate_limit.py +361 -0
  595. app/modules/proxy/_service/realtime_live.py +655 -0
  596. app/modules/proxy/_service/refresh.py +154 -0
  597. app/modules/proxy/_service/request_log.py +585 -0
  598. app/modules/proxy/_service/response_create.py +930 -0
  599. app/modules/proxy/_service/streaming/__init__.py +3 -0
  600. app/modules/proxy/_service/streaming/helpers.py +1317 -0
  601. app/modules/proxy/_service/streaming/mixin.py +1100 -0
  602. app/modules/proxy/_service/streaming/protocol.py +32 -0
  603. app/modules/proxy/_service/streaming/retry.py +3762 -0
  604. app/modules/proxy/_service/support.py +2122 -0
  605. app/modules/proxy/_service/transcribe.py +434 -0
  606. app/modules/proxy/_service/warmup.py +483 -0
  607. app/modules/proxy/_service/websocket/__init__.py +3 -0
  608. app/modules/proxy/_service/websocket/helpers.py +2360 -0
  609. app/modules/proxy/_service/websocket/mixin.py +7285 -0
  610. app/modules/proxy/_service/websocket/protocol.py +74 -0
  611. app/modules/proxy/account_cache.py +229 -0
  612. app/modules/proxy/account_eligibility.py +87 -0
  613. app/modules/proxy/additional_model_limits.py +40 -0
  614. app/modules/proxy/affinity.py +830 -0
  615. app/modules/proxy/affinity_observation.py +101 -0
  616. app/modules/proxy/api.py +10254 -0
  617. app/modules/proxy/api_key_usage.py +113 -0
  618. app/modules/proxy/cap_partitioning.py +280 -0
  619. app/modules/proxy/capability_lineage.py +65 -0
  620. app/modules/proxy/capability_lineage_repository.py +82 -0
  621. app/modules/proxy/capability_routing.py +253 -0
  622. app/modules/proxy/continuity.py +112 -0
  623. app/modules/proxy/downstream_delivery.py +277 -0
  624. app/modules/proxy/durable_bridge_coordinator.py +1217 -0
  625. app/modules/proxy/durable_bridge_repository.py +4516 -0
  626. app/modules/proxy/durable_bridge_runtime.py +9 -0
  627. app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
  628. app/modules/proxy/fair_share.py +131 -0
  629. app/modules/proxy/file_pin_repository.py +210 -0
  630. app/modules/proxy/helpers.py +439 -0
  631. app/modules/proxy/http_bridge_event_batcher.py +613 -0
  632. app/modules/proxy/http_bridge_forwarding.py +797 -0
  633. app/modules/proxy/http_continuation.py +76 -0
  634. app/modules/proxy/images_observability.py +62 -0
  635. app/modules/proxy/images_service.py +1054 -0
  636. app/modules/proxy/load_balancer.py +3016 -0
  637. app/modules/proxy/rate_limit_cache.py +46 -0
  638. app/modules/proxy/replay_safety.py +1063 -0
  639. app/modules/proxy/repo_bundle.py +31 -0
  640. app/modules/proxy/request_policy.py +1054 -0
  641. app/modules/proxy/ring_membership.py +251 -0
  642. app/modules/proxy/schemas.py +345 -0
  643. app/modules/proxy/selection_errors.py +41 -0
  644. app/modules/proxy/service.py +2600 -0
  645. app/modules/proxy/source_admission.py +125 -0
  646. app/modules/proxy/source_dispatch.py +999 -0
  647. app/modules/proxy/sticky_repository.py +831 -0
  648. app/modules/proxy/thread_anchors.py +491 -0
  649. app/modules/proxy/tool_call_dedupe.py +802 -0
  650. app/modules/proxy/types.py +54 -0
  651. app/modules/proxy/work_admission.py +121 -0
  652. app/modules/quota_planner/__init__.py +1 -0
  653. app/modules/quota_planner/api.py +276 -0
  654. app/modules/quota_planner/logic.py +743 -0
  655. app/modules/quota_planner/repository.py +722 -0
  656. app/modules/quota_planner/scheduler.py +212 -0
  657. app/modules/quota_planner/schemas.py +102 -0
  658. app/modules/quota_planner/warmup.py +693 -0
  659. app/modules/rate_limit_reset_credits/__init__.py +0 -0
  660. app/modules/rate_limit_reset_credits/api.py +601 -0
  661. app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
  662. app/modules/rate_limit_reset_credits/store.py +114 -0
  663. app/modules/reports/__init__.py +1 -0
  664. app/modules/reports/api.py +134 -0
  665. app/modules/reports/cache.py +74 -0
  666. app/modules/reports/filters.py +27 -0
  667. app/modules/reports/repository.py +525 -0
  668. app/modules/reports/rollup.py +150 -0
  669. app/modules/reports/rollup_read.py +148 -0
  670. app/modules/reports/schemas.py +123 -0
  671. app/modules/reports/service.py +287 -0
  672. app/modules/reports/thread_identity.py +256 -0
  673. app/modules/request_logs/__init__.py +0 -0
  674. app/modules/request_logs/api.py +224 -0
  675. app/modules/request_logs/cost_backfill.py +154 -0
  676. app/modules/request_logs/mappers.py +97 -0
  677. app/modules/request_logs/repository.py +1765 -0
  678. app/modules/request_logs/schemas.py +145 -0
  679. app/modules/request_logs/service.py +348 -0
  680. app/modules/role_mappings/__init__.py +0 -0
  681. app/modules/role_mappings/api.py +206 -0
  682. app/modules/role_mappings/matching.py +55 -0
  683. app/modules/role_mappings/repository.py +106 -0
  684. app/modules/role_mappings/schemas.py +67 -0
  685. app/modules/role_mappings/service.py +334 -0
  686. app/modules/runtime/__init__.py +1 -0
  687. app/modules/runtime/api.py +18 -0
  688. app/modules/runtime/schemas.py +14 -0
  689. app/modules/runtime/service.py +218 -0
  690. app/modules/scim/__init__.py +1 -0
  691. app/modules/scim/api.py +194 -0
  692. app/modules/scim/dependencies.py +112 -0
  693. app/modules/scim/errors.py +77 -0
  694. app/modules/scim/management_api.py +156 -0
  695. app/modules/scim/repository.py +212 -0
  696. app/modules/scim/schemas.py +145 -0
  697. app/modules/scim/service.py +786 -0
  698. app/modules/scim/tokens.py +43 -0
  699. app/modules/settings/__init__.py +1 -0
  700. app/modules/settings/api.py +1677 -0
  701. app/modules/settings/repository.py +606 -0
  702. app/modules/settings/schemas.py +496 -0
  703. app/modules/settings/service.py +769 -0
  704. app/modules/shared/__init__.py +0 -0
  705. app/modules/shared/schemas.py +22 -0
  706. app/modules/sticky_sessions/__init__.py +1 -0
  707. app/modules/sticky_sessions/api.py +118 -0
  708. app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
  709. app/modules/sticky_sessions/schemas.py +78 -0
  710. app/modules/sticky_sessions/service.py +196 -0
  711. app/modules/telemetry/__init__.py +5 -0
  712. app/modules/telemetry/api.py +115 -0
  713. app/modules/telemetry/clients.py +62 -0
  714. app/modules/telemetry/consent.py +115 -0
  715. app/modules/telemetry/scheduler.py +85 -0
  716. app/modules/telemetry/schemas.py +163 -0
  717. app/modules/telemetry/sender.py +221 -0
  718. app/modules/telemetry/snapshot.py +490 -0
  719. app/modules/usage/__init__.py +0 -0
  720. app/modules/usage/additional_quota_keys.py +379 -0
  721. app/modules/usage/api.py +45 -0
  722. app/modules/usage/background_repository.py +147 -0
  723. app/modules/usage/builders.py +559 -0
  724. app/modules/usage/depletion_service.py +277 -0
  725. app/modules/usage/live_ingest.py +356 -0
  726. app/modules/usage/mappers.py +29 -0
  727. app/modules/usage/plan_downgrade_observations.py +488 -0
  728. app/modules/usage/repository.py +1890 -0
  729. app/modules/usage/schemas.py +68 -0
  730. app/modules/usage/service.py +102 -0
  731. app/modules/usage/updater.py +1488 -0
  732. app/static/assets/Trans-Cj-J9mVA.js +1 -0
  733. app/static/assets/access-page-DHzZiXZi.js +1 -0
  734. app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
  735. app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
  736. app/static/assets/account-status-DIsU5sUS.js +1 -0
  737. app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
  738. app/static/assets/accounts-page-Bbai8ift.js +3 -0
  739. app/static/assets/api-Dwa7XxPe.js +1 -0
  740. app/static/assets/api-client-dugX1Pti.js +55 -0
  741. app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
  742. app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
  743. app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
  744. app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
  745. app/static/assets/apis-page-BvXz9c6i.js +2 -0
  746. app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
  747. app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
  748. app/static/assets/automations-page-DQgNAJOR.js +3 -0
  749. app/static/assets/badge-DqMb9es0.js +1 -0
  750. app/static/assets/blur-email-HEvKagG2.js +1 -0
  751. app/static/assets/button-BjVmmiGW.js +1 -0
  752. app/static/assets/checkbox-apsl9vzk.js +1 -0
  753. app/static/assets/chevron-left-1d_pHVa5.js +1 -0
  754. app/static/assets/circle-alert-BS0xd_o1.js +1 -0
  755. app/static/assets/clsx-DB0hHKMi.js +1 -0
  756. app/static/assets/colors-BBy8GEtr.js +1 -0
  757. app/static/assets/copy-button-D7nAvOKR.js +1 -0
  758. app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
  759. app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
  760. app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
  761. app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
  762. app/static/assets/donut-chart-Bxc69cra.js +1 -0
  763. app/static/assets/download-kQteIcsB.js +1 -0
  764. app/static/assets/ellipsis-B_5OgIre.js +1 -0
  765. app/static/assets/empty-state-JcA7tc8m.js +1 -0
  766. app/static/assets/es2015-bMH8YKZT.js +41 -0
  767. app/static/assets/es6-B264ni6W.js +69 -0
  768. app/static/assets/external-link-UVAOgMQ0.js +1 -0
  769. app/static/assets/i18n-BWqZ9Ua2.js +1 -0
  770. app/static/assets/inbox-B_RM2s1J.js +1 -0
  771. app/static/assets/index-DIpd264Z.js +23 -0
  772. app/static/assets/index-DMeLKRJx.css +2 -0
  773. app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
  774. app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
  775. app/static/assets/iso-Bpe5Ihbb.js +1 -0
  776. app/static/assets/label-CGaLmLNf.js +1 -0
  777. app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
  778. app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
  779. app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
  780. app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
  781. app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
  782. app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
  783. app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
  784. app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
  785. app/static/assets/payload-CIYnGZz9.js +1 -0
  786. app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
  787. app/static/assets/report-chart-card-BXDliwcc.js +1 -0
  788. app/static/assets/reports-page-CGCJeMHG.js +3 -0
  789. app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
  790. app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
  791. app/static/assets/search-HfhqwERt.js +1 -0
  792. app/static/assets/select-Cl33bz-g.js +1 -0
  793. app/static/assets/settings-page-mZj01HYl.js +2 -0
  794. app/static/assets/shim-CZwyM4Q7.js +1 -0
  795. app/static/assets/skeleton-C_EUc4Rp.js +1 -0
  796. app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
  797. app/static/assets/status-badge-CHHA5X6c.js +1 -0
  798. app/static/assets/switch-CzxgokcM.js +1 -0
  799. app/static/assets/table-DXaNSQ0G.js +1 -0
  800. app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
  801. app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
  802. app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
  803. app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
  804. app/static/assets/trash-D3KyWr1h.js +1 -0
  805. app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
  806. app/static/assets/use-accounts-PWlVGw3z.js +1 -0
  807. app/static/assets/use-auth-CuDlW3iC.js +1 -0
  808. app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
  809. app/static/assets/use-model-sources-Bul24JBu.js +1 -0
  810. app/static/assets/use-models-DUCPUuZg.js +1 -0
  811. app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
  812. app/static/assets/useTranslation--56nHhlH.js +1 -0
  813. app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
  814. app/static/assets/users-uIC3TIlE.js +1 -0
  815. app/static/assets/vendor-query-_bHZuxEx.js +1 -0
  816. app/static/assets/vendor-react-BVL0YZPW.js +9 -0
  817. app/static/favicon.svg +10 -0
  818. app/static/fonts/GeistSans-Variable.woff2 +0 -0
  819. app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
  820. app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
  821. app/static/index.html +29 -0
  822. codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
  823. codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
  824. codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
  825. codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
  826. codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
  827. config/additional_quota_registry.json +11 -0
@@ -0,0 +1,2442 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import logging
5
+ from dataclasses import dataclass
6
+ from types import SimpleNamespace
7
+ from typing import Any
8
+
9
+ import anyio
10
+
11
+ from app.core.clock import clock_for, scheduler_for
12
+ from app.core.errors import HTTP_BRIDGE_EVENTLESS_TIMEOUT_CODE
13
+ from app.core.metrics.prometheus import PROMETHEUS_AVAILABLE, http_bridge_retry_circuit_total
14
+ from app.modules.proxy._service.http_bridge.quarantine import (
15
+ _HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON,
16
+ _http_bridge_quarantine_clear_fence,
17
+ _http_bridge_quarantine_generation,
18
+ _http_bridge_quarantine_registry,
19
+ _http_bridge_session_key_poison_quarantined,
20
+ _quarantine_http_bridge_session,
21
+ _revoke_http_bridge_poison_quarantine,
22
+ )
23
+ from app.modules.proxy._service.observability import _hash_identifier
24
+ from app.modules.proxy._service.support import (
25
+ _HTTPBridgeResponseCreateAttempt,
26
+ _HTTPBridgeRetryCircuitAttemptSelection,
27
+ _HTTPBridgeSession,
28
+ _HTTPBridgeSessionKey,
29
+ )
30
+ from app.modules.proxy.durable_bridge_repository import DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
31
+
32
+ logger = logging.getLogger(__name__)
33
+
34
+ # Consecutive failures that open the circuit. The anchor-poison threshold is
35
+ # this same value: once the circuit opens it refuses the key for 60-600s per
36
+ # strike, so a higher poison threshold could never be reached at any useful
37
+ # rate (#1830/#1852), and the quarantine armed at the opening is process-local,
38
+ # so clearing the durable anchor no later than the opening is what keeps the
39
+ # decision replica-safe.
40
+ _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD = 2
41
+ _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS = 60.0
42
+ _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS = 600.0
43
+ _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS = 30.0
44
+ _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS = 600.0
45
+ _HTTP_BRIDGE_RETRY_CIRCUIT_CLAIM_TIMEOUT_SECONDS = 5.0
46
+ _HTTP_BRIDGE_RETRY_CIRCUIT_PLANNING_MISS_CAP = 4096
47
+ _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_DETAILS = frozenset(
48
+ {
49
+ "stream_incomplete",
50
+ "clean_close",
51
+ "stream_idle_timeout",
52
+ # Pre-response-start bridge silence. Deliberately *not* aliased onto
53
+ # stream_idle_timeout: the whole point is that a circuit opened before
54
+ # any response existed must be distinguishable from one opened by a
55
+ # started stream going quiet.
56
+ HTTP_BRIDGE_EVENTLESS_TIMEOUT_CODE,
57
+ }
58
+ )
59
+ _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES = {
60
+ # These diagnostics describe the same ambiguous idle/incomplete
61
+ # transport class. Keep the durable contract to the documented
62
+ # failure classes while retaining the more specific event in logs.
63
+ "upstream_keepalive_timeout": "stream_idle_timeout",
64
+ "missing_response_created_timeout": "stream_idle_timeout",
65
+ "response_create_gate_timeout_stuck_pending": "stream_idle_timeout",
66
+ }
67
+ # Sentinel: the consult could not capture the durable anchor; the
68
+ # abandonment then clears continuity unfenced (degraded, logged).
69
+ _POISON_ANCHOR_CAPTURE_UNAVAILABLE: object = object()
70
+ _HTTP_BRIDGE_ANCHOR_POISON_DETAILS = {
71
+ "stream_idle_timeout": "repeated_zero_event_idle_timeout",
72
+ "stream_incomplete": "repeated_zero_event_stream_incomplete",
73
+ # The pre-response eventless timeout keeps its distinguishable durable
74
+ # detail (deliberately not aliased onto stream_idle_timeout), but a
75
+ # repeatedly eventless anchored request is the original silent-wedge
76
+ # signature, so at threshold it authorizes the same fenced abandonment.
77
+ HTTP_BRIDGE_EVENTLESS_TIMEOUT_CODE: "repeated_zero_event_idle_timeout",
78
+ }
79
+ # Written over a surviving at-threshold row when a completion registered a
80
+ # fresh anchor but its settlement failed. Deliberately outside the poison
81
+ # detail map: a replica loading the row keeps the cooldown but neither arms
82
+ # quarantine against the fresh anchor nor authorizes an abandonment for
83
+ # failures recorded against the superseded one.
84
+ _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL = "anchor_superseded"
85
+ # Written onto the zeroed row when an ABANDONMENT settles the circuit: the
86
+ # continuity columns are already empty, so without a durable marker a
87
+ # restarted worker or another replica would dispatch a delta-only follow-up
88
+ # as a brand-new conversation and silently drop its context. A completion's
89
+ # settle writes None instead, erasing the tombstone on real recovery.
90
+ _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL = "anchor_abandoned"
91
+
92
+
93
+ def _http_bridge_anchor_poison_detail(detail: str | None) -> str | None:
94
+ """Map an eventless retry-circuit failure class to its anchor-poison detail.
95
+
96
+ Consecutive eventless failures on one bridge key are same-anchor failures:
97
+ the durable anchor only advances on a completed response, which resets the
98
+ circuit. Both ambiguous transport classes therefore count toward anchor
99
+ poison (issue #1830); ``clean_close`` never does.
100
+ """
101
+ if detail is None:
102
+ return None
103
+ aliased = _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES.get(detail, detail)
104
+ return _HTTP_BRIDGE_ANCHOR_POISON_DETAILS.get(aliased)
105
+
106
+
107
+ def _http_bridge_poison_quarantine_minimum_seconds(cooldown_remaining: float) -> float:
108
+ """Keep a poison quarantine alive across the cooldown and its probe window.
109
+
110
+ The probe that has to be planned unanchored is only admitted once the
111
+ cooldown expires, and it may then be admitted anywhere inside the
112
+ half-open lease that follows. Quarantine's default TTL equals the maximum
113
+ cooldown, so at that cooldown both would lapse in the same instant and the
114
+ probe would be planned with the anchor the circuit opened on. This is the
115
+ same span the durable retry-circuit row already reserves for itself.
116
+ """
117
+ return max(0.0, cooldown_remaining) + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS
118
+
119
+
120
+ def _http_bridge_retry_circuit_suppression_message(block_reason: str, retry_after_seconds: int) -> str:
121
+ """Describe the timer that is actually refusing a suppressed submission.
122
+
123
+ Naming the cooldown while the half-open lease is what refuses the request
124
+ tells the client to come back in about a second when it is barred for the
125
+ rest of the lease, which turns one wedged key into a retry storm.
126
+ """
127
+ if block_reason == "hard_key_half_open":
128
+ return (
129
+ "HTTP responses session bridge is probing recovery for this conversation; "
130
+ f"retry after {retry_after_seconds}s."
131
+ )
132
+ return (
133
+ "HTTP responses session bridge is recovering from repeated upstream failures; "
134
+ f"retry after {retry_after_seconds}s."
135
+ )
136
+
137
+
138
+ @dataclass(slots=True)
139
+ class _HTTPBridgeRetryCircuitState:
140
+ consecutive_failures: int = 0
141
+ cooldown_until: float = 0.0
142
+ last_detail: str | None = None
143
+ last_touched_monotonic: float = 0.0
144
+ persisted_updated_at_epoch: float = 0.0
145
+ # Admission generation observed with the epoch above: a replica's replay
146
+ # claim bumps only this column, so the settle fence must carry it or a
147
+ # reset can clear the circuit beneath a claim it never observed.
148
+ persisted_admission_generation: int = 0
149
+ last_failure_monotonic: float = 0.0
150
+ last_durable_load_monotonic: float = 0.0
151
+ half_open_until: float = 0.0
152
+ # One poisoned anchor is abandoned once. Capping the poison threshold at
153
+ # the circuit threshold makes every later strike in the same episode meet
154
+ # it too, so without this marker each one re-issues the durable clear. A
155
+ # failed clear leaves it unset, which is what lets the next strike retry.
156
+ poison_anchor_cleared: bool = False
157
+ # The poison class this episode still owes an abandonment for. A later
158
+ # non-poison strike (an unanchored probe closing clean) overwrites
159
+ # ``last_detail`` durably, and without this record the owed clear would
160
+ # never be retried and other replicas would stop arming against it.
161
+ owed_poison_detail: str | None = None
162
+
163
+
164
+ @dataclass(slots=True)
165
+ class _HTTPBridgeRetryCircuitKeyProbe:
166
+ """Key-scoped stand-in for the planning-time first-touch load.
167
+
168
+ Planning runs before any bridge session exists, but the load path and the
169
+ quarantine arm only need the key plus logging context; the probe carries
170
+ exactly that so an at-threshold poison row recorded by another replica can
171
+ arm this worker's quarantine before the anchor is planned into a payload.
172
+ """
173
+
174
+ key: _HTTPBridgeSessionKey
175
+ request_model: str | None = None
176
+ account: Any = None
177
+ quarantined: bool = False
178
+
179
+ def __post_init__(self) -> None:
180
+ if self.account is None:
181
+ self.account = SimpleNamespace(id=None)
182
+
183
+
184
+ def _initialize_http_bridge_retry_circuit(service: Any, reset_transient_cache: Any = None) -> None:
185
+ if reset_transient_cache is not None:
186
+ reset_transient_cache()
187
+ service._http_bridge_retry_circuits = {}
188
+ service._http_bridge_retry_circuit_loaded_keys = set()
189
+ service._http_bridge_retry_circuit_persisted_keys = set()
190
+ service._http_bridge_retry_circuit_lock = anyio.Lock()
191
+ service._http_bridge_retry_circuit_key_locks = {}
192
+ # A settlement pops the state object, taking its stale-load watermark
193
+ # with it; this per-key watermark survives the pop so a lookup that
194
+ # began before the settlement cannot adopt the pre-settlement row into
195
+ # a fresh state and resurrect the settled cooldown.
196
+ service._http_bridge_retry_circuit_reconcile_watermarks = {}
197
+ # Timestamped negative cache for confirmed durable misses: healthy hard
198
+ # keys have no circuit row, and without this every planning pass pays a
199
+ # durable read on top of the submit-time load. Entries live for the
200
+ # planning window only; a row another replica creates after a cached
201
+ # miss is still enforced at submission while its cooldown runs.
202
+ service._http_bridge_retry_circuit_planning_misses = {}
203
+
204
+
205
+ def _record_http_bridge_retry_circuit_duplicate_suppressed(
206
+ session: _HTTPBridgeSession,
207
+ *,
208
+ attempt: _HTTPBridgeResponseCreateAttempt,
209
+ consecutive_failures: int,
210
+ detail: str,
211
+ ) -> None:
212
+ if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
213
+ http_bridge_retry_circuit_total.labels(outcome="duplicate_suppressed").inc()
214
+ logger.info(
215
+ "http_bridge_retry_circuit event=duplicate_suppressed bridge_kind=%s bridge_key=%s "
216
+ "failures=%s detail=%s attempt=%s",
217
+ session.key.affinity_kind,
218
+ _hash_identifier(session.key.affinity_key),
219
+ consecutive_failures,
220
+ detail,
221
+ attempt.ordinal,
222
+ )
223
+
224
+
225
+ class _HTTPBridgeRetryCircuitMixin:
226
+ async def _http_bridge_retry_circuit_generation(
227
+ self: Any,
228
+ session: _HTTPBridgeSession,
229
+ ) -> tuple[bool, tuple[int, float, int, float, int, float, float] | None]:
230
+ return await self._http_bridge_retry_circuit_generation_for_key(session.key)
231
+
232
+ async def _http_bridge_retry_circuit_generation_for_key(
233
+ self: Any,
234
+ key: _HTTPBridgeSessionKey,
235
+ ) -> tuple[bool, tuple[int, float, int, float, int, float, float] | None]:
236
+ try:
237
+ persisted = await self._durable_bridge.lookup_retry_circuit(
238
+ session_key_kind=key.affinity_kind,
239
+ session_key_value=key.affinity_key,
240
+ api_key_id=key.api_key_id,
241
+ )
242
+ except Exception:
243
+ logger.warning(
244
+ "Failed to inspect HTTP bridge retry circuit generation bridge_kind=%s bridge_key=%s",
245
+ key.affinity_kind,
246
+ _hash_identifier(key.affinity_key),
247
+ exc_info=True,
248
+ )
249
+ return False, None
250
+ async with self._http_bridge_retry_circuit_lock:
251
+ state = self._http_bridge_retry_circuits.get(key)
252
+ if state is None and persisted is None:
253
+ return True, None
254
+ persisted_updated_at_epoch = max(
255
+ state.persisted_updated_at_epoch if state is not None else 0.0,
256
+ persisted.updated_at_epoch if persisted is not None else 0.0,
257
+ )
258
+ persisted_consecutive_failures = persisted.consecutive_failures if persisted is not None else 0
259
+ durable_cooldown_until_epoch = persisted.cooldown_until_epoch if persisted is not None else 0.0
260
+ local_consecutive_failures = state.consecutive_failures if state is not None else 0
261
+ last_failure_monotonic = state.last_failure_monotonic if state is not None else 0.0
262
+ local_cooldown_until = state.cooldown_until if state is not None else 0.0
263
+ return True, (
264
+ getattr(persisted, "admission_generation", 0) if persisted is not None else 0,
265
+ persisted_updated_at_epoch,
266
+ persisted_consecutive_failures,
267
+ durable_cooldown_until_epoch,
268
+ local_consecutive_failures,
269
+ last_failure_monotonic,
270
+ local_cooldown_until,
271
+ )
272
+
273
+ async def _http_bridge_retry_circuit_generation_is_not_newer(
274
+ self: Any,
275
+ *,
276
+ key: _HTTPBridgeSessionKey,
277
+ captured: bool,
278
+ generation: tuple[int, float, int, float, int, float, float] | None,
279
+ ) -> bool:
280
+ if not captured:
281
+ return False
282
+ load_succeeded, current_generation = await self._http_bridge_retry_circuit_generation_for_key(key)
283
+ if not load_succeeded:
284
+ return False
285
+ if generation is None:
286
+ return current_generation is None
287
+ if current_generation is None:
288
+ return True
289
+ return all(
290
+ current <= captured_value for current, captured_value in zip(current_generation, generation, strict=True)
291
+ )
292
+
293
+ async def _claim_http_bridge_retry_circuit_generation(
294
+ self: Any,
295
+ *,
296
+ key: _HTTPBridgeSessionKey,
297
+ captured: bool,
298
+ generation: tuple[int, float, int, float, int, float, float] | None,
299
+ ) -> bool | None:
300
+ """Atomically linearize replay admission against the captured circuit.
301
+
302
+ ``True`` means claimed; ``False`` means a confirmed loss — the durable
303
+ CAS answered and matched nothing, or local state already advanced past
304
+ the capture; ``None`` means the claim could not be decided (timeout,
305
+ durable error, or the capability being absent), which is
306
+ infrastructure trouble rather than evidence that a remote probe holds
307
+ the lease.
308
+ """
309
+ if not captured:
310
+ return False
311
+ expected_admission_generation = generation[0] if generation is not None else 0
312
+ expected_persisted_updated_at = generation[1] if generation is not None else 0.0
313
+ expected_persisted_failures = generation[2] if generation is not None else 0
314
+ expected_persisted_cooldown = generation[3] if generation is not None else 0.0
315
+ expected_local_failures = generation[4] if generation is not None else 0
316
+ expected_last_failure = generation[5] if generation is not None else 0.0
317
+ expected_local_cooldown = generation[6] if generation is not None else 0.0
318
+ claim_generation = getattr(self._durable_bridge, "claim_retry_circuit_generation", None)
319
+ if not callable(claim_generation):
320
+ return None
321
+
322
+ # Local failure recording for this key serializes on the key lock,
323
+ # so holding it across the durable CAS keeps the claim linearized
324
+ # against local strikes without parking every unrelated hard key
325
+ # behind the global registry lock for up to the claim timeout. The
326
+ # global lock is held only for the short state checks.
327
+ key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(key)
328
+ try:
329
+ async with self._http_bridge_retry_circuit_lock:
330
+ state = self._http_bridge_retry_circuits.get(key)
331
+ if state is not None and (
332
+ state.consecutive_failures > expected_local_failures
333
+ or state.last_failure_monotonic > expected_last_failure
334
+ or state.cooldown_until > expected_local_cooldown
335
+ ):
336
+ return False
337
+ try:
338
+ claimed = await scheduler_for(self).wait_for(
339
+ claim_generation(
340
+ session_key_kind=key.affinity_kind,
341
+ session_key_value=key.affinity_key,
342
+ api_key_id=key.api_key_id,
343
+ expected_updated_at_epoch=(
344
+ expected_persisted_updated_at if expected_persisted_updated_at > 0 else None
345
+ ),
346
+ expected_admission_generation=expected_admission_generation,
347
+ expected_consecutive_failures=expected_persisted_failures,
348
+ expected_cooldown_until_epoch=expected_persisted_cooldown,
349
+ ),
350
+ timeout=_HTTP_BRIDGE_RETRY_CIRCUIT_CLAIM_TIMEOUT_SECONDS,
351
+ )
352
+ except Exception:
353
+ logger.warning(
354
+ "Failed to claim HTTP bridge retry circuit generation bridge_kind=%s bridge_key=%s",
355
+ key.affinity_kind,
356
+ _hash_identifier(key.affinity_key),
357
+ exc_info=True,
358
+ )
359
+ return None
360
+ if claimed is None:
361
+ return False
362
+ async with self._http_bridge_retry_circuit_lock:
363
+ self._http_bridge_retry_circuit_loaded_keys.add(key)
364
+ self._http_bridge_retry_circuit_persisted_keys.add(key)
365
+ return True
366
+ finally:
367
+ key_lock.release()
368
+
369
+ async def _ensure_http_bridge_retry_circuit_loaded_for_key(
370
+ self: Any,
371
+ key: _HTTPBridgeSessionKey,
372
+ *,
373
+ request_model: str | None,
374
+ ) -> None:
375
+ """Planning-time durable load so a poison row arms quarantine pre-planning.
376
+
377
+ The submit-time load runs after the payload is already planned and
378
+ serialized, so on an expired at-threshold poison key the probe would
379
+ carry the poisoned anchor. The cached view is honored only while it
380
+ is younger than the minimum cooldown: a circuit another replica
381
+ opens after this worker's last load is then either still cooling
382
+ (the submit-time gate suppresses the request before dispatch) or
383
+ refreshed here before its expired cooldown can admit a probe, so a
384
+ cached below-threshold or reset row cannot hide a remote opening
385
+ from the anchor decision.
386
+ """
387
+ if key.strength != "hard":
388
+ return
389
+ async with self._http_bridge_retry_circuit_lock:
390
+ if key in self._http_bridge_retry_circuit_loaded_keys:
391
+ state = self._http_bridge_retry_circuits.get(key)
392
+ if (
393
+ state is not None
394
+ and clock_for(self).monotonic() - state.last_durable_load_monotonic
395
+ <= _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS
396
+ ):
397
+ return
398
+ missed_at = self._http_bridge_retry_circuit_planning_misses.get(key)
399
+ if (
400
+ missed_at is not None
401
+ and clock_for(self).monotonic() - missed_at <= _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS
402
+ ):
403
+ # A confirmed durable miss within the planning window: a row
404
+ # another replica created since is still enforced by the
405
+ # submit-time load while its cooldown runs, so planning does
406
+ # not need a second round trip per healthy request.
407
+ return
408
+ probe: Any = _HTTPBridgeRetryCircuitKeyProbe(key=key, request_model=request_model)
409
+ await self._load_http_bridge_retry_circuit(probe)
410
+
411
+ async def _http_bridge_retry_circuit_current_count(self: Any, session: _HTTPBridgeSession) -> int:
412
+ async with self._http_bridge_retry_circuit_lock:
413
+ current_state = self._http_bridge_retry_circuits.get(session.key)
414
+ return current_state.consecutive_failures if current_state is not None else 0
415
+
416
+ async def _await_http_bridge_retry_circuit_attempt_settlement(
417
+ self: Any,
418
+ session: _HTTPBridgeSession,
419
+ *,
420
+ attempt: _HTTPBridgeResponseCreateAttempt,
421
+ detail: str,
422
+ ) -> int:
423
+ settled = attempt.retry_circuit_failure_settled
424
+ if settled is not None:
425
+ await settled.wait()
426
+ consecutive_failures = await self._http_bridge_retry_circuit_current_count(session)
427
+ _record_http_bridge_retry_circuit_duplicate_suppressed(
428
+ session,
429
+ attempt=attempt,
430
+ consecutive_failures=consecutive_failures,
431
+ detail=detail,
432
+ )
433
+ return consecutive_failures
434
+
435
+ async def _record_http_bridge_retry_circuit_failure_for_attempt_selection(
436
+ self: Any,
437
+ session: _HTTPBridgeSession,
438
+ *,
439
+ detail: str,
440
+ selection: _HTTPBridgeRetryCircuitAttemptSelection,
441
+ ) -> int | None:
442
+ attempt = selection.attempt
443
+ if attempt is not None:
444
+ return await self._record_http_bridge_retry_circuit_failure(
445
+ session,
446
+ detail=detail,
447
+ attempt=attempt,
448
+ )
449
+ if selection.kind == "absent":
450
+ return await self._record_http_bridge_retry_circuit_failure(session, detail=detail)
451
+ if selection.kind == "recorded":
452
+ for recorded_attempt in selection.attempts:
453
+ settled = recorded_attempt.retry_circuit_failure_settled
454
+ if settled is not None:
455
+ await settled.wait()
456
+ consecutive_failures = await self._http_bridge_retry_circuit_current_count(session)
457
+ for recorded_attempt in selection.attempts:
458
+ _record_http_bridge_retry_circuit_duplicate_suppressed(
459
+ session,
460
+ attempt=recorded_attempt,
461
+ consecutive_failures=consecutive_failures,
462
+ detail=detail,
463
+ )
464
+ return consecutive_failures
465
+
466
+ outcome = "ambiguous_suppressed" if selection.ambiguous else "ineligible_suppressed"
467
+ if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
468
+ http_bridge_retry_circuit_total.labels(outcome=outcome).inc()
469
+ logger.info(
470
+ "http_bridge_retry_circuit event=%s bridge_kind=%s bridge_key=%s detail=%s candidate_attempts=%s",
471
+ outcome,
472
+ session.key.affinity_kind,
473
+ _hash_identifier(session.key.affinity_key),
474
+ detail,
475
+ len(selection.attempts),
476
+ )
477
+ return None
478
+
479
+ def _prune_http_bridge_retry_circuit_state(self: Any, now: float) -> None:
480
+ expiry = now - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
481
+ for key, state in list(self._http_bridge_retry_circuits.items()):
482
+ if state.last_touched_monotonic > expiry:
483
+ continue
484
+ self._http_bridge_retry_circuits.pop(key, None)
485
+ self._http_bridge_retry_circuit_loaded_keys.discard(key)
486
+ self._http_bridge_retry_circuit_persisted_keys.discard(key)
487
+ for key, watermark in list(self._http_bridge_retry_circuit_reconcile_watermarks.items()):
488
+ if watermark <= expiry:
489
+ self._http_bridge_retry_circuit_reconcile_watermarks.pop(key, None)
490
+ miss_expiry = now - _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS
491
+ misses = self._http_bridge_retry_circuit_planning_misses
492
+ # Entries are kept in insertion order (a re-record pops first), so
493
+ # expiry only ever sweeps the front. A full-map scan here would run
494
+ # under the shared circuit lock on every load and cost R²·window
495
+ # entry checks per second at R new healthy keys per second.
496
+ while misses:
497
+ oldest_key = next(iter(misses))
498
+ if misses[oldest_key] > miss_expiry:
499
+ break
500
+ del misses[oldest_key]
501
+ while len(misses) > _HTTP_BRIDGE_RETRY_CIRCUIT_PLANNING_MISS_CAP:
502
+ # The cap sheds the oldest confirmed misses first; an evicted
503
+ # key merely pays one planning-time load again.
504
+ del misses[next(iter(misses))]
505
+ for key, key_lock in list(self._http_bridge_retry_circuit_key_locks.items()):
506
+ if key not in self._http_bridge_retry_circuits and not key_lock.locked():
507
+ self._http_bridge_retry_circuit_key_locks.pop(key, None)
508
+
509
+ async def _acquire_http_bridge_retry_circuit_key_lock(
510
+ self: Any,
511
+ key: _HTTPBridgeSessionKey,
512
+ ) -> asyncio.Lock:
513
+ """Serialize durable circuit writes and settles for one key.
514
+
515
+ A strike write and a settle for the same key must not interleave
516
+ across their durable awaits: a settle landing under an in-flight
517
+ write resurrected the row, and a stale write landing after a settle
518
+ merged a finished episode's strike into whatever episode owned the
519
+ row next. The lock is per key, so unrelated keys never wait on each
520
+ other's durable I/O. The acquire loop re-checks registration because
521
+ pruning may drop an idle lock between the fetch and the acquire.
522
+ """
523
+ while True:
524
+ async with self._http_bridge_retry_circuit_lock:
525
+ key_lock = self._http_bridge_retry_circuit_key_locks.setdefault(key, asyncio.Lock())
526
+ await key_lock.acquire()
527
+ # Ownership transfers to the caller only on return. The
528
+ # registration re-check is itself a cancellable await; a
529
+ # cancellation landing there would otherwise leave the key lock
530
+ # held forever, wedging every later persist and settle for the
531
+ # key.
532
+ try:
533
+ async with self._http_bridge_retry_circuit_lock:
534
+ if self._http_bridge_retry_circuit_key_locks.get(key) is key_lock:
535
+ return key_lock
536
+ except BaseException:
537
+ key_lock.release()
538
+ raise
539
+ key_lock.release()
540
+
541
+ async def _load_http_bridge_retry_circuit(self: Any, session: _HTTPBridgeSession) -> bool:
542
+ key = session.key
543
+ if key.strength != "hard":
544
+ return True
545
+
546
+ now_monotonic = clock_for(self).monotonic()
547
+ async with self._http_bridge_retry_circuit_lock:
548
+ self._prune_http_bridge_retry_circuit_state(now_monotonic)
549
+ local_state = self._http_bridge_retry_circuits.get(key)
550
+ if local_state is not None:
551
+ local_state.last_touched_monotonic = now_monotonic
552
+ try:
553
+ persisted = await self._durable_bridge.lookup_retry_circuit(
554
+ session_key_kind=key.affinity_kind,
555
+ session_key_value=key.affinity_key,
556
+ api_key_id=key.api_key_id,
557
+ )
558
+ except Exception:
559
+ if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
560
+ http_bridge_retry_circuit_total.labels(outcome="lookup_failed").inc()
561
+ logger.warning(
562
+ "Failed to load persisted HTTP bridge retry circuit bridge_kind=%s bridge_key=%s",
563
+ key.affinity_kind,
564
+ _hash_identifier(key.affinity_key),
565
+ exc_info=True,
566
+ )
567
+ return False
568
+
569
+ if persisted is None:
570
+ # A durable miss clears state loaded from another replica, but it
571
+ # must not discard a failure recorded locally after the last
572
+ # durable read. That local circuit is the only protection against
573
+ # immediately replaying the same failing upstream request.
574
+ async with self._http_bridge_retry_circuit_lock:
575
+ local_state = self._http_bridge_retry_circuits.get(key)
576
+ key_watermark = self._http_bridge_retry_circuit_reconcile_watermarks.get(key, 0.0)
577
+ if now_monotonic < key_watermark or (
578
+ local_state is not None and now_monotonic < local_state.last_durable_load_monotonic
579
+ ):
580
+ # This lookup began before a same-key write completed:
581
+ # its miss predates the row that write created, and
582
+ # popping the episode here would let the admission
583
+ # decision that follows bypass the active cooldown the
584
+ # completed write just opened.
585
+ return True
586
+ locally_updated = bool(
587
+ local_state is not None
588
+ and local_state.last_failure_monotonic > local_state.last_durable_load_monotonic
589
+ )
590
+ # A confirmed miss past the stale-lookup guards is durable
591
+ # knowledge worth caching for the planning window; stamped
592
+ # with the load's start so it expires conservatively, and
593
+ # re-inserted so the map stays in insertion order for the
594
+ # front-only expiry sweep.
595
+ self._http_bridge_retry_circuit_planning_misses.pop(key, None)
596
+ self._http_bridge_retry_circuit_planning_misses[key] = now_monotonic
597
+ while (
598
+ len(self._http_bridge_retry_circuit_planning_misses) > _HTTP_BRIDGE_RETRY_CIRCUIT_PLANNING_MISS_CAP
599
+ ):
600
+ # The pre-lookup prune ran before this load's durable
601
+ # await, so a burst of concurrent first-touch misses can
602
+ # land past it; the cap is enforced where entries are
603
+ # added.
604
+ del self._http_bridge_retry_circuit_planning_misses[
605
+ next(iter(self._http_bridge_retry_circuit_planning_misses))
606
+ ]
607
+ if key in self._http_bridge_retry_circuit_persisted_keys and not locally_updated:
608
+ self._http_bridge_retry_circuits.pop(key, None)
609
+ self._http_bridge_retry_circuit_loaded_keys.discard(key)
610
+ self._http_bridge_retry_circuit_persisted_keys.discard(key)
611
+ if _http_bridge_session_key_poison_quarantined(self, key):
612
+ # The durable episode this quarantine fenced is gone
613
+ # — settled or purged by another replica — and a
614
+ # remotely recovered key must not stay suppressed
615
+ # for the stale deadline.
616
+ _revoke_http_bridge_poison_quarantine(
617
+ self,
618
+ key,
619
+ generation=_http_bridge_quarantine_clear_fence(self, key),
620
+ )
621
+ return True
622
+
623
+ now_epoch = clock_for(self).time()
624
+ row_age = now_epoch - persisted.updated_at_epoch
625
+ if (
626
+ row_age > DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
627
+ # An abandonment tombstone guards continuity that outlives the
628
+ # circuit TTL: purging it here would let the unanchored-delta
629
+ # gate dispatch a context-free request. Its lifetime belongs to
630
+ # the scheduled purge's bridge-retention cutoff.
631
+ and persisted.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
632
+ and (
633
+ getattr(persisted, "admission_generation", 0) == 0
634
+ or row_age > 2 * DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
635
+ )
636
+ ):
637
+ async with self._http_bridge_retry_circuit_lock:
638
+ stale_local_state = self._http_bridge_retry_circuits.get(key)
639
+ try:
640
+ stale_row_purged = await self._durable_bridge.purge_retry_circuit(
641
+ session_key_kind=key.affinity_kind,
642
+ session_key_value=key.affinity_key,
643
+ api_key_id=key.api_key_id,
644
+ expected_updated_at_epoch=persisted.updated_at_epoch,
645
+ expected_admission_generation=getattr(persisted, "admission_generation", 0),
646
+ # Detail-only transitions (the transitional tombstone
647
+ # supersede) and lagging-clock merges move neither the
648
+ # epoch nor the admission generation; fencing on the
649
+ # observed count and detail keeps this purge from
650
+ # deleting a row another replica just rewrote.
651
+ expected_consecutive_failures=persisted.consecutive_failures,
652
+ fence_last_detail=True,
653
+ expected_last_detail=persisted.last_detail,
654
+ )
655
+ except Exception:
656
+ logger.warning(
657
+ "Failed to remove stale HTTP bridge retry circuit bridge_kind=%s bridge_key=%s",
658
+ key.affinity_kind,
659
+ _hash_identifier(key.affinity_key),
660
+ exc_info=True,
661
+ )
662
+ # Keep a newer process-local circuit when persistence is
663
+ # unavailable. The next failure can still open the local
664
+ # circuit even though the expired durable row remains.
665
+ return False
666
+ if stale_row_purged is False:
667
+ # The fenced purge matched nothing: another replica moved
668
+ # the row after this worker's lookup — a fresh strike or
669
+ # opening — and proceeding as though it was deleted would
670
+ # pop the local circuit, revoke the quarantine, and let the
671
+ # very next admission dispatch with the poisoned anchor.
672
+ # Reconcile against the surviving row instead.
673
+ try:
674
+ refreshed = await self._durable_bridge.lookup_retry_circuit(
675
+ session_key_kind=key.affinity_kind,
676
+ session_key_value=key.affinity_key,
677
+ api_key_id=key.api_key_id,
678
+ )
679
+ except Exception:
680
+ logger.warning(
681
+ "Failed to reload HTTP bridge retry circuit after purge miss bridge_kind=%s bridge_key=%s",
682
+ key.affinity_kind,
683
+ _hash_identifier(key.affinity_key),
684
+ exc_info=True,
685
+ )
686
+ return False
687
+ if refreshed is not None and (
688
+ # A detail-only tombstone rewrite keeps the old epoch, so
689
+ # an age check would reject the very fence the missed
690
+ # purge just proved exists — and planning would inject
691
+ # the dead anchor the tombstone guards. Tombstones are
692
+ # adopted regardless of circuit age; their lifetime
693
+ # belongs to the bridge-retention purge.
694
+ refreshed.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
695
+ or now_epoch - refreshed.updated_at_epoch
696
+ <= (
697
+ DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
698
+ * (2 if getattr(refreshed, "admission_generation", 0) > 0 else 1)
699
+ )
700
+ ):
701
+ persisted = refreshed
702
+ else:
703
+ # The surviving row vanished again or is itself stale:
704
+ # nothing current to adopt, and nothing was deleted this
705
+ # worker can vouch for.
706
+ return True
707
+ if stale_row_purged is not False:
708
+ async with self._http_bridge_retry_circuit_lock:
709
+ current_local_state = self._http_bridge_retry_circuits.get(key)
710
+ local_state_is_newer = bool(
711
+ current_local_state is not None
712
+ and current_local_state.last_failure_monotonic > current_local_state.last_durable_load_monotonic
713
+ )
714
+ if current_local_state is None or (
715
+ current_local_state is stale_local_state and not local_state_is_newer
716
+ ):
717
+ self._http_bridge_retry_circuits.pop(key, None)
718
+ self._http_bridge_retry_circuit_loaded_keys.discard(key)
719
+ self._http_bridge_retry_circuit_persisted_keys.discard(key)
720
+ if _http_bridge_session_key_poison_quarantined(self, key):
721
+ # The expired row just purged is the episode this
722
+ # quarantine fenced; it ended long ago.
723
+ _revoke_http_bridge_poison_quarantine(
724
+ self,
725
+ key,
726
+ generation=_http_bridge_quarantine_clear_fence(self, key),
727
+ )
728
+ return True
729
+
730
+ cooldown_remaining = max(0.0, persisted.cooldown_until_epoch - now_epoch)
731
+ persisted_cooldown_until = now_monotonic + cooldown_remaining
732
+ arm_poison_quarantine = False
733
+ poison_cooldown_remaining = 0.0
734
+ async with self._http_bridge_retry_circuit_lock:
735
+ self._http_bridge_retry_circuit_persisted_keys.add(key)
736
+ self._http_bridge_retry_circuit_planning_misses.pop(key, None)
737
+ state = self._http_bridge_retry_circuits.get(key)
738
+ if state is None:
739
+ state = _HTTPBridgeRetryCircuitState(last_touched_monotonic=now_monotonic)
740
+ self._http_bridge_retry_circuits[key] = state
741
+ key_watermark = self._http_bridge_retry_circuit_reconcile_watermarks.get(key, 0.0)
742
+ if now_monotonic < state.last_durable_load_monotonic or now_monotonic < key_watermark:
743
+ # This load's lookup began before a same-key strike or
744
+ # settlement completed its durable write, so its row snapshot
745
+ # may predate that write: adopting it would erase a
746
+ # just-opened cooldown or resurrect a just-settled episode.
747
+ # The completed operation already reconciled the state from
748
+ # its own returned row (or popped it at settlement, which
749
+ # the per-key watermark records past the pop); an older
750
+ # snapshot has nothing newer to add.
751
+ self._http_bridge_retry_circuit_loaded_keys.add(key)
752
+ return True
753
+ local_failure_is_newer = state.last_failure_monotonic > state.last_durable_load_monotonic
754
+ # A foreign write is identified by ANY observed column moving,
755
+ # not the timestamp alone: a lagging-clock strike merges through
756
+ # greatest() without moving the epoch while incrementing the
757
+ # count and rewriting the detail, and an anchor supersession
758
+ # rewrites the detail in place by design. Epoch-only detection
759
+ # let a marker survive a new failure against the advanced
760
+ # anchor and left an active quarantine unextended.
761
+ episode_replaced = (
762
+ persisted.updated_at_epoch != state.persisted_updated_at_epoch
763
+ or persisted.consecutive_failures != state.consecutive_failures
764
+ or persisted.last_detail != state.last_detail
765
+ )
766
+ if not local_failure_is_newer:
767
+ # No local strike is waiting on its durable write, so the row
768
+ # is strictly newer knowledge and is adopted wholesale.
769
+ # Adoption compares no wall clocks: a reset stamped by a
770
+ # lagging replica clock still replaces the local episode
771
+ # instead of leaving a settled key suppressed for the rest
772
+ # of its stale cooldown with a base that no longer exists.
773
+ if episode_replaced:
774
+ # A write this worker did not produce: either the same
775
+ # episode struck elsewhere (then a set marker means the
776
+ # anchor is already cleared and the empty-continuity
777
+ # consult refuses regardless), or the episode was
778
+ # replaced — possibly at an equal or higher count — and
779
+ # the replacement's newly poisoned anchor must be
780
+ # allowed its one abandonment. Resetting on lineage
781
+ # rather than count ordering covers the equal-count
782
+ # replacement the count check below cannot see.
783
+ state.poison_anchor_cleared = False
784
+ # The local debt dies with every foreign write: counts
785
+ # and epochs cannot distinguish same-lineage advancement
786
+ # from a reset-and-overtaken replacement, and guessing
787
+ # wrong clears a replacement's fresh continuity. The
788
+ # durable row itself carries the cross-replica debt —
789
+ # an at-threshold poison detail is sticky against
790
+ # non-poison strikes in the merge — so the arm below
791
+ # re-derives the debt from the adopted row.
792
+ state.owed_poison_detail = None
793
+ if persisted.consecutive_failures < state.consecutive_failures:
794
+ # The row does not carry this worker's failures: the
795
+ # lineage was reset, purged, or replaced, and the marker
796
+ # belonged to the ended episode.
797
+ state.poison_anchor_cleared = False
798
+ state.owed_poison_detail = None
799
+ state.consecutive_failures = max(0, persisted.consecutive_failures)
800
+ state.cooldown_until = persisted_cooldown_until
801
+ state.last_detail = persisted.last_detail
802
+ if state.consecutive_failures == 0:
803
+ # A zero-failure row is a durable reset: the episode the
804
+ # marker belonged to is over, and the next poison episode
805
+ # on this state object must be allowed its one
806
+ # abandonment.
807
+ state.poison_anchor_cleared = False
808
+ state.owed_poison_detail = None
809
+ else:
810
+ # A local strike sits between its record and its durable
811
+ # write; keep it dominant here and let that write's own
812
+ # merge reconcile clock-free against the returned row.
813
+ state.consecutive_failures = max(state.consecutive_failures, max(0, persisted.consecutive_failures))
814
+ state.cooldown_until = max(state.cooldown_until, persisted_cooldown_until)
815
+ state.last_detail = state.last_detail or persisted.last_detail
816
+ if state.cooldown_until > now_monotonic:
817
+ # A cooling key is not probing. Without this, a merged
818
+ # persisted cooldown leaves a leftover half-open lease
819
+ # armed, and once the cooldown expires the admission gate
820
+ # reads that stale lease as an in-flight probe and keeps
821
+ # suppressing the key for the rest of the lease.
822
+ state.half_open_until = 0.0
823
+ elif episode_replaced and not local_failure_is_newer:
824
+ # The adopted row belongs to a replacement episode whose
825
+ # cooldown has already elapsed. A lease left over from the
826
+ # ended episode would read as this worker's in-flight probe
827
+ # for the new one and suppress the key for the rest of a
828
+ # window it never opened.
829
+ state.half_open_until = 0.0
830
+ state.persisted_updated_at_epoch = persisted.updated_at_epoch
831
+ state.persisted_admission_generation = getattr(persisted, "admission_generation", 0)
832
+ state.last_touched_monotonic = now_monotonic
833
+ state.last_durable_load_monotonic = now_monotonic
834
+ self._http_bridge_retry_circuit_loaded_keys.add(key)
835
+ # A poison opening recorded by another replica reaches this
836
+ # worker only through this load, and its quarantine is
837
+ # process-local: without re-arming here, this worker's probe is
838
+ # planned with the anchor the row's failures were recorded
839
+ # against.
840
+ effective_poison_threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
841
+ arm_poison_quarantine = (
842
+ not local_failure_is_newer
843
+ # An episode whose one-clear marker is set already abandoned
844
+ # its anchor (a completed verified replay deliberately keeps
845
+ # the row and clears the quarantine); re-arming off the
846
+ # unchanged row would re-fence the recovered key for another
847
+ # full deadline.
848
+ and not state.poison_anchor_cleared
849
+ and state.consecutive_failures >= effective_poison_threshold
850
+ and _http_bridge_anchor_poison_detail(state.last_detail) is not None
851
+ )
852
+ if arm_poison_quarantine:
853
+ state.owed_poison_detail = state.last_detail
854
+ # The adopted row can also disprove the episode a local poison
855
+ # quarantine fenced: a zero-failure reset, a deliberate anchor
856
+ # supersession, or a replaced lineage that has not proven poison
857
+ # ends it, and a remotely recovered key must not stay suppressed
858
+ # for the stale deadline. A replaced lineage at or past the
859
+ # threshold with a non-poison last strike is left fenced — its
860
+ # earlier strikes may still be the poison evidence.
861
+ revoke_stale_poison_quarantine = (
862
+ not local_failure_is_newer
863
+ # A transitional abandonment tombstone is not disproof: the
864
+ # fresh anchor's registration has not been confirmed, the
865
+ # old poisoned anchor may still be the stored one, and the
866
+ # surviving quarantine is what keeps full-resend planning
867
+ # from injecting it.
868
+ and persisted.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
869
+ and (
870
+ persisted.consecutive_failures == 0
871
+ or persisted.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL
872
+ or (
873
+ episode_replaced
874
+ and _http_bridge_anchor_poison_detail(persisted.last_detail) is None
875
+ and persisted.consecutive_failures < effective_poison_threshold
876
+ )
877
+ )
878
+ )
879
+ poison_cooldown_remaining = max(0.0, state.cooldown_until - now_monotonic)
880
+ if arm_poison_quarantine and (episode_replaced or not _http_bridge_session_key_poison_quarantined(self, key)):
881
+ # Skipped only for a truly unchanged episode, so ordinary loads
882
+ # do not bump the quarantine generation that recovery fences
883
+ # observe. An adopted foreign write re-arms even while a
884
+ # quarantine is active: a later durable strike can extend the
885
+ # cooldown past the old deadline, and leaving that deadline in
886
+ # place would let the quarantine lapse mid-cooldown — with the
887
+ # planning cache still fresh — and hand the next half-open probe
888
+ # the poisoned anchor. The re-arm extends the deadline against
889
+ # the adopted cooldown and refreshes the poison provenance to
890
+ # the lineage that now owns the row.
891
+ _quarantine_http_bridge_session(
892
+ self,
893
+ session,
894
+ reason=_HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON,
895
+ minimum_seconds=_http_bridge_poison_quarantine_minimum_seconds(poison_cooldown_remaining),
896
+ )
897
+ elif revoke_stale_poison_quarantine and _http_bridge_session_key_poison_quarantined(self, key):
898
+ _revoke_http_bridge_poison_quarantine(
899
+ self,
900
+ key,
901
+ generation=_http_bridge_quarantine_clear_fence(self, key),
902
+ )
903
+ return True
904
+
905
+ async def _persist_http_bridge_retry_circuit(
906
+ self: Any,
907
+ session: _HTTPBridgeSession,
908
+ state: _HTTPBridgeRetryCircuitState,
909
+ ) -> None:
910
+ now_monotonic = clock_for(self).monotonic()
911
+ now_wall = clock_for(self).time()
912
+ threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
913
+ key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(session.key)
914
+ try:
915
+ await self._persist_http_bridge_retry_circuit_serialized(
916
+ session,
917
+ state,
918
+ now_monotonic=now_monotonic,
919
+ now_wall=now_wall,
920
+ threshold=threshold,
921
+ )
922
+ finally:
923
+ key_lock.release()
924
+
925
+ async def _persist_http_bridge_retry_circuit_serialized(
926
+ self: Any,
927
+ session: _HTTPBridgeSession,
928
+ state: _HTTPBridgeRetryCircuitState,
929
+ *,
930
+ now_monotonic: float,
931
+ now_wall: float,
932
+ threshold: int,
933
+ ) -> None:
934
+ async with self._http_bridge_retry_circuit_lock:
935
+ # Re-checked under the key lock: a settle or a replacement
936
+ # episode that took the key while this writer waited means this
937
+ # strike belongs to a finished episode, and writing it would
938
+ # merge that stale failure into whatever episode owns the row
939
+ # now.
940
+ if self._http_bridge_retry_circuits.get(session.key) is not state:
941
+ return
942
+ consecutive_failures = state.consecutive_failures
943
+ cooldown_until = state.cooldown_until
944
+ last_detail = state.last_detail
945
+ persisted_updated_at_epoch = state.persisted_updated_at_epoch
946
+ poison_sticky_threshold = threshold
947
+ base_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS)
948
+ if last_detail == "clean_close":
949
+ base_backoff = min(
950
+ base_backoff,
951
+ max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS),
952
+ )
953
+ try:
954
+ persisted = await self._durable_bridge.persist_retry_circuit(
955
+ session_key_kind=session.key.affinity_kind,
956
+ session_key_value=session.key.affinity_key,
957
+ api_key_id=session.key.api_key_id,
958
+ consecutive_failures=consecutive_failures,
959
+ cooldown_until_epoch=now_wall + max(0.0, cooldown_until - now_monotonic),
960
+ last_detail=last_detail,
961
+ updated_at_epoch=now_wall,
962
+ base_updated_at_epoch=persisted_updated_at_epoch,
963
+ failure_threshold=threshold,
964
+ poison_sticky_threshold=poison_sticky_threshold,
965
+ conflict_cooldown_until_epoch=now_wall + base_backoff,
966
+ base_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS),
967
+ max_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS),
968
+ clean_close_max_backoff_seconds=max(
969
+ 0.001,
970
+ _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS,
971
+ ),
972
+ )
973
+ if persisted is not None and persisted_updated_at_epoch <= 0 and persisted.updated_at_epoch != now_wall:
974
+ # Blind base: the pre-strike load failed, so this write
975
+ # carried no valid base and dropped against a row that
976
+ # already existed. A stale-episode drop loses at most its
977
+ # own already-counted strike; a blind drop loses the only
978
+ # record of a genuinely new failure, both durably and — via
979
+ # the wholesale adoption below — locally. Re-strike exactly
980
+ # this one failure on top of the observed lineage: strikes
981
+ # are serialized per key, so this write carries one new
982
+ # strike, and earlier local strikes either already landed
983
+ # unobserved or were lost to their own failed writes. A
984
+ # second drop means the row moved again and the ordinary
985
+ # undercount trade applies.
986
+ persisted = await self._durable_bridge.persist_retry_circuit(
987
+ session_key_kind=session.key.affinity_kind,
988
+ session_key_value=session.key.affinity_key,
989
+ api_key_id=session.key.api_key_id,
990
+ consecutive_failures=max(0, persisted.consecutive_failures) + 1,
991
+ cooldown_until_epoch=now_wall + max(0.0, cooldown_until - now_monotonic),
992
+ last_detail=last_detail,
993
+ updated_at_epoch=now_wall,
994
+ base_updated_at_epoch=persisted.updated_at_epoch,
995
+ failure_threshold=threshold,
996
+ poison_sticky_threshold=poison_sticky_threshold,
997
+ conflict_cooldown_until_epoch=now_wall + base_backoff,
998
+ base_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS),
999
+ max_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS),
1000
+ clean_close_max_backoff_seconds=max(
1001
+ 0.001,
1002
+ _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS,
1003
+ ),
1004
+ )
1005
+ if persisted is not None:
1006
+ persisted_cooldown_until = now_monotonic + max(0.0, persisted.cooldown_until_epoch - now_wall)
1007
+ async with self._http_bridge_retry_circuit_lock:
1008
+ current = self._http_bridge_retry_circuits.get(session.key)
1009
+ if current is state:
1010
+ # The upsert returns the post-write row: when this
1011
+ # write landed the row reflects it, and when its base
1012
+ # mismatched the row is the lineage that owns the key
1013
+ # now and this writer must reconcile from it. Strikes
1014
+ # for one key are serialized across their durable
1015
+ # awaits, so no local failure can be recorded while
1016
+ # this persist is in flight, and the returned row is
1017
+ # adopted wholesale. Adoption compares no wall
1018
+ # clocks: a reset stamped by a lagging replica clock
1019
+ # still replaces the local episode, and taking the
1020
+ # row's epoch exactly — never the max — is what lets
1021
+ # the next strike carry a base that actually exists.
1022
+ if persisted.consecutive_failures < state.consecutive_failures:
1023
+ # The row does not carry this worker's failures:
1024
+ # the lineage was reset, purged, or replaced.
1025
+ # The marker belonged to the ended episode; a
1026
+ # spurious trip only allows one extra fenced
1027
+ # abandonment.
1028
+ state.poison_anchor_cleared = False
1029
+ state.owed_poison_detail = None
1030
+ if (
1031
+ persisted.consecutive_failures != consecutive_failures
1032
+ or persisted.last_detail != last_detail
1033
+ ):
1034
+ # The returned row carries a contribution this
1035
+ # write did not submit: a same-base foreign
1036
+ # strike merged under a lagging clock keeps the
1037
+ # epoch while moving the count or detail, and
1038
+ # its evidence targets whatever anchor is
1039
+ # current — the marker cannot survive it.
1040
+ state.poison_anchor_cleared = False
1041
+ state.owed_poison_detail = (
1042
+ persisted.last_detail
1043
+ if _http_bridge_anchor_poison_detail(persisted.last_detail) is not None
1044
+ else None
1045
+ )
1046
+ if (
1047
+ persisted.updated_at_epoch != now_wall
1048
+ and persisted.updated_at_epoch != persisted_updated_at_epoch
1049
+ ):
1050
+ # This write lost to a foreign replacement row —
1051
+ # a landed write stamps max(base, now_wall), and
1052
+ # an unchanged row keeps the base. Adopting the
1053
+ # foreign epoch below makes later loads see the
1054
+ # episode as unchanged, so this is the only point
1055
+ # that can see the replacement; an equal-or-higher
1056
+ # count hides it from the check above, and a
1057
+ # marker carried across it would permanently
1058
+ # refuse the replacement episode its one
1059
+ # abandonment.
1060
+ state.poison_anchor_cleared = False
1061
+ state.owed_poison_detail = None
1062
+ state.consecutive_failures = max(0, persisted.consecutive_failures)
1063
+ state.cooldown_until = persisted_cooldown_until
1064
+ state.last_detail = persisted.last_detail
1065
+ if state.consecutive_failures == 0:
1066
+ # A zero-failure row is a durable reset ending
1067
+ # the marker's episode.
1068
+ state.poison_anchor_cleared = False
1069
+ state.owed_poison_detail = None
1070
+ if state.cooldown_until > now_monotonic:
1071
+ # A cooling key is not probing: drop any leftover
1072
+ # half-open lease so the admission gate cannot
1073
+ # read it as an in-flight probe after this
1074
+ # cooldown expires.
1075
+ state.half_open_until = 0.0
1076
+ state.persisted_updated_at_epoch = persisted.updated_at_epoch
1077
+ state.persisted_admission_generation = getattr(persisted, "admission_generation", 0)
1078
+ # Post-write time, not this persist's entry time: a
1079
+ # load that started while the durable write was in
1080
+ # flight can carry a start stamp later than the entry
1081
+ # time while its row snapshot predates the write, and
1082
+ # the stale-load guard compares against this value.
1083
+ state.last_durable_load_monotonic = max(
1084
+ state.last_durable_load_monotonic,
1085
+ clock_for(self).monotonic(),
1086
+ )
1087
+ async with self._http_bridge_retry_circuit_lock:
1088
+ if self._http_bridge_retry_circuits.get(session.key) is state:
1089
+ self._http_bridge_retry_circuit_persisted_keys.add(session.key)
1090
+ except Exception:
1091
+ if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
1092
+ http_bridge_retry_circuit_total.labels(outcome="persist_failed").inc()
1093
+ logger.warning(
1094
+ "Failed to persist HTTP bridge retry circuit bridge_kind=%s bridge_key=%s",
1095
+ session.key.affinity_kind,
1096
+ _hash_identifier(session.key.affinity_key),
1097
+ exc_info=True,
1098
+ )
1099
+
1100
+ async def _http_bridge_precreated_retry_allowed(
1101
+ self: Any,
1102
+ session: _HTTPBridgeSession,
1103
+ *,
1104
+ allow_fresh_hard_account_switch: bool = False,
1105
+ allow_proof_gated_continuity_replay: bool = False,
1106
+ claimed_lease_out: list[float] | None = None,
1107
+ ) -> bool:
1108
+ """Avoid replaying a repeatedly failing hard-affinity request in a tight loop."""
1109
+ if session.key.strength != "hard":
1110
+ return True
1111
+
1112
+ await self._load_http_bridge_retry_circuit(session)
1113
+ now = clock_for(self).monotonic()
1114
+ async with self._http_bridge_retry_circuit_lock:
1115
+ state = self._http_bridge_retry_circuits.get(session.key)
1116
+ if state is None or state.cooldown_until <= now:
1117
+ if (
1118
+ state is not None
1119
+ and state.consecutive_failures >= _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD
1120
+ and state.half_open_until > now
1121
+ and not allow_fresh_hard_account_switch
1122
+ and not allow_proof_gated_continuity_replay
1123
+ ):
1124
+ if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
1125
+ http_bridge_retry_circuit_total.labels(outcome="suppressed").inc()
1126
+ return False
1127
+ if state is not None and state.cooldown_until > 0:
1128
+ state.cooldown_until = 0.0
1129
+ state.half_open_until = now + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS
1130
+ if claimed_lease_out is not None:
1131
+ # The exact lease this admission claimed, handed out
1132
+ # under the same lock that installed it, so a
1133
+ # fail-closed caller can return precisely its own
1134
+ # probe under any interleaving.
1135
+ claimed_lease_out.append(state.half_open_until)
1136
+ logger.info(
1137
+ "http_bridge_retry_circuit event=half_open bridge_kind=%s bridge_key=%s failures=%s",
1138
+ session.key.affinity_kind,
1139
+ _hash_identifier(session.key.affinity_key),
1140
+ state.consecutive_failures,
1141
+ )
1142
+ return True
1143
+
1144
+ retry_after = max(0.0, state.cooldown_until - now)
1145
+ if allow_fresh_hard_account_switch:
1146
+ logger.info(
1147
+ "http_bridge_retry_circuit event=bypass_fresh_account_switch bridge_kind=%s "
1148
+ "bridge_key=%s failures=%s retry_after_seconds=%.1f",
1149
+ session.key.affinity_kind,
1150
+ _hash_identifier(session.key.affinity_key),
1151
+ state.consecutive_failures,
1152
+ retry_after,
1153
+ )
1154
+ return True
1155
+ if allow_proof_gated_continuity_replay:
1156
+ logger.info(
1157
+ "http_bridge_retry_circuit event=bypass_proof_gated_continuity_replay bridge_kind=%s "
1158
+ "bridge_key=%s failures=%s retry_after_seconds=%.1f",
1159
+ session.key.affinity_kind,
1160
+ _hash_identifier(session.key.affinity_key),
1161
+ state.consecutive_failures,
1162
+ retry_after,
1163
+ )
1164
+ return True
1165
+ if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
1166
+ http_bridge_retry_circuit_total.labels(outcome="suppressed").inc()
1167
+ logger.info(
1168
+ "http_bridge_retry_circuit event=suppressed bridge_kind=%s bridge_key=%s "
1169
+ "failures=%s retry_after_seconds=%.1f detail=%s",
1170
+ session.key.affinity_kind,
1171
+ _hash_identifier(session.key.affinity_key),
1172
+ state.consecutive_failures,
1173
+ retry_after,
1174
+ state.last_detail,
1175
+ )
1176
+ return False
1177
+
1178
+ async def _http_bridge_poison_anchor_clear_owed(
1179
+ self: Any,
1180
+ session: _HTTPBridgeSession,
1181
+ *,
1182
+ consecutive_failures: int | None,
1183
+ ) -> "tuple[_HTTPBridgeRetryCircuitState | None, object]":
1184
+ """Return the owed episode and the durable anchor captured with it.
1185
+
1186
+ Returns ``(episode, captured_anchor)`` — the validated registered
1187
+ episode (or ``None`` when nothing is owed) and the durable session's
1188
+ continuity anchor sampled before the validation reads. The caller
1189
+ passes exactly this episode to the post-abandonment marker and fences
1190
+ the continuity clear on the captured anchor: with the completion path
1191
+ settling the circuit before it registers a fresh anchor, any
1192
+ completion this consult did not veto changes the anchor after this
1193
+ capture, and the fenced clear then matches nothing.
1194
+
1195
+ The poison threshold is the circuit threshold, which is what makes the
1196
+ clear reachable at all, but it also means every later strike in the
1197
+ same episode meets the threshold too. Only the first successful
1198
+ abandonment settles the anchor; a failed one leaves the marker unset so
1199
+ the next strike retries it, which is the contract the retirement
1200
+ funnels already advertise in their failure telemetry.
1201
+
1202
+ The decision reads the currently registered episode, not the caller's
1203
+ captured count. Between the strike and this consult a multiplexed
1204
+ sibling can complete, settle the circuit, and persist a fresh valid
1205
+ anchor; the caller's detached state still reports the old count, and
1206
+ clearing on it would delete the anchor the sibling just stored. An
1207
+ absent or below-threshold registered episode therefore owes nothing.
1208
+ """
1209
+ if consecutive_failures is None:
1210
+ return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1211
+ effective_threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
1212
+ async with self._http_bridge_retry_circuit_lock:
1213
+ state = self._http_bridge_retry_circuits.get(session.key)
1214
+ live_episode_owes = (
1215
+ state is not None
1216
+ and state.consecutive_failures >= effective_threshold
1217
+ and not state.poison_anchor_cleared
1218
+ )
1219
+ if not live_episode_owes:
1220
+ return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1221
+ # An abandonment without a captured fence is refused outright: an
1222
+ # unfenced clear on this degraded path could delete a fresh anchor a
1223
+ # sibling registered after the recheck below. A later strike retries
1224
+ # the consult with a working capture.
1225
+ anchor_reader = getattr(self._durable_bridge, "session_latest_continuity", None)
1226
+ durable_session_id = getattr(session, "durable_session_id", None)
1227
+ if anchor_reader is None or durable_session_id is None:
1228
+ return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1229
+ try:
1230
+ expected_anchor = await anchor_reader(session_id=durable_session_id)
1231
+ except Exception:
1232
+ logger.warning(
1233
+ "Failed to capture durable anchor before poison clear bridge_kind=%s bridge_key=%s",
1234
+ session.key.affinity_kind,
1235
+ _hash_identifier(session.key.affinity_key),
1236
+ exc_info=True,
1237
+ )
1238
+ return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1239
+ if expected_anchor is None:
1240
+ # The durable session row itself is gone; there is no continuity
1241
+ # to clear and no columns for a fence to protect.
1242
+ return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1243
+ if expected_anchor == (None, None):
1244
+ # Both continuity columns are already empty: there is no anchor
1245
+ # left to abandon, so a clear here removes no failure cause.
1246
+ # Authorizing it anyway would let the settle-on-abandon path
1247
+ # reset a circuit that is cooling on genuinely unanchored
1248
+ # upstream failures, and that cooldown is the only protection
1249
+ # those failures have.
1250
+ return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1251
+ # The one-clear marker lives in process memory, so a restart or
1252
+ # another replica cannot see it. The durable circuit row is the
1253
+ # episode's replica-visible record instead: a completed response
1254
+ # settles the circuit and deletes the row, and the fresh anchor that
1255
+ # completion persisted must not be deleted by a stale local episode
1256
+ # that survived the reset. The settle updates the row to zero rather
1257
+ # than deleting it, so a reset row proves the episode ended exactly
1258
+ # as an absent one does; only a row still at the effective threshold
1259
+ # is a live episode. A lookup failure proves nothing either, and the
1260
+ # next strike retries.
1261
+ try:
1262
+ persisted = await self._durable_bridge.lookup_retry_circuit(
1263
+ session_key_kind=session.key.affinity_kind,
1264
+ session_key_value=session.key.affinity_key,
1265
+ api_key_id=session.key.api_key_id,
1266
+ )
1267
+ except Exception:
1268
+ logger.warning(
1269
+ "Failed to confirm durable retry-circuit episode before anchor clear bridge_kind=%s bridge_key=%s",
1270
+ session.key.affinity_kind,
1271
+ _hash_identifier(session.key.affinity_key),
1272
+ exc_info=True,
1273
+ )
1274
+ return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1275
+ if persisted is None or persisted.consecutive_failures < effective_threshold:
1276
+ return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1277
+ # The row must be a poison episode of its own: another replica can
1278
+ # reset the old episode, register a fresh anchor, and open a new
1279
+ # at-threshold episode on clean_close, and a count-only check would
1280
+ # let the stale local episode clear that fresh anchor. An episode
1281
+ # that still OWES a poison clear stays authorized even after a later
1282
+ # clean probe failure overwrote the durable detail — fenced on the
1283
+ # exact reconciled lineage, so a replaced row never inherits the
1284
+ # stale owed state.
1285
+ row_detail_is_poison = _http_bridge_anchor_poison_detail(persisted.last_detail) is not None
1286
+ # Re-check the live episode after the durable await: a multiplexed
1287
+ # sibling can complete, settle the registry, and reset the row while
1288
+ # the lookup was in flight, and the stale snapshot must not authorize
1289
+ # a clear against the fresh anchor that completion persisted.
1290
+ async with self._http_bridge_retry_circuit_lock:
1291
+ state = self._http_bridge_retry_circuits.get(session.key)
1292
+ if (
1293
+ state is not None
1294
+ and state.consecutive_failures >= effective_threshold
1295
+ and not state.poison_anchor_cleared
1296
+ and (
1297
+ row_detail_is_poison
1298
+ or (
1299
+ state.owed_poison_detail is not None
1300
+ and persisted.updated_at_epoch == state.persisted_updated_at_epoch
1301
+ )
1302
+ )
1303
+ ):
1304
+ if row_detail_is_poison:
1305
+ # The consulted poison row IS the authorization: the
1306
+ # abandonment fences from this episode snapshot, and a
1307
+ # stale local identity — an unpersisted write (epoch
1308
+ # zero) or a cross-replica strike that moved the row
1309
+ # after this worker's own — would make both settlement
1310
+ # attempts reject the newer fence and leave the removed
1311
+ # anchor's cooldown and quarantine standing. Adopt the
1312
+ # row's epoch, admission generation, and higher count
1313
+ # unconditionally.
1314
+ state.persisted_updated_at_epoch = persisted.updated_at_epoch
1315
+ state.persisted_admission_generation = getattr(persisted, "admission_generation", 0)
1316
+ if persisted.consecutive_failures > state.consecutive_failures:
1317
+ state.consecutive_failures = persisted.consecutive_failures
1318
+ return state, expected_anchor
1319
+ return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
1320
+
1321
+ async def _http_bridge_effective_anchor_poison_detail(
1322
+ self: Any,
1323
+ session: _HTTPBridgeSession,
1324
+ detail: str | None,
1325
+ ) -> str | None:
1326
+ """Map a strike to its poison class, falling back to the owed one.
1327
+
1328
+ A failed abandonment leaves the episode's clear owed, and the spec
1329
+ promises the NEXT strike retries it — not only the next poison-class
1330
+ strike. A clean probe failure would otherwise overwrite the durable
1331
+ detail and strand the owed clear forever.
1332
+ """
1333
+ mapped = _http_bridge_anchor_poison_detail(detail)
1334
+ if mapped is not None:
1335
+ return mapped
1336
+ async with self._http_bridge_retry_circuit_lock:
1337
+ state = self._http_bridge_retry_circuits.get(session.key)
1338
+ owed = state.owed_poison_detail if state is not None else None
1339
+ return _http_bridge_anchor_poison_detail(owed)
1340
+
1341
+ async def _http_bridge_mark_poison_anchor_cleared(
1342
+ self: Any,
1343
+ session: _HTTPBridgeSession,
1344
+ *,
1345
+ episode: _HTTPBridgeRetryCircuitState,
1346
+ ) -> None:
1347
+ """Record that this episode's poisoned anchor has been abandoned.
1348
+
1349
+ The marker belongs to the episode that performed the abandonment. If
1350
+ that episode was settled and replaced while the durable rebind was in
1351
+ flight, marking whatever is registered now would suppress the new
1352
+ episode's own required abandonment and leave its poisoned anchor
1353
+ reusable, so a caller that captured its episode marks only that one.
1354
+ """
1355
+ async with self._http_bridge_retry_circuit_lock:
1356
+ state = self._http_bridge_retry_circuits.get(session.key)
1357
+ if state is not None and state is episode:
1358
+ state.poison_anchor_cleared = True
1359
+ state.owed_poison_detail = None
1360
+
1361
+ async def _http_bridge_precreated_retry_block(
1362
+ self: Any,
1363
+ session: _HTTPBridgeSession,
1364
+ ) -> tuple[float, str]:
1365
+ """Return ``(seconds_blocked, reason)`` for a suppressed submission.
1366
+
1367
+ The cooldown is not always what is refusing the request: once it has
1368
+ expired, the half-open lease keeps refusing everything except the one
1369
+ admitted probe. Reporting the cooldown in that window advertises a
1370
+ retry-after of ~1s while the caller is barred for the rest of the
1371
+ lease, which turns a wedged key into a client retry storm.
1372
+ """
1373
+ if session.key.strength != "hard":
1374
+ return 0.0, "none"
1375
+
1376
+ await self._load_http_bridge_retry_circuit(session)
1377
+ now = clock_for(self).monotonic()
1378
+ async with self._http_bridge_retry_circuit_lock:
1379
+ state = self._http_bridge_retry_circuits.get(session.key)
1380
+ if state is None:
1381
+ return 0.0, "none"
1382
+ cooldown_remaining = max(0.0, state.cooldown_until - now)
1383
+ half_open_remaining = (
1384
+ max(0.0, state.half_open_until - now)
1385
+ if state.consecutive_failures >= _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD
1386
+ else 0.0
1387
+ )
1388
+ if half_open_remaining > cooldown_remaining:
1389
+ return half_open_remaining, "hard_key_half_open"
1390
+ return cooldown_remaining, "hard_key_cooldown"
1391
+
1392
+ async def _http_bridge_precreated_retry_block_for_key(
1393
+ self: Any,
1394
+ key: _HTTPBridgeSessionKey,
1395
+ *,
1396
+ assume_remote_half_open_lease: bool = False,
1397
+ ) -> tuple[float, str]:
1398
+ """Return ``(seconds_blocked, reason)`` for a suppressed replacement.
1399
+
1400
+ The same-owner stale-anchor recovery dispatches on a unique internal
1401
+ session while the circuit that refused it lives on the source hard
1402
+ key, so the block has to be read from that key. During the half-open
1403
+ lease the cooldown is already zero; reading only the replacement
1404
+ session (which has no circuit) and the source key's cooldown
1405
+ advertises a ~1s retry-after while the caller is barred for the rest
1406
+ of the lease.
1407
+
1408
+ ``assume_remote_half_open_lease`` covers the timer this process
1409
+ cannot see: a caller that just lost its dispatch claim was refused by
1410
+ a probe admitted somewhere, and when neither a local cooldown nor a
1411
+ local lease is visible that probe's lease lives in another process
1412
+ (or died with a replaced local state). The lease deadline is not
1413
+ persisted, so the block is reported at the configured lease duration
1414
+ — the upper bound of the timer actually refusing the caller — rather
1415
+ than a fabricated ~1s.
1416
+ """
1417
+ if key.strength != "hard":
1418
+ return 0.0, "none"
1419
+ now = clock_for(self).monotonic()
1420
+ async with self._http_bridge_retry_circuit_lock:
1421
+ state = self._http_bridge_retry_circuits.get(key)
1422
+ local_cooldown_remaining = max(0.0, state.cooldown_until - now) if state is not None else 0.0
1423
+ half_open_remaining = (
1424
+ max(0.0, state.half_open_until - now)
1425
+ if state is not None and state.consecutive_failures >= _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD
1426
+ else 0.0
1427
+ )
1428
+ cooldown_remaining = max(
1429
+ local_cooldown_remaining,
1430
+ await self._http_bridge_retry_circuit_cooldown_seconds_for_key(key),
1431
+ )
1432
+ if cooldown_remaining <= 0.0 and half_open_remaining <= 0.0:
1433
+ if assume_remote_half_open_lease:
1434
+ return _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS, "hard_key_half_open"
1435
+ return 0.0, "none"
1436
+ if half_open_remaining > cooldown_remaining:
1437
+ return half_open_remaining, "hard_key_half_open"
1438
+ return cooldown_remaining, "hard_key_cooldown"
1439
+
1440
+ async def _http_bridge_precreated_retry_cooldown_seconds(self: Any, session: _HTTPBridgeSession) -> float:
1441
+ if session.key.strength != "hard":
1442
+ return 0.0
1443
+
1444
+ await self._load_http_bridge_retry_circuit(session)
1445
+ now = clock_for(self).monotonic()
1446
+ async with self._http_bridge_retry_circuit_lock:
1447
+ state = self._http_bridge_retry_circuits.get(session.key)
1448
+ if state is None:
1449
+ return 0.0
1450
+ return max(0.0, state.cooldown_until - now)
1451
+
1452
+ async def _http_bridge_retry_circuit_cooldown_seconds_for_key(
1453
+ self: Any,
1454
+ key: _HTTPBridgeSessionKey,
1455
+ ) -> float:
1456
+ """Return the source-key cooldown used to suppress a replacement."""
1457
+ load_succeeded, generation = await self._http_bridge_retry_circuit_generation_for_key(key)
1458
+ if not load_succeeded or generation is None:
1459
+ return 0.0
1460
+ return max(
1461
+ 0.0,
1462
+ generation[3] - clock_for(self).time(),
1463
+ generation[6] - clock_for(self).monotonic(),
1464
+ )
1465
+
1466
+ async def _record_http_bridge_retry_circuit_failure(
1467
+ self: Any,
1468
+ session: _HTTPBridgeSession,
1469
+ *,
1470
+ detail: str,
1471
+ attempt: _HTTPBridgeResponseCreateAttempt | None = None,
1472
+ terminal_pre_response_frame: bool = False,
1473
+ ) -> int | None:
1474
+ """Count one hard-key failure against the retry circuit.
1475
+
1476
+ ``terminal_pre_response_frame`` is asserted only by the settlement path
1477
+ for an upstream terminal frame that failed the request before any
1478
+ response event. ``response.failed``/``response.incomplete`` mark the
1479
+ attempt observed without counting a response event, so without that
1480
+ assertion a native terminal envelope is rejected below as already
1481
+ settled and never consumes a strike. The eventless retirement funnel
1482
+ never asserts it, so its "upstream answered" guard is unchanged.
1483
+ """
1484
+ detail = _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES.get(detail, detail)
1485
+ if session.key.strength != "hard" or detail not in _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_DETAILS:
1486
+ return None
1487
+
1488
+ scoped_attempt = attempt
1489
+ if scoped_attempt is not None:
1490
+ if scoped_attempt.retry_circuit_failure_recorded:
1491
+ return await self._await_http_bridge_retry_circuit_attempt_settlement(
1492
+ session,
1493
+ attempt=scoped_attempt,
1494
+ detail=detail,
1495
+ )
1496
+ if (
1497
+ scoped_attempt.disarmed
1498
+ or (scoped_attempt.response_observed and not terminal_pre_response_frame)
1499
+ or (terminal_pre_response_frame and scoped_attempt.non_terminal_response_observed)
1500
+ ):
1501
+ return None
1502
+
1503
+ threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
1504
+ base_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS)
1505
+ max_backoff = max(base_backoff, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS)
1506
+ clean_close_max_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS)
1507
+ duplicate_attempt: _HTTPBridgeResponseCreateAttempt | None = None
1508
+ state: _HTTPBridgeRetryCircuitState | None = None
1509
+ poison_class_failure = _http_bridge_anchor_poison_detail(detail) is not None
1510
+ quarantine_poisoned_anchor = False
1511
+ quarantine_cooldown_remaining = 0.0
1512
+ recorded_failures: int | None = None
1513
+ # The registry mutation and its durable write hold the key lock
1514
+ # together: a settle for this key either completes before the strike
1515
+ # registers (the strike then opens a fresh episode) or waits until
1516
+ # the strike's write has landed. Without this, a failure recorded
1517
+ # after a settlement linearized could be dropped as superseded and
1518
+ # disappear instead of opening the circuit.
1519
+ key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(session.key)
1520
+ try:
1521
+ # The load runs under the key lock too: outside it, a settle in
1522
+ # flight lets the load re-hydrate the just-popped counts from the
1523
+ # not-yet-reset row, and this strike would then extend the ended
1524
+ # episode instead of opening a fresh one.
1525
+ await self._load_http_bridge_retry_circuit(session)
1526
+ # Sampled after the keyed wait and the load: a wait approaching
1527
+ # the base backoff would otherwise persist an already-aged
1528
+ # cooldown and make the fresh failure look older than the
1529
+ # durable load for merge bookkeeping.
1530
+ now = clock_for(self).monotonic()
1531
+ async with self._http_bridge_retry_circuit_lock:
1532
+ if scoped_attempt is not None and scoped_attempt.retry_circuit_failure_recorded:
1533
+ duplicate_attempt = scoped_attempt
1534
+ elif scoped_attempt is not None and (
1535
+ scoped_attempt.disarmed
1536
+ or (scoped_attempt.response_observed and not terminal_pre_response_frame)
1537
+ # ``terminal_pre_response_frame`` exists because the terminal
1538
+ # frame itself marks the attempt observed. It must not also
1539
+ # steamroll genuine midstream evidence: a deferred-reasoning
1540
+ # prelude observed a non-terminal response event without
1541
+ # counting it, so a later terminal failure is a midstream
1542
+ # failure, not a pre-response strike.
1543
+ or (terminal_pre_response_frame and scoped_attempt.non_terminal_response_observed)
1544
+ ):
1545
+ return None
1546
+ else:
1547
+ state = self._http_bridge_retry_circuits.setdefault(
1548
+ session.key,
1549
+ _HTTPBridgeRetryCircuitState(last_touched_monotonic=now),
1550
+ )
1551
+ state.last_touched_monotonic = now
1552
+ state.last_failure_monotonic = now
1553
+ state.half_open_until = 0.0
1554
+ if scoped_attempt is not None:
1555
+ scoped_attempt.retry_circuit_failure_recorded = True
1556
+ scoped_attempt.retry_circuit_failure_settled = anyio.Event()
1557
+ state.consecutive_failures += 1
1558
+ prior_recorded_detail = state.last_detail
1559
+ state.last_detail = detail
1560
+ if _http_bridge_anchor_poison_detail(detail) is not None:
1561
+ if prior_recorded_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL:
1562
+ # The supersession marked the OLD anchor's clear
1563
+ # as no longer owed; this strike is evidence
1564
+ # against the freshly registered anchor and
1565
+ # starts a new abandonment story of its own.
1566
+ state.poison_anchor_cleared = False
1567
+ if state.consecutive_failures >= threshold:
1568
+ # Debt exists only for an episode that reached the
1569
+ # poison threshold on poison evidence — the same
1570
+ # threshold that authorizes the abandonment, so
1571
+ # the strike that opens the circuit arms the debt
1572
+ # its failed clear leaves owed.
1573
+ # Arming below it let a clean_close opener
1574
+ # resurrect an earlier poison detail and clear a
1575
+ # valid anchor with no quarantine covering the
1576
+ # race.
1577
+ state.owed_poison_detail = detail
1578
+ if state.consecutive_failures >= threshold:
1579
+ backoff = min(
1580
+ max_backoff,
1581
+ base_backoff * (2 ** min(state.consecutive_failures - threshold, 30)),
1582
+ )
1583
+ if detail == "clean_close":
1584
+ backoff = min(backoff, clean_close_max_backoff)
1585
+ state.cooldown_until = max(state.cooldown_until, now + backoff)
1586
+ # The probe admitted after this cooldown is planned before
1587
+ # it reaches the gate, so an anchor the circuit opened on
1588
+ # has to be suppressed at planning time. Quarantining the
1589
+ # key routes a full-resend probe through the existing
1590
+ # unanchored fresh path; delta-only payloads keep their
1591
+ # anchor there, because it is their only context.
1592
+ quarantine_poisoned_anchor = poison_class_failure
1593
+ quarantine_cooldown_remaining = max(0.0, state.cooldown_until - now)
1594
+ if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
1595
+ http_bridge_retry_circuit_total.labels(outcome="opened").inc()
1596
+ logger.warning(
1597
+ "http_bridge_retry_circuit event=opened bridge_kind=%s bridge_key=%s "
1598
+ "failures=%s cooldown_seconds=%.1f detail=%s",
1599
+ session.key.affinity_kind,
1600
+ _hash_identifier(session.key.affinity_key),
1601
+ state.consecutive_failures,
1602
+ backoff,
1603
+ detail,
1604
+ )
1605
+ if duplicate_attempt is None:
1606
+ assert state is not None
1607
+ armed_quarantine_generation: int | None = None
1608
+ pre_arm_quarantine_reason: str | None = None
1609
+ pre_arm_quarantine_until = 0.0
1610
+ if quarantine_poisoned_anchor:
1611
+ # A revocation after a lost cross-replica race must not
1612
+ # evict a weaker fence this arm merely upgraded, so the
1613
+ # prior active entry is captured for restoration.
1614
+ prior_entry = _http_bridge_quarantine_registry(self).get(session.key)
1615
+ if (
1616
+ prior_entry is not None
1617
+ and prior_entry.quarantined_until > clock_for(self).monotonic()
1618
+ and prior_entry.reason != _HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON
1619
+ ):
1620
+ pre_arm_quarantine_reason = prior_entry.reason
1621
+ pre_arm_quarantine_until = prior_entry.quarantined_until
1622
+ _quarantine_http_bridge_session(
1623
+ self,
1624
+ session,
1625
+ reason=_HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON,
1626
+ minimum_seconds=_http_bridge_poison_quarantine_minimum_seconds(quarantine_cooldown_remaining),
1627
+ )
1628
+ # Captured so a lost cross-replica race can revoke exactly
1629
+ # this speculative arm and nothing re-armed after it.
1630
+ armed_quarantine_generation = _http_bridge_quarantine_generation(self, session.key)
1631
+ recorded_failures = await self._record_http_bridge_retry_circuit_failure_locked(
1632
+ session,
1633
+ state,
1634
+ scoped_attempt=scoped_attempt,
1635
+ threshold=threshold,
1636
+ quarantine_poisoned_anchor=quarantine_poisoned_anchor,
1637
+ quarantine_cooldown_remaining=quarantine_cooldown_remaining,
1638
+ armed_quarantine_generation=armed_quarantine_generation,
1639
+ pre_arm_quarantine_reason=pre_arm_quarantine_reason,
1640
+ pre_arm_quarantine_until=pre_arm_quarantine_until,
1641
+ )
1642
+ finally:
1643
+ key_lock.release()
1644
+ if duplicate_attempt is not None:
1645
+ return await self._await_http_bridge_retry_circuit_attempt_settlement(
1646
+ session,
1647
+ attempt=duplicate_attempt,
1648
+ detail=detail,
1649
+ )
1650
+ return recorded_failures
1651
+
1652
+ async def _record_http_bridge_retry_circuit_failure_locked(
1653
+ self: Any,
1654
+ session: _HTTPBridgeSession,
1655
+ state: _HTTPBridgeRetryCircuitState,
1656
+ *,
1657
+ scoped_attempt: _HTTPBridgeResponseCreateAttempt | None,
1658
+ threshold: int,
1659
+ quarantine_poisoned_anchor: bool,
1660
+ quarantine_cooldown_remaining: float,
1661
+ armed_quarantine_generation: int | None = None,
1662
+ pre_arm_quarantine_reason: str | None = None,
1663
+ pre_arm_quarantine_until: float = 0.0,
1664
+ ) -> int | None:
1665
+ try:
1666
+ await self._persist_http_bridge_retry_circuit_serialized(
1667
+ session,
1668
+ state,
1669
+ now_monotonic=clock_for(self).monotonic(),
1670
+ now_wall=clock_for(self).time(),
1671
+ threshold=threshold,
1672
+ )
1673
+ merged_cooldown_remaining = 0.0
1674
+ merged_poison_opened = False
1675
+ merged_poison_revoked = False
1676
+ async with self._http_bridge_retry_circuit_lock:
1677
+ if self._http_bridge_retry_circuits.get(session.key) is state:
1678
+ self._http_bridge_retry_circuit_loaded_keys.add(session.key)
1679
+ consecutive_failures = state.consecutive_failures
1680
+ # Replicas that each record their locally-first failure stay
1681
+ # below the threshold under their own lock, and it is the
1682
+ # durable merge that opens the circuit. Neither worker ever saw
1683
+ # an open circuit above, so the quarantine decision has to be
1684
+ # revisited against the merged state or the probe admitted
1685
+ # after this cooldown is planned with the poisoned anchor.
1686
+ #
1687
+ # The merged cooldown can already have elapsed — another replica
1688
+ # may have opened the circuit long enough ago that its deadline
1689
+ # is in the past by the time this write returns. That key is at
1690
+ # its threshold with no cooldown left, so the very next request
1691
+ # is the half-open probe: it needs the quarantine most, not
1692
+ # least. Track the opening itself, and let a zero remainder fall
1693
+ # through to the bare half-open lease.
1694
+ #
1695
+ # This deliberately does not skip keys quarantined from the
1696
+ # local opening. A local open arms the floor from its own
1697
+ # backoff, which can be 60s, and the merge can then replace
1698
+ # `cooldown_until` with a deadline up to 600s out. Re-arming is
1699
+ # idempotent because the entry keeps the later of the two
1700
+ # deadlines, so recomputing against the merged cooldown can only
1701
+ # extend a floor that would otherwise expire mid-cooldown.
1702
+ #
1703
+ # Only when the registry still maps this key to this state. A
1704
+ # multiplexed sibling that completed while the persist was in
1705
+ # flight clears the circuit and drops this object, and it also
1706
+ # advanced the anchor; re-arming off the detached state would
1707
+ # resurrect a poisoned key whose anchor is now valid and let a
1708
+ # later explicit rejection discard it. Nor is there anything to
1709
+ # re-arm when this call already quarantined the key from its own
1710
+ # opening and the merge did not move the deadline: that arm is a
1711
+ # no-op except for the generation bump, which is the fence the
1712
+ # verified stale-anchor replay claims at dispatch (#1863).
1713
+ if self._http_bridge_retry_circuits.get(session.key) is state:
1714
+ merged_cooldown_remaining = max(0.0, state.cooldown_until - clock_for(self).monotonic())
1715
+ # The persist merge adopted the returned row, so the
1716
+ # verdict must come from the adopted detail and count,
1717
+ # not the local strike's class: a clean_close losing to
1718
+ # a poison opening must still quarantine, and a poison
1719
+ # strike losing to a reset or clean lineage must not
1720
+ # leave its speculative quarantine suppressing a valid
1721
+ # anchor.
1722
+ adopted_poison_class = _http_bridge_anchor_poison_detail(state.last_detail) is not None
1723
+ # The anchor-poison threshold is the circuit threshold:
1724
+ # a non-poison local strike adopting a poison row that
1725
+ # reaches it must still arm the quarantine, or the
1726
+ # loaded-key cache hands the next request the dead
1727
+ # anchor.
1728
+ poison_arm_threshold = threshold
1729
+ merged_poison_opened = (
1730
+ adopted_poison_class
1731
+ and consecutive_failures >= poison_arm_threshold
1732
+ and (
1733
+ not quarantine_poisoned_anchor or merged_cooldown_remaining > quarantine_cooldown_remaining
1734
+ )
1735
+ )
1736
+ # Revocation mirrors the arm's own justification: the
1737
+ # quarantine survives whenever the adopted state still
1738
+ # satisfies that same bar.
1739
+ merged_poison_revoked = quarantine_poisoned_anchor and not (
1740
+ adopted_poison_class and consecutive_failures >= poison_arm_threshold
1741
+ )
1742
+ if merged_poison_opened:
1743
+ _quarantine_http_bridge_session(
1744
+ self,
1745
+ session,
1746
+ reason=_HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON,
1747
+ minimum_seconds=_http_bridge_poison_quarantine_minimum_seconds(merged_cooldown_remaining),
1748
+ )
1749
+ elif merged_poison_revoked:
1750
+ _revoke_http_bridge_poison_quarantine(
1751
+ self,
1752
+ session.key,
1753
+ generation=armed_quarantine_generation,
1754
+ restore_reason=pre_arm_quarantine_reason,
1755
+ restore_until=pre_arm_quarantine_until,
1756
+ )
1757
+ return consecutive_failures
1758
+ finally:
1759
+ if scoped_attempt is not None and scoped_attempt.retry_circuit_failure_settled is not None:
1760
+ scoped_attempt.retry_circuit_failure_settled.set()
1761
+
1762
+ async def _http_bridge_suppress_poison_clear_after_anchor_advance(
1763
+ self: Any,
1764
+ session: _HTTPBridgeSession,
1765
+ ) -> tuple[Any, str | None, str | None, float, int, bool] | None:
1766
+ """Mark the surviving episode's clear as no longer owed.
1767
+
1768
+ A completed response just advanced the durable anchor while its
1769
+ circuit settlement failed and the old episode was restored. That
1770
+ episode's recorded failures were all against the superseded anchor,
1771
+ so its owed abandonment must not fire against the fresh one: one
1772
+ later eventless failure would otherwise ride the restored
1773
+ at-threshold count straight into clearing continuity the episode
1774
+ never proved dead. The cooldown itself is left standing and settles
1775
+ at the next opportunity.
1776
+
1777
+ The suppression is also persisted by rewriting the surviving row's
1778
+ failure detail under the row's version fence: the local marker
1779
+ protects only this worker, and another replica loading the
1780
+ surviving at-threshold poison row would otherwise arm quarantine
1781
+ against the fresh anchor and authorize an abandonment for failures
1782
+ recorded against the superseded one. The durable transitional
1783
+ detail is the fail-closed ``anchor_abandoned`` tombstone, not
1784
+ ``anchor_superseded``: the registration has not landed yet, so a
1785
+ crash inside the window must read as "deltas fail closed", never
1786
+ as a disproved episode that hands replicas the old poisoned
1787
+ anchor. The caller promotes the tombstone to ``anchor_superseded``
1788
+ once the fresh anchor commits. A concurrent strike outracing the
1789
+ rewrite keeps its poison class — it is new evidence — and a failed
1790
+ rewrite leaves the suppression process-local, no worse than
1791
+ before.
1792
+ """
1793
+ key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(session.key)
1794
+ try:
1795
+ async with self._http_bridge_retry_circuit_lock:
1796
+ state = self._http_bridge_retry_circuits.get(session.key)
1797
+ if state is None:
1798
+ return None
1799
+ if (
1800
+ _http_bridge_anchor_poison_detail(state.last_detail) is None
1801
+ and state.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
1802
+ # Owed debt IS poison evidence even when a later
1803
+ # non-poison strike overwrote the local detail (its
1804
+ # durable persist and reload can both fail): without a
1805
+ # token the debt survives the fresh registration and a
1806
+ # later strike abandons the anchor just registered.
1807
+ and state.owed_poison_detail is None
1808
+ ):
1809
+ # A clean episode owes no abandonment; marking it would
1810
+ # refuse the clear a LATER poison strike genuinely owes,
1811
+ # since only the supersession sentinel resets the marker
1812
+ # on a fresh strike. An existing tombstone is NOT clean:
1813
+ # it is already the transitional durable state (a sticky
1814
+ # tombstone can carry later strikes onto a positive
1815
+ # count), and returning no token here would leave a
1816
+ # committed registration with nothing to promote — every
1817
+ # later submission carrying the freshly registered
1818
+ # anchor would then be rejected against the lingering
1819
+ # tombstone.
1820
+ return None
1821
+ prior_owed = state.owed_poison_detail
1822
+ state.poison_anchor_cleared = True
1823
+ state.owed_poison_detail = None
1824
+ if state.persisted_updated_at_epoch <= 0:
1825
+ return (state, None, prior_owed, state.persisted_updated_at_epoch, 0, False)
1826
+ captured_state = state
1827
+ captured_epoch = state.persisted_updated_at_epoch
1828
+ prior_detail = state.last_detail
1829
+ supersede_base_epoch = state.persisted_updated_at_epoch
1830
+ supersede_base_count = state.consecutive_failures
1831
+ if state.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL:
1832
+ # A local tombstone stays a tombstone until the fresh
1833
+ # anchor's registration commits and the promotion
1834
+ # rewrites the durable row: flipping the local cache to
1835
+ # the superseded sentinel early would let a concurrent
1836
+ # proof-gated resend that cannot reload bypass the
1837
+ # fail-closed gates and send the still-stored poisoned
1838
+ # anchor.
1839
+ state.last_detail = _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL
1840
+ superseded = False
1841
+ try:
1842
+ # A detail-only fenced update, not a strike write: the strike
1843
+ # upsert increments the merged count by design, and a phantom
1844
+ # strike here would extend the cooldown. The row's version is
1845
+ # left unchanged so a concurrent strike still merges onto it
1846
+ # and its poison class overwrites this sentinel — new
1847
+ # evidence outranks the supersession in either order. The
1848
+ # fence carries the observed count as well: a lagging-clock
1849
+ # strike can merge without moving the version, and every
1850
+ # landed merge increments the count, so the count is what
1851
+ # makes such a strike outrank a supersession that follows it.
1852
+ superseded = await self._durable_bridge.supersede_retry_circuit_detail(
1853
+ session_key_kind=session.key.affinity_kind,
1854
+ session_key_value=session.key.affinity_key,
1855
+ api_key_id=session.key.api_key_id,
1856
+ expected_updated_at_epoch=supersede_base_epoch,
1857
+ expected_consecutive_failures=supersede_base_count,
1858
+ expected_last_detail=prior_detail,
1859
+ last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
1860
+ )
1861
+ except Exception:
1862
+ logger.warning(
1863
+ "Failed to persist HTTP bridge anchor supersession bridge_kind=%s bridge_key=%s",
1864
+ session.key.affinity_kind,
1865
+ _hash_identifier(session.key.affinity_key),
1866
+ exc_info=True,
1867
+ )
1868
+ if not superseded:
1869
+ async with self._http_bridge_retry_circuit_lock:
1870
+ current = self._http_bridge_retry_circuits.get(session.key)
1871
+ if current is state and state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL:
1872
+ # The fenced rewrite missed (the row moved) or the
1873
+ # write failed: keep the local view matching the
1874
+ # durable row; the suppression stays process-local,
1875
+ # no worse than before.
1876
+ state.last_detail = prior_detail
1877
+ return (captured_state, prior_detail, prior_owed, captured_epoch, supersede_base_count, superseded)
1878
+ finally:
1879
+ key_lock.release()
1880
+
1881
+ async def _http_bridge_restore_poison_row_after_failed_registration(
1882
+ self: Any,
1883
+ session: _HTTPBridgeSession,
1884
+ poison_detail: str,
1885
+ ) -> None:
1886
+ """Re-seed durable poison evidence after a failed anchor registration.
1887
+
1888
+ The settle-before-registration ordering is fence-critical, so a
1889
+ registration failure lands after the circuit was already zeroed
1890
+ while the old poisoned anchor is still the stored one. The kept
1891
+ local quarantine alone cannot hold: the next planning load reads
1892
+ the zeroed row as a disproved episode and revokes it, and other
1893
+ replicas never arm. Re-opening the row at the circuit threshold
1894
+ with the prior poison class restores the protection everywhere; a
1895
+ CAS drop against a row that moved concurrently defers to that
1896
+ newer evidence.
1897
+ """
1898
+ key = session.key
1899
+ now_wall = clock_for(self).time()
1900
+ now_monotonic = clock_for(self).monotonic()
1901
+ try:
1902
+ row = await self._durable_bridge.lookup_retry_circuit(
1903
+ session_key_kind=key.affinity_kind,
1904
+ session_key_value=key.affinity_key,
1905
+ api_key_id=key.api_key_id,
1906
+ )
1907
+ except Exception:
1908
+ logger.warning(
1909
+ "Failed to reload HTTP bridge retry circuit before poison restore bridge_kind=%s bridge_key=%s",
1910
+ key.affinity_kind,
1911
+ _hash_identifier(key.affinity_key),
1912
+ exc_info=True,
1913
+ )
1914
+ return
1915
+ threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
1916
+ if row is not None and row.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL:
1917
+ # The failed registration follows this completion's OWN settle,
1918
+ # which wrote the transitional tombstone; the strike merge's
1919
+ # sticky-tombstone rule would silently refuse to write the
1920
+ # poison class back over it, degrading the re-seed to a
1921
+ # threshold tombstone no replica arms a quarantine from. The
1922
+ # fenced detail-only supersede is the one path allowed through
1923
+ # that stickiness; a CAS miss means newer evidence owns the row
1924
+ # and the merge below defers to it.
1925
+ try:
1926
+ await self._durable_bridge.supersede_retry_circuit_detail(
1927
+ session_key_kind=key.affinity_kind,
1928
+ session_key_value=key.affinity_key,
1929
+ api_key_id=key.api_key_id,
1930
+ expected_updated_at_epoch=row.updated_at_epoch,
1931
+ expected_consecutive_failures=row.consecutive_failures,
1932
+ expected_last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
1933
+ last_detail=poison_detail,
1934
+ )
1935
+ except Exception:
1936
+ logger.warning(
1937
+ "Failed to transition HTTP bridge tombstone before poison restore bridge_kind=%s bridge_key=%s",
1938
+ key.affinity_kind,
1939
+ _hash_identifier(key.affinity_key),
1940
+ exc_info=True,
1941
+ )
1942
+ if row.consecutive_failures >= threshold:
1943
+ return
1944
+ elif row is not None and row.consecutive_failures >= threshold:
1945
+ return
1946
+ base_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS)
1947
+ try:
1948
+ await self._durable_bridge.persist_retry_circuit(
1949
+ session_key_kind=key.affinity_kind,
1950
+ session_key_value=key.affinity_key,
1951
+ api_key_id=key.api_key_id,
1952
+ consecutive_failures=threshold,
1953
+ cooldown_until_epoch=now_wall + base_backoff,
1954
+ last_detail=poison_detail,
1955
+ updated_at_epoch=now_wall,
1956
+ base_updated_at_epoch=row.updated_at_epoch if row is not None else 0.0,
1957
+ failure_threshold=threshold,
1958
+ poison_sticky_threshold=threshold,
1959
+ conflict_cooldown_until_epoch=now_wall + base_backoff,
1960
+ base_backoff_seconds=base_backoff,
1961
+ max_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS),
1962
+ clean_close_max_backoff_seconds=max(
1963
+ 0.001,
1964
+ _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS,
1965
+ ),
1966
+ )
1967
+ except Exception:
1968
+ logger.warning(
1969
+ "Failed to restore HTTP bridge poison evidence bridge_kind=%s bridge_key=%s",
1970
+ key.affinity_kind,
1971
+ _hash_identifier(key.affinity_key),
1972
+ exc_info=True,
1973
+ )
1974
+ return
1975
+ del now_monotonic
1976
+ # Adopt the restored row locally and re-arm the quarantine through
1977
+ # the ordinary load path.
1978
+ await self._load_http_bridge_retry_circuit(session)
1979
+
1980
+ async def _http_bridge_restore_poison_clear_after_failed_anchor_advance(
1981
+ self: Any,
1982
+ session: _HTTPBridgeSession,
1983
+ supersession: tuple[Any, str | None, str | None, float, int, bool] | None,
1984
+ ) -> None:
1985
+ """Roll a transitional anchor-advance suppression back.
1986
+
1987
+ The suppression is applied before the fresh anchor is registered, so
1988
+ a concurrent consult whose continuity read lands after publication
1989
+ cannot validate the old poison row against the just-published
1990
+ anchor. When that registration then fails, the old anchor is still
1991
+ the stored one, and leaving the suppression in place would strand it
1992
+ with no funnel willing to clear it — the failure the
1993
+ registration-gated ordering existed to prevent.
1994
+ """
1995
+ if supersession is None:
1996
+ return
1997
+ captured_state, prior_detail, prior_owed, base_epoch, base_count, superseded = supersession
1998
+ key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(session.key)
1999
+ try:
2000
+ async with self._http_bridge_retry_circuit_lock:
2001
+ state = self._http_bridge_retry_circuits.get(session.key)
2002
+ if (
2003
+ state is not None
2004
+ # Fenced to the exact captured episode and lineage: a
2005
+ # same-key load, strike, or settlement replacing or
2006
+ # reconciling the state while the registration was in
2007
+ # flight owns the key now, and copying the ended
2008
+ # lineage's owed evidence into it would let a later
2009
+ # strike clear continuity that episode never proved
2010
+ # dead.
2011
+ and state is captured_state
2012
+ and state.persisted_updated_at_epoch == base_epoch
2013
+ ):
2014
+ state.poison_anchor_cleared = False
2015
+ if prior_owed is not None:
2016
+ state.owed_poison_detail = prior_owed
2017
+ if (
2018
+ prior_detail is not None
2019
+ and state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL
2020
+ ):
2021
+ state.last_detail = prior_detail
2022
+ if superseded and prior_detail is not None:
2023
+ try:
2024
+ # A fence miss here means a strike moved the row and its
2025
+ # own poison class already replaced the sentinel.
2026
+ await self._durable_bridge.supersede_retry_circuit_detail(
2027
+ session_key_kind=session.key.affinity_kind,
2028
+ session_key_value=session.key.affinity_key,
2029
+ api_key_id=session.key.api_key_id,
2030
+ expected_updated_at_epoch=base_epoch,
2031
+ expected_consecutive_failures=base_count,
2032
+ expected_last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
2033
+ last_detail=prior_detail,
2034
+ )
2035
+ except Exception:
2036
+ logger.warning(
2037
+ "Failed to roll back HTTP bridge anchor supersession bridge_kind=%s bridge_key=%s",
2038
+ session.key.affinity_kind,
2039
+ _hash_identifier(session.key.affinity_key),
2040
+ exc_info=True,
2041
+ )
2042
+ finally:
2043
+ key_lock.release()
2044
+
2045
+ async def _http_bridge_reconcile_transitional_tombstone(self: Any, session: _HTTPBridgeSession) -> None:
2046
+ """Rewrite a lingering transitional tombstone off the row's own values.
2047
+
2048
+ Strike merges keep ``anchor_abandoned`` sticky, so a CAS miss on a
2049
+ promote or erase does NOT mean the tombstone was replaced — it means
2050
+ the count moved underneath the fence while the tombstone stood, and
2051
+ the freshly committed anchor would stay rejected on every replica.
2052
+ One reload-and-rewrite round converges the interleavings: a zeroed
2053
+ row erases to a plain reset, a positive count promotes to the
2054
+ superseded sentinel, and a second miss defers to newer evidence and
2055
+ the next completion's settle-and-erase.
2056
+ """
2057
+ for reconcile_attempt in range(2):
2058
+ try:
2059
+ row = await self._durable_bridge.lookup_retry_circuit(
2060
+ session_key_kind=session.key.affinity_kind,
2061
+ session_key_value=session.key.affinity_key,
2062
+ api_key_id=session.key.api_key_id,
2063
+ )
2064
+ if row is None or row.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL:
2065
+ return
2066
+ rewrote = await self._durable_bridge.supersede_retry_circuit_detail(
2067
+ session_key_kind=session.key.affinity_kind,
2068
+ session_key_value=session.key.affinity_key,
2069
+ api_key_id=session.key.api_key_id,
2070
+ expected_updated_at_epoch=row.updated_at_epoch,
2071
+ expected_consecutive_failures=row.consecutive_failures,
2072
+ expected_last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
2073
+ last_detail=(
2074
+ None if row.consecutive_failures == 0 else _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL
2075
+ ),
2076
+ )
2077
+ if rewrote:
2078
+ return
2079
+ except Exception:
2080
+ if reconcile_attempt == 0:
2081
+ continue
2082
+ logger.debug(
2083
+ "Failed to reconcile transitional circuit tombstone",
2084
+ exc_info=True,
2085
+ )
2086
+ return
2087
+
2088
+ async def _http_bridge_promote_transitional_supersession(
2089
+ self: Any,
2090
+ session: _HTTPBridgeSession,
2091
+ supersession: tuple[Any, str | None, str | None, float, int, bool] | None,
2092
+ ) -> None:
2093
+ """Promote a landed transitional tombstone to ``anchor_superseded``.
2094
+
2095
+ The fresh anchor just committed, so the fail-closed tombstone the
2096
+ suppression persisted is no longer the right durable record: left
2097
+ in place it would 404 healthy delta follow-ups riding the valid
2098
+ fresh anchor, and other replicas' stale quarantines would never
2099
+ see the disproof. A fence miss means a concurrent strike moved the
2100
+ row and its newer evidence stands; a failed promote leaves the
2101
+ conservative tombstone, which only fails closed.
2102
+ """
2103
+ if supersession is None:
2104
+ return
2105
+ _captured_state, _prior_detail, _prior_owed, base_epoch, base_count, superseded = supersession
2106
+ if not superseded:
2107
+ return
2108
+ for promote_attempt in range(2):
2109
+ try:
2110
+ promoted = await self._durable_bridge.supersede_retry_circuit_detail(
2111
+ session_key_kind=session.key.affinity_kind,
2112
+ session_key_value=session.key.affinity_key,
2113
+ api_key_id=session.key.api_key_id,
2114
+ expected_updated_at_epoch=base_epoch,
2115
+ expected_consecutive_failures=base_count,
2116
+ expected_last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
2117
+ last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL,
2118
+ )
2119
+ if promoted is False:
2120
+ # A concurrent strike moved the count while stickiness
2121
+ # kept the tombstone detail: reporting success here
2122
+ # would leave the freshly committed anchor rejected on
2123
+ # every replica. Reconcile on the row's own values.
2124
+ await self._http_bridge_reconcile_transitional_tombstone(session)
2125
+ return
2126
+ except Exception:
2127
+ # One immediate retry covers a transient durable blip: a
2128
+ # skipped promotion leaves the fresh anchor rejected on
2129
+ # every replica until the next completion's settle-and-erase
2130
+ # heals it. A second failure defers to that healing.
2131
+ if promote_attempt == 0:
2132
+ continue
2133
+ logger.warning(
2134
+ "Failed to promote HTTP bridge anchor supersession bridge_kind=%s bridge_key=%s",
2135
+ session.key.affinity_kind,
2136
+ _hash_identifier(session.key.affinity_key),
2137
+ exc_info=True,
2138
+ )
2139
+
2140
+ async def _clear_http_bridge_retry_circuit(
2141
+ self: Any,
2142
+ session: _HTTPBridgeSession,
2143
+ *,
2144
+ settled_detail: str | None = None,
2145
+ settled_detail_authoritative: bool = False,
2146
+ expected_episode: tuple[float, int, int] | None = None,
2147
+ ) -> bool:
2148
+ """Settle a hard key's retry circuit; ``True`` when settlement held.
2149
+
2150
+ The load below re-hydrates any persisted row into the local state, so
2151
+ after a successful load the fence is absent only when no durable row
2152
+ was observed at all — and an unfenced delete there could only remove a
2153
+ row another writer created concurrently. A strike write still in
2154
+ flight while this clear runs is undone by its own writer:
2155
+ ``_persist_http_bridge_retry_circuit`` re-checks the registry after
2156
+ its durable write lands and deletes exactly the row it created.
2157
+ """
2158
+ key = session.key
2159
+ if key.strength != "hard":
2160
+ return True
2161
+ key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(key)
2162
+ try:
2163
+ return await self._clear_http_bridge_retry_circuit_serialized(
2164
+ session,
2165
+ settled_detail=settled_detail,
2166
+ settled_detail_authoritative=settled_detail_authoritative,
2167
+ expected_episode=expected_episode,
2168
+ )
2169
+ finally:
2170
+ key_lock.release()
2171
+
2172
+ async def _clear_http_bridge_retry_circuit_serialized(
2173
+ self: Any,
2174
+ session: _HTTPBridgeSession,
2175
+ *,
2176
+ settled_detail: str | None = None,
2177
+ settled_detail_authoritative: bool = False,
2178
+ expected_episode: tuple[float, int, int] | None = None,
2179
+ ) -> bool:
2180
+ key = session.key
2181
+ durable_load_succeeded = await self._load_http_bridge_retry_circuit(session)
2182
+ if expected_episode is not None:
2183
+ # The caller settles a specific authorized episode — an
2184
+ # abandonment clears only the circuit its abandonment
2185
+ # invalidated. A replacement episode opened between the
2186
+ # authorization and this settle (fresh strikes against a newly
2187
+ # registered anchor) carries its own valid cooldown; resetting
2188
+ # it would let the newly poisoned anchor retry immediately.
2189
+ # The count discriminates lagging-clock merges the epoch alone
2190
+ # cannot see. Nothing is popped on a mismatch: the settlement
2191
+ # stays owed and the newer row stands untouched.
2192
+ async with self._http_bridge_retry_circuit_lock:
2193
+ fenced_state = self._http_bridge_retry_circuits.get(key)
2194
+ if fenced_state is not None and (
2195
+ fenced_state.persisted_updated_at_epoch != expected_episode[0]
2196
+ or fenced_state.consecutive_failures != expected_episode[1]
2197
+ # A replay claim advances only the admission generation:
2198
+ # resetting beneath it would clear the circuit a claimed
2199
+ # replay depends on and admit a second dispatch beside it.
2200
+ or fenced_state.persisted_admission_generation != expected_episode[2]
2201
+ ):
2202
+ return False
2203
+ async with self._http_bridge_retry_circuit_lock:
2204
+ state = self._http_bridge_retry_circuits.pop(key, None)
2205
+ self._http_bridge_retry_circuit_loaded_keys.discard(key)
2206
+ self._http_bridge_retry_circuit_persisted_keys.discard(key)
2207
+ if state is not None or not durable_load_succeeded:
2208
+ # The pop takes the state's stale-load watermark with it;
2209
+ # keep a per-key watermark so a lookup that began before
2210
+ # this settlement cannot adopt the pre-settlement row into a
2211
+ # fresh state and resurrect the settled cooldown. A healthy
2212
+ # key with no state and a confirmed durable miss has nothing
2213
+ # to resurrect, and stamping every completed conversation
2214
+ # would grow this map — and its global-lock prune scan —
2215
+ # with every key the process ever serves.
2216
+ self._http_bridge_retry_circuit_reconcile_watermarks[key] = clock_for(self).monotonic()
2217
+ expected_updated_at_epoch = (
2218
+ state.persisted_updated_at_epoch if state is not None and state.persisted_updated_at_epoch > 0 else None
2219
+ )
2220
+ expected_admission_generation = state.persisted_admission_generation if state is not None else None
2221
+ # A lagging-clock strike merges a higher count without moving the
2222
+ # epoch; the reset CAS carries the observed count so it cannot
2223
+ # zero a newer episode's cooldown it never saw. A completion
2224
+ # settle defeated this way still wins through the chase.
2225
+ expected_consecutive_failures = state.consecutive_failures if state is not None else None
2226
+ if (
2227
+ settled_detail is None
2228
+ # An authoritative None was chosen after consulting continuity —
2229
+ # the caller SAW fresh evidence replacing the poisoned anchor —
2230
+ # and upgrading it back to a tombstone would fail the fresh
2231
+ # continuity closed. Only a blind None is upgraded.
2232
+ and not settled_detail_authoritative
2233
+ and state is not None
2234
+ and (
2235
+ _http_bridge_anchor_poison_detail(state.last_detail) is not None
2236
+ or state.owed_poison_detail is not None
2237
+ or state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
2238
+ )
2239
+ ):
2240
+ # The caller's pre-settle capture can be blind — a transient
2241
+ # read failure before this call — while this settle's own load
2242
+ # just adopted a poison episode or an existing tombstone.
2243
+ # Resetting such a row plain would leave the old poisoned
2244
+ # anchor behind a reset no replica reads as suspect, so the
2245
+ # reset detail is derived from the state actually settled: the
2246
+ # fail-closed tombstone, healed by a later completion's
2247
+ # fence-aware settle-and-erase.
2248
+ settled_detail = _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
2249
+ # A confirmed miss has no version fence to protect a row created
2250
+ # concurrently, so leave the durable row untouched when no state was
2251
+ # observed. Preserve the existing best-effort clear on read failures,
2252
+ # which is still useful for settling a row after a transient outage.
2253
+ if durable_load_succeeded and (state is None or expected_updated_at_epoch is None):
2254
+ return True
2255
+ if state is None and not durable_load_succeeded:
2256
+ # No local episode and no durable observation: this worker holds
2257
+ # no fence at all, and an unfenced reset here could clear a
2258
+ # poison episode or a claimed admission generation another
2259
+ # replica created during the outage. The best-effort clear is
2260
+ # reserved for a worker that at least carries a local episode;
2261
+ # with nothing observed, the settlement stays owed.
2262
+ return False
2263
+ try:
2264
+ # Clearing is idempotent and must be attempted even when the
2265
+ # preceding lookup failed; a successful request should settle
2266
+ # a previously persisted circuit after a transient read error.
2267
+ reset_matched = await self._durable_bridge.clear_retry_circuit(
2268
+ session_key_kind=key.affinity_kind,
2269
+ session_key_value=key.affinity_key,
2270
+ api_key_id=key.api_key_id,
2271
+ expected_updated_at_epoch=expected_updated_at_epoch,
2272
+ expected_admission_generation=expected_admission_generation,
2273
+ expected_consecutive_failures=expected_consecutive_failures,
2274
+ reset_detail=settled_detail,
2275
+ )
2276
+ if reset_matched is False and expected_updated_at_epoch is not None:
2277
+ # The fenced reset matched no row: another writer moved it
2278
+ # after this worker's lookup. This settlement carries the
2279
+ # newer evidence — the completed response proved the key
2280
+ # works — so reload the moved row once and retry the fence
2281
+ # against its current version, the same settle-wins rule the
2282
+ # in-process race already follows (a strike writer undoes
2283
+ # its own row when the registry no longer maps its state).
2284
+ moved = await self._durable_bridge.lookup_retry_circuit(
2285
+ session_key_kind=key.affinity_kind,
2286
+ session_key_value=key.affinity_key,
2287
+ api_key_id=key.api_key_id,
2288
+ )
2289
+ if moved is None or moved.consecutive_failures <= 0:
2290
+ # The moved row was purged or already reset: settled.
2291
+ reset_matched = True
2292
+ elif (
2293
+ expected_admission_generation is not None
2294
+ and moved.updated_at_epoch == expected_updated_at_epoch
2295
+ and getattr(moved, "admission_generation", 0) != expected_admission_generation
2296
+ ):
2297
+ # Same failure observation, advanced admission
2298
+ # generation: a replica claimed the replay admission
2299
+ # this circuit fences, and the claim deliberately holds
2300
+ # the circuit open for its dispatch. Chasing the moved
2301
+ # generation here would clear it anyway — leave the
2302
+ # settlement owed instead.
2303
+ pass
2304
+ elif expected_episode is not None and (
2305
+ moved.updated_at_epoch != expected_episode[0]
2306
+ # Strictly greater, not unequal: merges only ever
2307
+ # increment the count, so a LOWER count at the same
2308
+ # epoch and admission generation cannot be foreign — it
2309
+ # is this worker's own episode observed before local
2310
+ # strikes whose durable writes failed. Refusing to
2311
+ # settle it would leave a confirmed abandonment's
2312
+ # cooldown standing against an anchor already removed.
2313
+ or moved.consecutive_failures > expected_episode[1]
2314
+ or getattr(moved, "admission_generation", 0) != expected_episode[2]
2315
+ ):
2316
+ # An episode-fenced settle never chases a moved row: the
2317
+ # caller authorized clearing exactly one episode, and a
2318
+ # row that no longer matches the captured epoch, count,
2319
+ # and admission generation is a replacement lineage —
2320
+ # re-fencing on ITS version would durably zero the valid
2321
+ # cooldown a cross-replica completion-plus-strikes just
2322
+ # opened against the fresh anchor. The settlement stays
2323
+ # owed; the settle-wins chase is for completion callers,
2324
+ # whose own evidence outranks any concurrent strike.
2325
+ pass
2326
+ else:
2327
+ reset_matched = await self._durable_bridge.clear_retry_circuit(
2328
+ session_key_kind=key.affinity_kind,
2329
+ session_key_value=key.affinity_key,
2330
+ api_key_id=key.api_key_id,
2331
+ expected_updated_at_epoch=moved.updated_at_epoch,
2332
+ expected_admission_generation=getattr(moved, "admission_generation", None),
2333
+ expected_consecutive_failures=moved.consecutive_failures,
2334
+ reset_detail=settled_detail,
2335
+ )
2336
+ if reset_matched is False and expected_updated_at_epoch is not None:
2337
+ # Two fenced attempts both matched nothing: the row is
2338
+ # moving faster than this settlement can chase it, so
2339
+ # dropping the local state would report settlement while the
2340
+ # durable episode survives to be reloaded. Keep the episode
2341
+ # for the next clear opportunity — but reconcile it onto the
2342
+ # row that actually survived first: restoring the original
2343
+ # snapshot would hand the anchor-supersession rewrite an
2344
+ # obsolete fence, and the next load would then read the
2345
+ # surviving row as a foreign episode, reset the one-clear
2346
+ # marker, and authorize abandoning the fresh anchor this
2347
+ # completion just registered.
2348
+ surviving = None
2349
+ refresh_failed = False
2350
+ try:
2351
+ surviving = await self._durable_bridge.lookup_retry_circuit(
2352
+ session_key_kind=key.affinity_kind,
2353
+ session_key_value=key.affinity_key,
2354
+ api_key_id=key.api_key_id,
2355
+ )
2356
+ except Exception:
2357
+ refresh_failed = True
2358
+ logger.warning(
2359
+ "Failed to reload HTTP bridge retry circuit after settle misses bridge_kind=%s bridge_key=%s",
2360
+ key.affinity_kind,
2361
+ _hash_identifier(key.affinity_key),
2362
+ exc_info=True,
2363
+ )
2364
+ if not refresh_failed and (surviving is None or surviving.consecutive_failures <= 0):
2365
+ # The chase ended settled after all: the row was purged
2366
+ # or reset by another writer while this settlement was
2367
+ # missing its fences.
2368
+ reset_matched = True
2369
+ else:
2370
+ async with self._http_bridge_retry_circuit_lock:
2371
+ if state is not None and self._http_bridge_retry_circuits.get(key) is None:
2372
+ if surviving is not None and surviving.consecutive_failures > 0:
2373
+ if surviving.updated_at_epoch != state.persisted_updated_at_epoch:
2374
+ # Adopting a lineage this worker did not
2375
+ # produce: same marker rule as the load
2376
+ # and persist merges.
2377
+ state.poison_anchor_cleared = False
2378
+ state.owed_poison_detail = None
2379
+ state.consecutive_failures = max(0, surviving.consecutive_failures)
2380
+ state.cooldown_until = clock_for(self).monotonic() + max(
2381
+ 0.0, surviving.cooldown_until_epoch - clock_for(self).time()
2382
+ )
2383
+ state.last_detail = surviving.last_detail
2384
+ if state.cooldown_until > clock_for(self).monotonic():
2385
+ state.half_open_until = 0.0
2386
+ state.persisted_updated_at_epoch = surviving.updated_at_epoch
2387
+ state.persisted_admission_generation = getattr(surviving, "admission_generation", 0)
2388
+ state.last_durable_load_monotonic = max(
2389
+ state.last_durable_load_monotonic,
2390
+ clock_for(self).monotonic(),
2391
+ )
2392
+ self._http_bridge_retry_circuits[key] = state
2393
+ self._http_bridge_retry_circuit_loaded_keys.add(key)
2394
+ self._http_bridge_retry_circuit_persisted_keys.add(key)
2395
+ return False
2396
+ except Exception:
2397
+ logger.warning(
2398
+ "Failed to clear persisted HTTP bridge retry circuit bridge_kind=%s bridge_key=%s",
2399
+ key.affinity_kind,
2400
+ _hash_identifier(key.affinity_key),
2401
+ exc_info=True,
2402
+ )
2403
+ # A fenced delete failing means a durable row this worker wrote
2404
+ # is known to have survived, so this is not a completed
2405
+ # settlement: put the popped episode back (unless a newer one
2406
+ # already took the key) so the process keeps its version fence
2407
+ # and the next clear opportunity retries the fenced delete,
2408
+ # instead of a later load resurrecting the row as a fresh
2409
+ # cooldown against a cause that is already gone. An unfenced
2410
+ # best-effort delete after a failed load proves nothing about
2411
+ # any row and keeps its old settle-anyway semantics.
2412
+ if state is not None and expected_updated_at_epoch is not None:
2413
+ async with self._http_bridge_retry_circuit_lock:
2414
+ if self._http_bridge_retry_circuits.get(key) is None:
2415
+ self._http_bridge_retry_circuits[key] = state
2416
+ self._http_bridge_retry_circuit_loaded_keys.add(key)
2417
+ self._http_bridge_retry_circuit_persisted_keys.add(key)
2418
+ return False
2419
+ async with self._http_bridge_retry_circuit_lock:
2420
+ # Re-stamp past the durable delete: the pop-time watermark above
2421
+ # predates the commit, so a load that started between them passes
2422
+ # the stale-load guard while its snapshot still shows the deleted
2423
+ # row. Strikes hold this key's lock for the whole settle, so any
2424
+ # state present here was resurrected by exactly such a load —
2425
+ # drop it with the same fence.
2426
+ self._http_bridge_retry_circuit_reconcile_watermarks[key] = clock_for(self).monotonic()
2427
+ resurrected = self._http_bridge_retry_circuits.get(key)
2428
+ if resurrected is not None and resurrected is not state:
2429
+ self._http_bridge_retry_circuits.pop(key, None)
2430
+ self._http_bridge_retry_circuit_loaded_keys.discard(key)
2431
+ self._http_bridge_retry_circuit_persisted_keys.discard(key)
2432
+ if state is None:
2433
+ return True
2434
+ if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
2435
+ http_bridge_retry_circuit_total.labels(outcome="reset").inc()
2436
+ logger.info(
2437
+ "http_bridge_retry_circuit event=reset bridge_kind=%s bridge_key=%s failures=%s",
2438
+ key.affinity_kind,
2439
+ _hash_identifier(key.affinity_key),
2440
+ state.consecutive_failures,
2441
+ )
2442
+ return True