codexLB-up 1.25.0b10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of codexLB-up might be problematic. Click here for more details.

Files changed (827) hide show
  1. app/__init__.py +10 -0
  2. app/admin_cli.py +494 -0
  3. app/cli.py +339 -0
  4. app/codex_sessions_retag.py +487 -0
  5. app/core/__init__.py +0 -0
  6. app/core/audit/__init__.py +0 -0
  7. app/core/audit/service.py +216 -0
  8. app/core/audit/types.py +94 -0
  9. app/core/auth/__init__.py +236 -0
  10. app/core/auth/api_key_cache.py +65 -0
  11. app/core/auth/dashboard_access.py +429 -0
  12. app/core/auth/dashboard_mode.py +171 -0
  13. app/core/auth/dashboard_session_ttl.py +74 -0
  14. app/core/auth/dashboard_users_cache.py +120 -0
  15. app/core/auth/dependencies.py +710 -0
  16. app/core/auth/external_identity.py +78 -0
  17. app/core/auth/guardian.py +374 -0
  18. app/core/auth/models.py +49 -0
  19. app/core/auth/providers/__init__.py +129 -0
  20. app/core/auth/providers/oidc.py +838 -0
  21. app/core/auth/providers/registry.py +118 -0
  22. app/core/auth/refresh.py +394 -0
  23. app/core/auth/step_up.py +117 -0
  24. app/core/auth/totp.py +81 -0
  25. app/core/balancer/__init__.py +71 -0
  26. app/core/balancer/logic.py +1612 -0
  27. app/core/balancer/rendezvous_hash.py +21 -0
  28. app/core/balancer/types.py +21 -0
  29. app/core/bootstrap.py +172 -0
  30. app/core/cache/invalidation.py +426 -0
  31. app/core/clients/__init__.py +0 -0
  32. app/core/clients/codex.py +996 -0
  33. app/core/clients/codex_version.py +187 -0
  34. app/core/clients/codex_version_snapshot.py +2 -0
  35. app/core/clients/files.py +534 -0
  36. app/core/clients/headers.py +29 -0
  37. app/core/clients/http.py +511 -0
  38. app/core/clients/model_fetcher.py +276 -0
  39. app/core/clients/native_egress.py +1304 -0
  40. app/core/clients/oauth.py +477 -0
  41. app/core/clients/proxy.py +6089 -0
  42. app/core/clients/proxy_websocket.py +1411 -0
  43. app/core/clients/rate_limit_reset_credits.py +412 -0
  44. app/core/clients/stream_errors.py +12 -0
  45. app/core/clients/thread_cache_identity.py +347 -0
  46. app/core/clients/usage.py +467 -0
  47. app/core/clock.py +170 -0
  48. app/core/config/__init__.py +0 -0
  49. app/core/config/background_jobs.py +73 -0
  50. app/core/config/context_window_overrides.py +166 -0
  51. app/core/config/dashboard_overrides.py +145 -0
  52. app/core/config/inheritable.py +47 -0
  53. app/core/config/key_fingerprint.py +115 -0
  54. app/core/config/settings.py +872 -0
  55. app/core/config/settings_cache.py +100 -0
  56. app/core/config/spool_retention.py +217 -0
  57. app/core/config/tiers.py +174 -0
  58. app/core/conversation_archive.py +568 -0
  59. app/core/crypto.py +37 -0
  60. app/core/draining.py +11 -0
  61. app/core/errors.py +365 -0
  62. app/core/exceptions.py +155 -0
  63. app/core/handlers/__init__.py +3 -0
  64. app/core/handlers/exceptions.py +478 -0
  65. app/core/http_protocol.py +176 -0
  66. app/core/http_protocol_httptools.py +204 -0
  67. app/core/ingress_limits.py +17 -0
  68. app/core/metadata_store.py +18 -0
  69. app/core/metrics/__init__.py +0 -0
  70. app/core/metrics/middleware.py +74 -0
  71. app/core/metrics/prometheus.py +604 -0
  72. app/core/middleware/__init__.py +25 -0
  73. app/core/middleware/api_firewall.py +111 -0
  74. app/core/middleware/app_version.py +37 -0
  75. app/core/middleware/dashboard_auth_proxy.py +55 -0
  76. app/core/middleware/dashboard_csrf.py +177 -0
  77. app/core/middleware/dashboard_gzip.py +41 -0
  78. app/core/middleware/dashboard_overrides.py +59 -0
  79. app/core/middleware/firewall_cache.py +91 -0
  80. app/core/middleware/inflight.py +126 -0
  81. app/core/middleware/multipart_content_encoding.py +129 -0
  82. app/core/middleware/path_rewrite.py +148 -0
  83. app/core/middleware/request_body_limit.py +196 -0
  84. app/core/middleware/request_decompression.py +209 -0
  85. app/core/middleware/request_id.py +72 -0
  86. app/core/middleware/required_capability_http.py +116 -0
  87. app/core/middleware/trusted_proxy_headers.py +36 -0
  88. app/core/multipart.py +237 -0
  89. app/core/multipart_fields.py +91 -0
  90. app/core/openai/__init__.py +0 -0
  91. app/core/openai/chat_requests.py +493 -0
  92. app/core/openai/chat_responses.py +931 -0
  93. app/core/openai/contracts.py +46 -0
  94. app/core/openai/exceptions.py +24 -0
  95. app/core/openai/host_models.py +14 -0
  96. app/core/openai/images.py +413 -0
  97. app/core/openai/message_coercion.py +376 -0
  98. app/core/openai/model_refresh_scheduler.py +423 -0
  99. app/core/openai/model_registry.py +997 -0
  100. app/core/openai/model_registry_store.py +548 -0
  101. app/core/openai/models.py +172 -0
  102. app/core/openai/parsing.py +97 -0
  103. app/core/openai/requests.py +1980 -0
  104. app/core/openai/strict_schema.py +199 -0
  105. app/core/openai/tool_call_safety.py +99 -0
  106. app/core/openai/v1_requests.py +187 -0
  107. app/core/plan_types.py +90 -0
  108. app/core/prestop.py +202 -0
  109. app/core/rate_limiter/__init__.py +0 -0
  110. app/core/rate_limiter/db_rate_limiter.py +215 -0
  111. app/core/request_locality.py +473 -0
  112. app/core/resilience/__init__.py +0 -0
  113. app/core/resilience/backpressure.py +57 -0
  114. app/core/resilience/bulkhead.py +202 -0
  115. app/core/resilience/circuit_breaker.py +150 -0
  116. app/core/resilience/degradation.py +55 -0
  117. app/core/resilience/loop_lag_monitor.py +60 -0
  118. app/core/resilience/memory_monitor.py +137 -0
  119. app/core/resilience/network_recovery.py +256 -0
  120. app/core/resilience/overload.py +98 -0
  121. app/core/resilience/toggles.py +114 -0
  122. app/core/retention/__init__.py +0 -0
  123. app/core/retention/job.py +240 -0
  124. app/core/retention/scheduler.py +69 -0
  125. app/core/runtime_logging.py +601 -0
  126. app/core/scheduling/__init__.py +0 -0
  127. app/core/scheduling/leader_election.py +993 -0
  128. app/core/scheduling/leader_election_handle.py +25 -0
  129. app/core/server.py +140 -0
  130. app/core/shutdown.py +326 -0
  131. app/core/socket_peer.py +22 -0
  132. app/core/startup.py +41 -0
  133. app/core/timeout_invariants.py +347 -0
  134. app/core/tracing/__init__.py +0 -0
  135. app/core/tracing/otel.py +207 -0
  136. app/core/types.py +6 -0
  137. app/core/upstream_proxy/__init__.py +13 -0
  138. app/core/upstream_proxy/cache.py +130 -0
  139. app/core/upstream_proxy/resolver.py +189 -0
  140. app/core/upstream_proxy/types.py +104 -0
  141. app/core/usage/__init__.py +407 -0
  142. app/core/usage/depletion.py +109 -0
  143. app/core/usage/live_hub.py +41 -0
  144. app/core/usage/live_snapshots.py +174 -0
  145. app/core/usage/logs.py +212 -0
  146. app/core/usage/metadata_scheduler.py +118 -0
  147. app/core/usage/models.py +54 -0
  148. app/core/usage/pricing.py +576 -0
  149. app/core/usage/pricing_catalog.py +241 -0
  150. app/core/usage/pricing_snapshot.json +821 -0
  151. app/core/usage/quota.py +133 -0
  152. app/core/usage/refresh_policy.py +21 -0
  153. app/core/usage/refresh_scheduler.py +713 -0
  154. app/core/usage/reset_credits_refresh_scheduler.py +435 -0
  155. app/core/usage/types.py +168 -0
  156. app/core/utils/__init__.py +0 -0
  157. app/core/utils/json_guards.py +18 -0
  158. app/core/utils/locks.py +33 -0
  159. app/core/utils/masking.py +20 -0
  160. app/core/utils/proxy_env.py +104 -0
  161. app/core/utils/request_id.py +65 -0
  162. app/core/utils/retry.py +77 -0
  163. app/core/utils/shared_future.py +172 -0
  164. app/core/utils/sse.py +286 -0
  165. app/core/utils/time.py +24 -0
  166. app/db/__init__.py +0 -0
  167. app/db/account_identity_lock.py +56 -0
  168. app/db/alembic/__init__.py +1 -0
  169. app/db/alembic/env.py +70 -0
  170. app/db/alembic/revision_ids.py +43 -0
  171. app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
  172. app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
  173. app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
  174. app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
  175. app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
  176. app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
  177. app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
  178. app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
  179. app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
  180. app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
  181. app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
  182. app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
  183. app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
  184. app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
  185. app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
  186. app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
  187. app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
  188. app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
  189. app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
  190. app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
  191. app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
  192. app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
  193. app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
  194. app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
  195. app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
  196. app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
  197. app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
  198. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
  199. app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
  200. app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
  201. app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
  202. app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
  203. app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
  204. app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
  205. app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
  206. app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
  207. app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
  208. app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
  209. app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
  210. app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
  211. app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
  212. app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
  213. app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
  214. app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
  215. app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
  216. app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
  217. app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
  218. app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
  219. app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
  220. app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
  221. app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
  222. app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
  223. app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
  224. app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
  225. app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
  226. app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
  227. app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
  228. app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
  229. app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
  230. app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
  231. app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
  232. app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
  233. app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
  234. app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
  235. app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
  236. app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
  237. app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
  238. app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
  239. app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
  240. app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
  241. app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
  242. app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
  243. app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
  244. app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
  245. app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
  246. app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
  247. app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
  248. app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
  249. app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
  250. app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
  251. app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
  252. app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
  253. app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
  254. app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
  255. app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
  256. app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
  257. app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
  258. app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
  259. app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
  260. app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
  261. app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
  262. app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
  263. app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
  264. app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
  265. app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
  266. app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
  267. app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
  268. app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
  269. app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
  270. app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
  271. app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
  272. app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
  273. app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
  274. app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
  275. app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
  276. app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
  277. app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
  278. app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
  279. app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
  280. app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
  281. app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
  282. app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
  283. app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
  284. app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
  285. app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
  286. app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
  287. app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
  288. app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
  289. app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
  290. app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
  291. app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
  292. app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
  293. app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
  294. app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
  295. app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
  296. app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
  297. app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
  298. app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
  299. app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
  300. app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
  301. app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
  302. app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
  303. app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
  304. app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
  305. app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
  306. app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
  307. app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
  308. app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
  309. app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
  310. app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
  311. app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
  312. app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
  313. app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
  314. app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
  315. app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
  316. app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
  317. app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
  318. app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
  319. app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
  320. app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
  321. app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
  322. app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
  323. app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
  324. app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
  325. app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
  326. app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
  327. app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
  328. app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
  329. app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
  330. app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
  331. app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
  332. app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
  333. app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
  334. app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
  335. app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
  336. app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
  337. app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
  338. app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
  339. app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
  340. app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
  341. app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
  342. app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
  343. app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
  344. app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
  345. app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
  346. app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
  347. app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
  348. app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
  349. app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
  350. app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
  351. app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
  352. app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
  353. app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
  354. app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
  355. app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
  356. app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
  357. app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
  358. app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
  359. app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
  360. app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
  361. app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
  362. app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
  363. app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
  364. app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
  365. app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
  366. app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
  367. app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
  368. app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
  369. app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
  370. app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
  371. app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
  372. app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
  373. app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
  374. app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
  375. app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
  376. app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
  377. app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
  378. app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
  379. app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
  380. app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
  381. app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
  382. app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
  383. app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
  384. app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
  385. app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
  386. app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
  387. app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
  388. app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
  389. app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
  390. app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
  391. app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
  392. app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
  393. app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
  394. app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
  395. app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
  396. app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
  397. app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
  398. app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
  399. app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
  400. app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
  401. app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
  402. app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
  403. app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
  404. app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
  405. app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
  406. app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
  407. app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
  408. app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
  409. app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
  410. app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
  411. app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
  412. app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
  413. app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
  414. app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
  415. app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
  416. app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
  417. app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
  418. app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
  419. app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
  420. app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
  421. app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
  422. app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
  423. app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
  424. app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
  425. app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
  426. app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
  427. app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
  428. app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
  429. app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
  430. app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
  431. app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
  432. app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
  433. app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
  434. app/db/alembic/versions/__init__.py +1 -0
  435. app/db/backup.py +64 -0
  436. app/db/migrate.py +1249 -0
  437. app/db/migration_lock.py +160 -0
  438. app/db/migration_url.py +18 -0
  439. app/db/models.py +3100 -0
  440. app/db/recover.py +244 -0
  441. app/db/session.py +1295 -0
  442. app/db/snapshot.py +46 -0
  443. app/db/sqlite_lock_retry.py +190 -0
  444. app/db/sqlite_utils.py +549 -0
  445. app/dependencies.py +462 -0
  446. app/main.py +1240 -0
  447. app/modules/__init__.py +0 -0
  448. app/modules/accounts/__init__.py +0 -0
  449. app/modules/accounts/api.py +426 -0
  450. app/modules/accounts/auth_manager.py +1439 -0
  451. app/modules/accounts/background_repository.py +160 -0
  452. app/modules/accounts/deletion.py +540 -0
  453. app/modules/accounts/mappers.py +654 -0
  454. app/modules/accounts/refresh_claims.py +352 -0
  455. app/modules/accounts/repository.py +1679 -0
  456. app/modules/accounts/schemas.py +308 -0
  457. app/modules/accounts/service.py +809 -0
  458. app/modules/accounts/usage_rollup.py +486 -0
  459. app/modules/accounts/usage_rollup_scheduler.py +75 -0
  460. app/modules/accounts/usage_time_rollup.py +1037 -0
  461. app/modules/accounts/usage_time_rollup_read.py +522 -0
  462. app/modules/api_keys/__init__.py +1 -0
  463. app/modules/api_keys/api.py +344 -0
  464. app/modules/api_keys/last_used_coalescer.py +216 -0
  465. app/modules/api_keys/limit_windows.py +41 -0
  466. app/modules/api_keys/repository.py +1200 -0
  467. app/modules/api_keys/reset_scheduler.py +85 -0
  468. app/modules/api_keys/schemas.py +135 -0
  469. app/modules/api_keys/service.py +2123 -0
  470. app/modules/audit/__init__.py +0 -0
  471. app/modules/audit/api.py +88 -0
  472. app/modules/audit/repository.py +74 -0
  473. app/modules/audit/schemas.py +33 -0
  474. app/modules/audit/service.py +71 -0
  475. app/modules/auth_providers/__init__.py +0 -0
  476. app/modules/auth_providers/api.py +127 -0
  477. app/modules/auth_providers/config.py +343 -0
  478. app/modules/auth_providers/repository.py +101 -0
  479. app/modules/auth_providers/schemas.py +88 -0
  480. app/modules/auth_providers/seed.py +105 -0
  481. app/modules/auth_providers/service.py +289 -0
  482. app/modules/automations/__init__.py +1 -0
  483. app/modules/automations/api.py +369 -0
  484. app/modules/automations/repository.py +1896 -0
  485. app/modules/automations/scheduler.py +107 -0
  486. app/modules/automations/schemas.py +158 -0
  487. app/modules/automations/service.py +2548 -0
  488. app/modules/cache_isolation_probe/__init__.py +0 -0
  489. app/modules/cache_isolation_probe/api.py +188 -0
  490. app/modules/cache_isolation_probe/prefix.py +144 -0
  491. app/modules/cache_isolation_probe/schemas.py +94 -0
  492. app/modules/cache_isolation_probe/sender.py +229 -0
  493. app/modules/cache_isolation_probe/service.py +525 -0
  494. app/modules/conversation_archive/__init__.py +1 -0
  495. app/modules/conversation_archive/api.py +101 -0
  496. app/modules/conversation_archive/schemas.py +28 -0
  497. app/modules/conversation_archive/service.py +165 -0
  498. app/modules/dashboard/__init__.py +1 -0
  499. app/modules/dashboard/api.py +106 -0
  500. app/modules/dashboard/builders.py +107 -0
  501. app/modules/dashboard/repository.py +304 -0
  502. app/modules/dashboard/schemas.py +148 -0
  503. app/modules/dashboard/service.py +534 -0
  504. app/modules/dashboard/timeframes.py +19 -0
  505. app/modules/dashboard/weekly_pace.py +617 -0
  506. app/modules/dashboard_auth/__init__.py +1 -0
  507. app/modules/dashboard_auth/api.py +1344 -0
  508. app/modules/dashboard_auth/oidc_api.py +753 -0
  509. app/modules/dashboard_auth/oidc_flows.py +330 -0
  510. app/modules/dashboard_auth/repository.py +452 -0
  511. app/modules/dashboard_auth/schemas.py +232 -0
  512. app/modules/dashboard_auth/service.py +1384 -0
  513. app/modules/dashboard_roles/__init__.py +0 -0
  514. app/modules/dashboard_roles/api.py +64 -0
  515. app/modules/dashboard_roles/repository.py +54 -0
  516. app/modules/dashboard_roles/schemas.py +29 -0
  517. app/modules/dashboard_roles/seed.py +80 -0
  518. app/modules/dashboard_roles/service.py +142 -0
  519. app/modules/dashboard_users/__init__.py +0 -0
  520. app/modules/dashboard_users/api.py +278 -0
  521. app/modules/dashboard_users/break_glass.py +169 -0
  522. app/modules/dashboard_users/credentials.py +18 -0
  523. app/modules/dashboard_users/identity_resolver.py +665 -0
  524. app/modules/dashboard_users/repository.py +725 -0
  525. app/modules/dashboard_users/schemas.py +117 -0
  526. app/modules/dashboard_users/service.py +1054 -0
  527. app/modules/firewall/__init__.py +1 -0
  528. app/modules/firewall/api.py +78 -0
  529. app/modules/firewall/repository.py +51 -0
  530. app/modules/firewall/schemas.py +28 -0
  531. app/modules/firewall/service.py +93 -0
  532. app/modules/fleet/__init__.py +1 -0
  533. app/modules/fleet/api.py +177 -0
  534. app/modules/fleet/mappers.py +56 -0
  535. app/modules/fleet/observability.py +372 -0
  536. app/modules/fleet/schemas.py +119 -0
  537. app/modules/health/__init__.py +0 -0
  538. app/modules/health/api.py +246 -0
  539. app/modules/health/schemas.py +27 -0
  540. app/modules/limit_warmup/__init__.py +1 -0
  541. app/modules/limit_warmup/repository.py +159 -0
  542. app/modules/limit_warmup/service.py +1001 -0
  543. app/modules/model_sources/__init__.py +1 -0
  544. app/modules/model_sources/api.py +99 -0
  545. app/modules/model_sources/catalog.py +307 -0
  546. app/modules/model_sources/forwarding.py +1435 -0
  547. app/modules/model_sources/projection.py +55 -0
  548. app/modules/model_sources/repository.py +201 -0
  549. app/modules/model_sources/schemas.py +79 -0
  550. app/modules/model_sources/selection.py +166 -0
  551. app/modules/model_sources/service.py +236 -0
  552. app/modules/oauth/__init__.py +0 -0
  553. app/modules/oauth/api.py +101 -0
  554. app/modules/oauth/repository.py +238 -0
  555. app/modules/oauth/schemas.py +49 -0
  556. app/modules/oauth/service.py +1196 -0
  557. app/modules/oauth/templates/oauth_success.html +122 -0
  558. app/modules/proxy/__init__.py +0 -0
  559. app/modules/proxy/_load_balancer/__init__.py +1 -0
  560. app/modules/proxy/_load_balancer/error_rate.py +116 -0
  561. app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
  562. app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
  563. app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
  564. app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
  565. app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
  566. app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
  567. app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
  568. app/modules/proxy/_load_balancer/tunables.py +97 -0
  569. app/modules/proxy/_load_balancer/types.py +96 -0
  570. app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
  571. app/modules/proxy/_service/__init__.py +6 -0
  572. app/modules/proxy/_service/api_key_usage.py +745 -0
  573. app/modules/proxy/_service/clock_budget.py +38 -0
  574. app/modules/proxy/_service/codex_control.py +611 -0
  575. app/modules/proxy/_service/compact.py +2211 -0
  576. app/modules/proxy/_service/file_ops.py +749 -0
  577. app/modules/proxy/_service/http_bridge/__init__.py +3 -0
  578. app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
  579. app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
  580. app/modules/proxy/_service/http_bridge/activity.py +178 -0
  581. app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
  582. app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
  583. app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
  584. app/modules/proxy/_service/http_bridge/protocol.py +167 -0
  585. app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
  586. app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
  587. app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
  588. app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
  589. app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
  590. app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
  591. app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
  592. app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
  593. app/modules/proxy/_service/observability.py +410 -0
  594. app/modules/proxy/_service/rate_limit.py +361 -0
  595. app/modules/proxy/_service/realtime_live.py +655 -0
  596. app/modules/proxy/_service/refresh.py +154 -0
  597. app/modules/proxy/_service/request_log.py +585 -0
  598. app/modules/proxy/_service/response_create.py +930 -0
  599. app/modules/proxy/_service/streaming/__init__.py +3 -0
  600. app/modules/proxy/_service/streaming/helpers.py +1317 -0
  601. app/modules/proxy/_service/streaming/mixin.py +1100 -0
  602. app/modules/proxy/_service/streaming/protocol.py +32 -0
  603. app/modules/proxy/_service/streaming/retry.py +3762 -0
  604. app/modules/proxy/_service/support.py +2122 -0
  605. app/modules/proxy/_service/transcribe.py +434 -0
  606. app/modules/proxy/_service/warmup.py +483 -0
  607. app/modules/proxy/_service/websocket/__init__.py +3 -0
  608. app/modules/proxy/_service/websocket/helpers.py +2360 -0
  609. app/modules/proxy/_service/websocket/mixin.py +7285 -0
  610. app/modules/proxy/_service/websocket/protocol.py +74 -0
  611. app/modules/proxy/account_cache.py +229 -0
  612. app/modules/proxy/account_eligibility.py +87 -0
  613. app/modules/proxy/additional_model_limits.py +40 -0
  614. app/modules/proxy/affinity.py +830 -0
  615. app/modules/proxy/affinity_observation.py +101 -0
  616. app/modules/proxy/api.py +10254 -0
  617. app/modules/proxy/api_key_usage.py +113 -0
  618. app/modules/proxy/cap_partitioning.py +280 -0
  619. app/modules/proxy/capability_lineage.py +65 -0
  620. app/modules/proxy/capability_lineage_repository.py +82 -0
  621. app/modules/proxy/capability_routing.py +253 -0
  622. app/modules/proxy/continuity.py +112 -0
  623. app/modules/proxy/downstream_delivery.py +277 -0
  624. app/modules/proxy/durable_bridge_coordinator.py +1217 -0
  625. app/modules/proxy/durable_bridge_repository.py +4516 -0
  626. app/modules/proxy/durable_bridge_runtime.py +9 -0
  627. app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
  628. app/modules/proxy/fair_share.py +131 -0
  629. app/modules/proxy/file_pin_repository.py +210 -0
  630. app/modules/proxy/helpers.py +439 -0
  631. app/modules/proxy/http_bridge_event_batcher.py +613 -0
  632. app/modules/proxy/http_bridge_forwarding.py +797 -0
  633. app/modules/proxy/http_continuation.py +76 -0
  634. app/modules/proxy/images_observability.py +62 -0
  635. app/modules/proxy/images_service.py +1054 -0
  636. app/modules/proxy/load_balancer.py +3016 -0
  637. app/modules/proxy/rate_limit_cache.py +46 -0
  638. app/modules/proxy/replay_safety.py +1063 -0
  639. app/modules/proxy/repo_bundle.py +31 -0
  640. app/modules/proxy/request_policy.py +1054 -0
  641. app/modules/proxy/ring_membership.py +251 -0
  642. app/modules/proxy/schemas.py +345 -0
  643. app/modules/proxy/selection_errors.py +41 -0
  644. app/modules/proxy/service.py +2600 -0
  645. app/modules/proxy/source_admission.py +125 -0
  646. app/modules/proxy/source_dispatch.py +999 -0
  647. app/modules/proxy/sticky_repository.py +831 -0
  648. app/modules/proxy/thread_anchors.py +491 -0
  649. app/modules/proxy/tool_call_dedupe.py +802 -0
  650. app/modules/proxy/types.py +54 -0
  651. app/modules/proxy/work_admission.py +121 -0
  652. app/modules/quota_planner/__init__.py +1 -0
  653. app/modules/quota_planner/api.py +276 -0
  654. app/modules/quota_planner/logic.py +743 -0
  655. app/modules/quota_planner/repository.py +722 -0
  656. app/modules/quota_planner/scheduler.py +212 -0
  657. app/modules/quota_planner/schemas.py +102 -0
  658. app/modules/quota_planner/warmup.py +693 -0
  659. app/modules/rate_limit_reset_credits/__init__.py +0 -0
  660. app/modules/rate_limit_reset_credits/api.py +601 -0
  661. app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
  662. app/modules/rate_limit_reset_credits/store.py +114 -0
  663. app/modules/reports/__init__.py +1 -0
  664. app/modules/reports/api.py +134 -0
  665. app/modules/reports/cache.py +74 -0
  666. app/modules/reports/filters.py +27 -0
  667. app/modules/reports/repository.py +525 -0
  668. app/modules/reports/rollup.py +150 -0
  669. app/modules/reports/rollup_read.py +148 -0
  670. app/modules/reports/schemas.py +123 -0
  671. app/modules/reports/service.py +287 -0
  672. app/modules/reports/thread_identity.py +256 -0
  673. app/modules/request_logs/__init__.py +0 -0
  674. app/modules/request_logs/api.py +224 -0
  675. app/modules/request_logs/cost_backfill.py +154 -0
  676. app/modules/request_logs/mappers.py +97 -0
  677. app/modules/request_logs/repository.py +1765 -0
  678. app/modules/request_logs/schemas.py +145 -0
  679. app/modules/request_logs/service.py +348 -0
  680. app/modules/role_mappings/__init__.py +0 -0
  681. app/modules/role_mappings/api.py +206 -0
  682. app/modules/role_mappings/matching.py +55 -0
  683. app/modules/role_mappings/repository.py +106 -0
  684. app/modules/role_mappings/schemas.py +67 -0
  685. app/modules/role_mappings/service.py +334 -0
  686. app/modules/runtime/__init__.py +1 -0
  687. app/modules/runtime/api.py +18 -0
  688. app/modules/runtime/schemas.py +14 -0
  689. app/modules/runtime/service.py +218 -0
  690. app/modules/scim/__init__.py +1 -0
  691. app/modules/scim/api.py +194 -0
  692. app/modules/scim/dependencies.py +112 -0
  693. app/modules/scim/errors.py +77 -0
  694. app/modules/scim/management_api.py +156 -0
  695. app/modules/scim/repository.py +212 -0
  696. app/modules/scim/schemas.py +145 -0
  697. app/modules/scim/service.py +786 -0
  698. app/modules/scim/tokens.py +43 -0
  699. app/modules/settings/__init__.py +1 -0
  700. app/modules/settings/api.py +1677 -0
  701. app/modules/settings/repository.py +606 -0
  702. app/modules/settings/schemas.py +496 -0
  703. app/modules/settings/service.py +769 -0
  704. app/modules/shared/__init__.py +0 -0
  705. app/modules/shared/schemas.py +22 -0
  706. app/modules/sticky_sessions/__init__.py +1 -0
  707. app/modules/sticky_sessions/api.py +118 -0
  708. app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
  709. app/modules/sticky_sessions/schemas.py +78 -0
  710. app/modules/sticky_sessions/service.py +196 -0
  711. app/modules/telemetry/__init__.py +5 -0
  712. app/modules/telemetry/api.py +115 -0
  713. app/modules/telemetry/clients.py +62 -0
  714. app/modules/telemetry/consent.py +115 -0
  715. app/modules/telemetry/scheduler.py +85 -0
  716. app/modules/telemetry/schemas.py +163 -0
  717. app/modules/telemetry/sender.py +221 -0
  718. app/modules/telemetry/snapshot.py +490 -0
  719. app/modules/usage/__init__.py +0 -0
  720. app/modules/usage/additional_quota_keys.py +379 -0
  721. app/modules/usage/api.py +45 -0
  722. app/modules/usage/background_repository.py +147 -0
  723. app/modules/usage/builders.py +559 -0
  724. app/modules/usage/depletion_service.py +277 -0
  725. app/modules/usage/live_ingest.py +356 -0
  726. app/modules/usage/mappers.py +29 -0
  727. app/modules/usage/plan_downgrade_observations.py +488 -0
  728. app/modules/usage/repository.py +1890 -0
  729. app/modules/usage/schemas.py +68 -0
  730. app/modules/usage/service.py +102 -0
  731. app/modules/usage/updater.py +1488 -0
  732. app/static/assets/Trans-Cj-J9mVA.js +1 -0
  733. app/static/assets/access-page-DHzZiXZi.js +1 -0
  734. app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
  735. app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
  736. app/static/assets/account-status-DIsU5sUS.js +1 -0
  737. app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
  738. app/static/assets/accounts-page-Bbai8ift.js +3 -0
  739. app/static/assets/api-Dwa7XxPe.js +1 -0
  740. app/static/assets/api-client-dugX1Pti.js +55 -0
  741. app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
  742. app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
  743. app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
  744. app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
  745. app/static/assets/apis-page-BvXz9c6i.js +2 -0
  746. app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
  747. app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
  748. app/static/assets/automations-page-DQgNAJOR.js +3 -0
  749. app/static/assets/badge-DqMb9es0.js +1 -0
  750. app/static/assets/blur-email-HEvKagG2.js +1 -0
  751. app/static/assets/button-BjVmmiGW.js +1 -0
  752. app/static/assets/checkbox-apsl9vzk.js +1 -0
  753. app/static/assets/chevron-left-1d_pHVa5.js +1 -0
  754. app/static/assets/circle-alert-BS0xd_o1.js +1 -0
  755. app/static/assets/clsx-DB0hHKMi.js +1 -0
  756. app/static/assets/colors-BBy8GEtr.js +1 -0
  757. app/static/assets/copy-button-D7nAvOKR.js +1 -0
  758. app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
  759. app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
  760. app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
  761. app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
  762. app/static/assets/donut-chart-Bxc69cra.js +1 -0
  763. app/static/assets/download-kQteIcsB.js +1 -0
  764. app/static/assets/ellipsis-B_5OgIre.js +1 -0
  765. app/static/assets/empty-state-JcA7tc8m.js +1 -0
  766. app/static/assets/es2015-bMH8YKZT.js +41 -0
  767. app/static/assets/es6-B264ni6W.js +69 -0
  768. app/static/assets/external-link-UVAOgMQ0.js +1 -0
  769. app/static/assets/i18n-BWqZ9Ua2.js +1 -0
  770. app/static/assets/inbox-B_RM2s1J.js +1 -0
  771. app/static/assets/index-DIpd264Z.js +23 -0
  772. app/static/assets/index-DMeLKRJx.css +2 -0
  773. app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
  774. app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
  775. app/static/assets/iso-Bpe5Ihbb.js +1 -0
  776. app/static/assets/label-CGaLmLNf.js +1 -0
  777. app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
  778. app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
  779. app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
  780. app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
  781. app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
  782. app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
  783. app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
  784. app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
  785. app/static/assets/payload-CIYnGZz9.js +1 -0
  786. app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
  787. app/static/assets/report-chart-card-BXDliwcc.js +1 -0
  788. app/static/assets/reports-page-CGCJeMHG.js +3 -0
  789. app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
  790. app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
  791. app/static/assets/search-HfhqwERt.js +1 -0
  792. app/static/assets/select-Cl33bz-g.js +1 -0
  793. app/static/assets/settings-page-mZj01HYl.js +2 -0
  794. app/static/assets/shim-CZwyM4Q7.js +1 -0
  795. app/static/assets/skeleton-C_EUc4Rp.js +1 -0
  796. app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
  797. app/static/assets/status-badge-CHHA5X6c.js +1 -0
  798. app/static/assets/switch-CzxgokcM.js +1 -0
  799. app/static/assets/table-DXaNSQ0G.js +1 -0
  800. app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
  801. app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
  802. app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
  803. app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
  804. app/static/assets/trash-D3KyWr1h.js +1 -0
  805. app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
  806. app/static/assets/use-accounts-PWlVGw3z.js +1 -0
  807. app/static/assets/use-auth-CuDlW3iC.js +1 -0
  808. app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
  809. app/static/assets/use-model-sources-Bul24JBu.js +1 -0
  810. app/static/assets/use-models-DUCPUuZg.js +1 -0
  811. app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
  812. app/static/assets/useTranslation--56nHhlH.js +1 -0
  813. app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
  814. app/static/assets/users-uIC3TIlE.js +1 -0
  815. app/static/assets/vendor-query-_bHZuxEx.js +1 -0
  816. app/static/assets/vendor-react-BVL0YZPW.js +9 -0
  817. app/static/favicon.svg +10 -0
  818. app/static/fonts/GeistSans-Variable.woff2 +0 -0
  819. app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
  820. app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
  821. app/static/index.html +29 -0
  822. codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
  823. codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
  824. codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
  825. codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
  826. codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
  827. config/additional_quota_registry.json +11 -0
@@ -0,0 +1,4516 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import json
5
+ import time
6
+ from collections.abc import Collection, Mapping, Sequence
7
+ from dataclasses import dataclass
8
+ from datetime import datetime, timedelta
9
+ from enum import StrEnum
10
+ from hashlib import sha256
11
+ from typing import Any, cast
12
+
13
+ from sqlalchemy import Row, and_, case, delete, exists, func, or_, select, text, true, update
14
+ from sqlalchemy.dialects.postgresql import insert as pg_insert
15
+ from sqlalchemy.dialects.sqlite import insert as sqlite_insert
16
+ from sqlalchemy.exc import IntegrityError
17
+ from sqlalchemy.ext.asyncio import AsyncSession
18
+
19
+ from app.core.utils.time import naive_utc_to_epoch, to_utc_naive, utcnow
20
+ from app.db.models import (
21
+ HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2,
22
+ HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
23
+ HTTP_BRIDGE_TERMINAL_APPEND_PHASE_APPENDED,
24
+ HTTP_BRIDGE_TERMINAL_APPEND_PHASE_PENDING,
25
+ HTTP_BRIDGE_TERMINAL_APPEND_PHASE_SETTLED,
26
+ Account,
27
+ HttpBridgeOperationEvent,
28
+ HttpBridgeOperationEventChunk,
29
+ HttpBridgeOperationRecord,
30
+ HttpBridgeRecoveryAttemptRecord,
31
+ HttpBridgeRecoveryAttemptState,
32
+ HttpBridgeRetryCircuit,
33
+ HttpBridgeSessionAlias,
34
+ HttpBridgeSessionRecord,
35
+ HttpBridgeSessionState,
36
+ )
37
+ from app.db.session import sqlite_writer_section
38
+ from app.modules.proxy.account_eligibility import HARD_OWNER_UNAVAILABLE_STATUSES
39
+ from app.modules.proxy.continuity import (
40
+ HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX,
41
+ HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KIND,
42
+ HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_REBINDABLE_KINDS,
43
+ is_http_bridge_account_neutral_replay,
44
+ )
45
+ from app.modules.proxy.durable_bridge_transcript_codec import (
46
+ DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS,
47
+ DurableBridgeTranscriptDecodeError,
48
+ decode_durable_bridge_transcript_chunk,
49
+ encode_durable_bridge_transcript_chunk,
50
+ )
51
+
52
+ _ANONYMOUS_API_KEY_SCOPE = "__anonymous__"
53
+ REQUIRED_DURABLE_BRIDGE_TABLES = (
54
+ "http_bridge_sessions",
55
+ "http_bridge_session_aliases",
56
+ "http_bridge_retry_circuits",
57
+ "http_bridge_recovery_attempts",
58
+ "http_bridge_operations",
59
+ "http_bridge_operation_events",
60
+ "http_bridge_operation_event_chunks",
61
+ )
62
+ DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS = 3600.0
63
+ # Mirrors the HTTP bridge retry circuit's abandonment tombstone detail; the
64
+ # scheduled purge preserves such rows until the bridge-retention cutoff.
65
+ _RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL = "anchor_abandoned"
66
+ DURABLE_BRIDGE_OPERATION_SPOOL_PURGE_BATCH_SIZE = 50
67
+ _PURGE_CLOSED_BATCH_SIZE = 500
68
+ # Marks a retirement taken on the request path rather than by the sweep.
69
+ # The sweep writes the global timestamp form instead; both read as retired.
70
+ _REQUEST_PATH_ABANDONMENT_SCOPE = "request_path"
71
+ # Claim retry budget: insert races and epoch-CAS losses re-read and retry;
72
+ # each round has a winner, so a small budget converges under any realistic
73
+ # same-row claim contention.
74
+ _CLAIM_CAS_ATTEMPTS = 5
75
+ _SESSION_ID_LOOKUP_CHUNK_SIZE = 500
76
+ # Keep expanding ``IN`` predicates below the smallest supported SQLite bind
77
+ # budget. The bridge heartbeat can collect IDs from every pending request and
78
+ # event-spool context, so the caller's protection snapshot is not inherently
79
+ # bounded by this repository's batch size.
80
+ _PROTECTED_OPERATION_ID_SAFE_LIMIT = _SESSION_ID_LOOKUP_CHUNK_SIZE
81
+ # An oversized protection snapshot is scanned in small maintenance slices. The
82
+ # coordinator resumes from the returned keyset cursor on the next heartbeat so
83
+ # a large protected prefix cannot hold the SQLite writer section indefinitely.
84
+ _PROTECTED_OPERATION_SCAN_BUDGET = 128
85
+ _ABANDONMENT_LOG_AGE_CAP_SECONDS = 30 * 24 * 60 * 60
86
+ # Operation states that publish a final upstream outcome. ``abandoned`` is a
87
+ # duplicate-suppression fence rather than an outcome and is handled separately.
88
+ _HTTP_BRIDGE_TERMINAL_OPERATION_STATES = frozenset({"completed", "incomplete", "failed"})
89
+ # A terminal transcript outcome was already recorded for this dispatch, so a
90
+ # second terminal append must not rewrite it.
91
+ _HTTP_BRIDGE_TERMINAL_APPEND_RECORDED_PHASES = frozenset(
92
+ {HTTP_BRIDGE_TERMINAL_APPEND_PHASE_APPENDED, HTTP_BRIDGE_TERMINAL_APPEND_PHASE_SETTLED}
93
+ )
94
+
95
+
96
+ # Sentinel: rebind continuity clears without an anchor fence (legacy callers).
97
+ REBIND_ANCHOR_UNFENCED: object = object()
98
+
99
+
100
+ class DurableBridgeAliasRegistration(StrEnum):
101
+ REGISTERED = "registered"
102
+ OWNER_FENCED = "owner_fenced"
103
+ ALIAS_PROTECTED = "alias_protected"
104
+
105
+
106
+ @dataclass(frozen=True, slots=True)
107
+ class DurableBridgeAliasRegistrationReceipt:
108
+ """Rollback data for a continuity alias published before dispatch."""
109
+
110
+ status: DurableBridgeAliasRegistration
111
+ session_id: str
112
+ api_key_scope: str
113
+ alias_kind: str
114
+ alias_value: str
115
+ instance_id: str
116
+ owner_epoch: int
117
+ previous_alias_session_id: str | None
118
+ previous_alias_owner_epoch: int | None
119
+ previous_alias_account_id: str | None
120
+ previous_latest_turn_state: str | None
121
+
122
+
123
+ def durable_bridge_api_key_scope(api_key_id: str | None) -> str:
124
+ if api_key_id is None:
125
+ return _ANONYMOUS_API_KEY_SCOPE
126
+ stripped = api_key_id.strip()
127
+ return stripped or _ANONYMOUS_API_KEY_SCOPE
128
+
129
+
130
+ def durable_bridge_hash(value: str) -> str:
131
+ return sha256(value.encode("utf-8")).hexdigest()
132
+
133
+
134
+ def durable_bridge_operation_fingerprint(*, api_key_scope: str, request_text: str) -> str:
135
+ """Hash the logical turn together with its authorization namespace."""
136
+ return durable_bridge_hash(f"{api_key_scope}:{request_text}")
137
+
138
+
139
+ def durable_bridge_operation_id(session_id: str, request_fingerprint: str) -> str:
140
+ """Derive a stable, non-secret operation key for a continuity-bound turn."""
141
+ return f"op_{durable_bridge_hash(f'{session_id}:{request_fingerprint}')[:64]}"
142
+
143
+
144
+ def _encode_pending_tool_calls(response_id: str, value: Mapping[str, str] | None) -> str | None:
145
+ if value is None:
146
+ return None
147
+ return json.dumps(
148
+ {"response_id": response_id, "calls": dict(sorted(value.items()))},
149
+ separators=(",", ":"),
150
+ )
151
+
152
+
153
+ def _decode_pending_tool_calls(response_id: str | None, value: str | None) -> dict[str, str] | None:
154
+ if response_id is None or value is None:
155
+ return None
156
+ try:
157
+ payload = json.loads(value)
158
+ except (TypeError, ValueError):
159
+ return None
160
+ if not isinstance(payload, dict) or payload.get("response_id") != response_id:
161
+ return None
162
+ calls = payload.get("calls")
163
+ if not isinstance(calls, dict):
164
+ return None
165
+ result: dict[str, str] = {}
166
+ for call_id, call_type in calls.items():
167
+ if not isinstance(call_id, str) or not call_id.strip():
168
+ return None
169
+ if not isinstance(call_type, str) or not call_type.strip():
170
+ return None
171
+ result[call_id] = call_type
172
+ return result
173
+
174
+
175
+ @dataclass(frozen=True, slots=True)
176
+ class DurableBridgeSessionSnapshot:
177
+ id: str
178
+ session_key_kind: str
179
+ session_key_value: str
180
+ session_key_hash: str
181
+ api_key_scope: str
182
+ owner_instance_id: str | None
183
+ owner_epoch: int
184
+ lease_expires_at: datetime | None
185
+ state: HttpBridgeSessionState
186
+ account_id: str | None
187
+ model: str | None
188
+ service_tier: str | None
189
+ latest_turn_state: str | None
190
+ latest_response_id: str | None
191
+ latest_input_item_count: int | None
192
+ latest_input_full_fingerprint: str | None
193
+ last_seen_at: datetime
194
+ closed_at: datetime | None
195
+ latest_pending_tool_calls: dict[str, str] | None = None
196
+ owner_process_epoch: str | None = None
197
+ # True once a writer retired this row's continuity owner. Readers must
198
+ # treat ``account_id`` as absent and may use ``abandoned_account_id`` only
199
+ # as exclusion evidence, never as a routing target.
200
+ continuity_abandoned: bool = False
201
+ abandoned_account_id: str | None = None
202
+
203
+
204
+ @dataclass(frozen=True, slots=True)
205
+ class DurableBridgeRetryCircuitSnapshot:
206
+ session_key_kind: str
207
+ session_key_hash: str
208
+ api_key_scope: str
209
+ consecutive_failures: int
210
+ cooldown_until_epoch: float
211
+ last_detail: str | None
212
+ updated_at_epoch: float
213
+ admission_generation: int
214
+
215
+
216
+ @dataclass(frozen=True, slots=True)
217
+ class DurableBridgeRecoveryAttemptSnapshot:
218
+ session_id: str
219
+ request_fingerprint: str
220
+ request_id: str
221
+ account_id: str | None
222
+ model: str | None
223
+ replay_safe: bool
224
+ state: HttpBridgeRecoveryAttemptState
225
+ response_id: str | None
226
+
227
+
228
+ @dataclass(frozen=True, slots=True)
229
+ class DurableBridgeOperationSnapshot:
230
+ operation_id: str
231
+ session_id: str
232
+ request_fingerprint: str
233
+ account_id: str | None
234
+ model: str | None
235
+ parent_response_id: str | None
236
+ state: str
237
+ response_id: str | None
238
+ recovery_dispatch_count: int = 0
239
+ request_text: str | None = None
240
+ event_spool_complete: bool = True
241
+ created: bool = False
242
+ rebound: bool = False
243
+ rebound_from_session_id: str | None = None
244
+ rebound_from_account_id: str | None = None
245
+ rebound_from_model: str | None = None
246
+ rebound_from_parent_response_id: str | None = None
247
+
248
+
249
+ @dataclass(frozen=True, slots=True)
250
+ class DurableBridgeOperationAbandonment:
251
+ """Low-cardinality evidence for one operation abandoned by maintenance."""
252
+
253
+ source_state: str
254
+ age_seconds: float
255
+ owner_lease_outcome: str
256
+ session_hash: str
257
+
258
+
259
+ @dataclass(frozen=True, slots=True)
260
+ class DurableBridgeOperationAbandonmentScanCursor:
261
+ """Keyset position for the next oversized-protection maintenance slice."""
262
+
263
+ updated_at: datetime
264
+ operation_id: str
265
+
266
+
267
+ @dataclass(frozen=True, slots=True)
268
+ class DurableBridgeOperationAbandonmentSweep:
269
+ """Abandonments plus the cursor needed to continue a bounded sweep."""
270
+
271
+ abandonments: tuple[DurableBridgeOperationAbandonment, ...]
272
+ next_cursor: DurableBridgeOperationAbandonmentScanCursor | None
273
+
274
+
275
+ @dataclass(frozen=True, slots=True)
276
+ class DurableBridgeTranscriptTurn:
277
+ operation: DurableBridgeOperationSnapshot
278
+ events: tuple[str, ...]
279
+
280
+
281
+ @dataclass(frozen=True, slots=True)
282
+ class DurableBridgeOperationEventInput:
283
+ operation_id: str
284
+ session_id: str
285
+ instance_id: str
286
+ owner_epoch: int
287
+ event_text: str
288
+
289
+
290
+ @dataclass(frozen=True, slots=True)
291
+ class DurableBridgeOperationPurgeBatchResult:
292
+ selected_operations: int
293
+ deleted_operations: int
294
+
295
+
296
+ class DurableBridgeRepository:
297
+ def __init__(self, session: AsyncSession) -> None:
298
+ self._session = session
299
+
300
+ async def _commit_writer_section(self) -> None:
301
+ async with sqlite_writer_section():
302
+ await self._session.commit()
303
+
304
+ async def _delete_operation_spool_material(self, operation_ids: Sequence[str]) -> None:
305
+ if not operation_ids:
306
+ return
307
+ await self._session.execute(
308
+ delete(HttpBridgeOperationEvent).where(HttpBridgeOperationEvent.operation_id.in_(operation_ids))
309
+ )
310
+ await self._session.execute(
311
+ delete(HttpBridgeOperationEventChunk).where(HttpBridgeOperationEventChunk.operation_id.in_(operation_ids))
312
+ )
313
+
314
+ async def _operation_has_spool_material(self, operation_id: str) -> bool:
315
+ legacy_event = await self._session.scalar(
316
+ select(HttpBridgeOperationEvent.event_id)
317
+ .where(HttpBridgeOperationEvent.operation_id == operation_id)
318
+ .limit(1)
319
+ )
320
+ if legacy_event is not None:
321
+ return True
322
+ chunk_sequence = await self._session.scalar(
323
+ select(HttpBridgeOperationEventChunk.first_sequence_number)
324
+ .where(HttpBridgeOperationEventChunk.operation_id == operation_id)
325
+ .limit(1)
326
+ )
327
+ return chunk_sequence is not None
328
+
329
+ async def _operation_has_legacy_events(self, operation_id: str) -> bool:
330
+ event_id = await self._session.scalar(
331
+ select(HttpBridgeOperationEvent.event_id)
332
+ .where(HttpBridgeOperationEvent.operation_id == operation_id)
333
+ .limit(1)
334
+ )
335
+ return event_id is not None
336
+
337
+ async def _lock_operation_for_chunk_append(
338
+ self,
339
+ *,
340
+ operation_id: str,
341
+ session_id: str,
342
+ instance_id: str,
343
+ owner_epoch: int,
344
+ expected_recovery_dispatch_count: int | None = None,
345
+ ) -> tuple[HttpBridgeOperationRecord, bool] | None:
346
+ owner_exists = await self._session.scalar(
347
+ select(HttpBridgeSessionRecord.id)
348
+ .where(
349
+ HttpBridgeSessionRecord.id == session_id,
350
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
351
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
352
+ )
353
+ .with_for_update()
354
+ )
355
+ operation_statement = select(HttpBridgeOperationRecord).where(
356
+ HttpBridgeOperationRecord.operation_id == operation_id,
357
+ HttpBridgeOperationRecord.session_id == session_id,
358
+ )
359
+ if expected_recovery_dispatch_count is not None:
360
+ operation_statement = operation_statement.where(
361
+ HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count
362
+ )
363
+ operation = await self._session.scalar(operation_statement.with_for_update())
364
+ # ``abandoned`` is a terminal duplicate-suppression fence that the
365
+ # maintenance sweep applies without clearing session ownership, so the
366
+ # original owner still passes the instance/epoch fence above. Refuse
367
+ # the lock like the rows_v1 writers do; otherwise a late terminal chunk
368
+ # rewrites ``state`` and a late batch grows the abandoned spool.
369
+ if owner_exists is None or operation is None or operation.state == "abandoned":
370
+ await self._session.rollback()
371
+ return None
372
+ # This dispatch already recorded a terminal transcript outcome (an
373
+ # append committed, or a fallback settlement published it), so the row
374
+ # is final: refuse the lock rather than let a duplicate or late
375
+ # terminal write rewrite it. The predicate is the dedicated phase
376
+ # marker, never ``event_spool_complete`` — an ordinary operation is
377
+ # ``state = completed`` with an incomplete spool for the whole window
378
+ # in which its terminal append runs, because the relay publishes the
379
+ # operation state before appending.
380
+ if operation.terminal_append_phase in _HTTP_BRIDGE_TERMINAL_APPEND_RECORDED_PHASES:
381
+ return operation, False
382
+ if operation.spool_format == HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2:
383
+ if await self._operation_has_legacy_events(operation_id):
384
+ return operation, False
385
+ return operation, True
386
+ if operation.spool_format != HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1:
387
+ return operation, False
388
+ if int(operation.event_bytes or 0) != 0 or await self._operation_has_spool_material(operation_id):
389
+ return operation, False
390
+ return operation, True
391
+
392
+ async def _next_operation_chunk_sequence(self, operation_id: str) -> int:
393
+ latest = (
394
+ await self._session.execute(
395
+ select(
396
+ HttpBridgeOperationEventChunk.first_sequence_number,
397
+ HttpBridgeOperationEventChunk.event_count,
398
+ )
399
+ .where(HttpBridgeOperationEventChunk.operation_id == operation_id)
400
+ .order_by(HttpBridgeOperationEventChunk.first_sequence_number.desc())
401
+ .limit(1)
402
+ )
403
+ ).one_or_none()
404
+ if latest is None:
405
+ return 1
406
+ first_sequence_number, event_count = latest
407
+ return int(first_sequence_number) + int(event_count)
408
+
409
+ async def _chunk_append_owner_exists(
410
+ self,
411
+ *,
412
+ session_id: str,
413
+ instance_id: str,
414
+ owner_epoch: int,
415
+ ) -> bool:
416
+ return (
417
+ await self._session.scalar(
418
+ select(HttpBridgeSessionRecord.id).where(
419
+ HttpBridgeSessionRecord.id == session_id,
420
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
421
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
422
+ )
423
+ )
424
+ ) is not None
425
+
426
+ async def _chunk_append_preflight_allows_encoding(
427
+ self,
428
+ *,
429
+ operation_id: str,
430
+ session_id: str,
431
+ instance_id: str,
432
+ owner_epoch: int,
433
+ event_bytes: int,
434
+ event_count: int,
435
+ max_bytes: int,
436
+ ) -> bool:
437
+ if not await self._chunk_append_owner_exists(
438
+ session_id=session_id,
439
+ instance_id=instance_id,
440
+ owner_epoch=owner_epoch,
441
+ ):
442
+ return False
443
+ current_event_bytes = await self._session.scalar(
444
+ select(HttpBridgeOperationRecord.event_bytes).where(
445
+ HttpBridgeOperationRecord.operation_id == operation_id,
446
+ HttpBridgeOperationRecord.session_id == session_id,
447
+ )
448
+ )
449
+ if current_event_bytes is None:
450
+ return False
451
+ first_sequence_number = await self._next_operation_chunk_sequence(operation_id)
452
+ return (
453
+ int(current_event_bytes) + event_bytes <= max_bytes
454
+ and first_sequence_number - 1 + event_count <= DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS
455
+ )
456
+
457
+ async def get_session(
458
+ self,
459
+ *,
460
+ session_key_kind: str,
461
+ session_key_value: str,
462
+ api_key_scope: str,
463
+ ) -> DurableBridgeSessionSnapshot | None:
464
+ statement = select(HttpBridgeSessionRecord).where(
465
+ HttpBridgeSessionRecord.session_key_kind == session_key_kind,
466
+ HttpBridgeSessionRecord.session_key_hash == durable_bridge_hash(session_key_value),
467
+ HttpBridgeSessionRecord.api_key_scope == api_key_scope,
468
+ )
469
+ result = await self._session.execute(statement)
470
+ row = result.scalar_one_or_none()
471
+ return _to_snapshot(row)
472
+
473
+ async def get_retry_circuit(
474
+ self,
475
+ *,
476
+ session_key_kind: str,
477
+ session_key_value: str,
478
+ api_key_scope: str,
479
+ ) -> DurableBridgeRetryCircuitSnapshot | None:
480
+ result = await self._session.execute(
481
+ select(HttpBridgeRetryCircuit).where(
482
+ HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
483
+ HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
484
+ HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
485
+ )
486
+ )
487
+ return _to_retry_circuit_snapshot(result.scalar_one_or_none())
488
+
489
+ async def upsert_retry_circuit(
490
+ self,
491
+ *,
492
+ session_key_kind: str,
493
+ session_key_value: str,
494
+ api_key_scope: str,
495
+ consecutive_failures: int,
496
+ cooldown_until_epoch: float,
497
+ last_detail: str | None,
498
+ updated_at_epoch: float,
499
+ base_updated_at_epoch: float = 0.0,
500
+ failure_threshold: int = 1,
501
+ poison_sticky_threshold: int | None = None,
502
+ conflict_cooldown_until_epoch: float | None = None,
503
+ base_backoff_seconds: float = 60.0,
504
+ max_backoff_seconds: float = 600.0,
505
+ clean_close_max_backoff_seconds: float = 30.0,
506
+ ) -> None:
507
+ sticky_threshold = poison_sticky_threshold if poison_sticky_threshold is not None else failure_threshold
508
+ values = {
509
+ "session_key_kind": session_key_kind,
510
+ "session_key_hash": durable_bridge_hash(session_key_value),
511
+ "api_key_scope": api_key_scope,
512
+ "consecutive_failures": consecutive_failures,
513
+ "cooldown_until_epoch": cooldown_until_epoch,
514
+ "last_detail": last_detail,
515
+ "updated_at_epoch": updated_at_epoch,
516
+ }
517
+ threshold = max(1, failure_threshold)
518
+ cooldown_floor = (
519
+ max(0.0, conflict_cooldown_until_epoch)
520
+ if conflict_cooldown_until_epoch is not None
521
+ else max(0.0, cooldown_until_epoch)
522
+ )
523
+ # A reset starts a new failure lineage. Never carry the incoming
524
+ # cooldown into that fresh lineage, even when the threshold is one.
525
+ base_backoff = max(0.001, base_backoff_seconds)
526
+ max_backoff = max(base_backoff, max_backoff_seconds)
527
+ clean_close_max_backoff = max(0.001, clean_close_max_backoff_seconds)
528
+
529
+ def cooldown_for_failure_count(failure_count: Any, last_detail: Any) -> Any:
530
+ regular_cooldown = case(
531
+ (failure_count < threshold, 0.0),
532
+ (failure_count == threshold, base_backoff),
533
+ (failure_count == threshold + 1, min(max_backoff, base_backoff * 2.0)),
534
+ (failure_count == threshold + 2, min(max_backoff, base_backoff * 4.0)),
535
+ else_=max_backoff,
536
+ )
537
+ clean_cooldown = case(
538
+ (failure_count < threshold, 0.0),
539
+ else_=clean_close_max_backoff,
540
+ )
541
+ return case((last_detail == "clean_close", clean_cooldown), else_=regular_cooldown)
542
+
543
+ dialect = self._session.get_bind().dialect.name
544
+ if dialect == "postgresql":
545
+ insert_statement = pg_insert(HttpBridgeRetryCircuit).values(**values)
546
+ excluded = insert_statement.excluded
547
+ # ``updated_at_epoch`` is an observation timestamp, not a
548
+ # concurrency version. Equality with the loaded base is the CAS
549
+ # match, even when a replica's wall clock lags it; the failure
550
+ # count guard still rejects an older snapshot that was loaded
551
+ # from the same row after a newer failure had already been
552
+ # merged. Every base-mismatched write is stale — its episode was
553
+ # settled, replaced, or outrun while the write waited — and
554
+ # drops, leaving the row byte-identical so in-flight fences and
555
+ # bases stay valid; the writer reconciles from the returned row.
556
+ # Wall-clock recency cannot stand in for lineage: a delayed
557
+ # write always carries a newer timestamp than the base it
558
+ # loaded, so admitting "newer than base" writes would merge a
559
+ # finished episode's count into whatever lineage owns the row
560
+ # now, including a reset row another replica has already
561
+ # re-struck. A concurrent same-lineage strike that loses this
562
+ # race undercounts by exactly its own strike, which is the
563
+ # accepted direction: the circuit opens at most one failure
564
+ # later, and a stale count can never reopen a settled episode's
565
+ # cooldown against a fresh lineage.
566
+ failure_from_current_row = and_(
567
+ HttpBridgeRetryCircuit.updated_at_epoch == base_updated_at_epoch,
568
+ excluded.consecutive_failures >= HttpBridgeRetryCircuit.consecutive_failures,
569
+ )
570
+ # An at-threshold poison detail is sticky against non-poison
571
+ # strikes: the row is the cross-replica record that a poisoned
572
+ # anchor's clear is still owed, and letting a clean probe
573
+ # failure overwrite it would strand that debt for every worker.
574
+ # Only a reset/settle (which rewrites the row wholesale) or
575
+ # another poison-class strike may change it.
576
+ sticky_poison_detail = and_(
577
+ HttpBridgeRetryCircuit.last_detail.in_(
578
+ ("stream_incomplete", "stream_idle_timeout", "bridge_eventless_timeout")
579
+ ),
580
+ # The effective anchor-poison threshold, not the circuit
581
+ # threshold: a configured threshold of one authorizes the
582
+ # abandonment at the first poison strike, and its failed
583
+ # clear leaves a one-failure debt this predicate must keep.
584
+ HttpBridgeRetryCircuit.consecutive_failures >= sticky_threshold,
585
+ excluded.last_detail.notin_(("stream_incomplete", "stream_idle_timeout", "bridge_eventless_timeout")),
586
+ )
587
+ # An abandonment tombstone is sticky against every strike
588
+ # detail: continuity is already gone, the tombstone is what
589
+ # fails anchorless deltas closed on every replica, and only the
590
+ # fenced settle/supersede paths — a completion establishing
591
+ # fresh continuity — may rewrite it.
592
+ # NULL-safe: a NULL detail must make this term FALSE (so its
593
+ # negation stays TRUE), not NULL — three-valued logic would
594
+ # otherwise freeze the detail of every reset row.
595
+ sticky_tombstone = and_(
596
+ HttpBridgeRetryCircuit.last_detail.is_not(None),
597
+ HttpBridgeRetryCircuit.last_detail == _RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL,
598
+ )
599
+ conflict_failures = case(
600
+ (
601
+ failure_from_current_row,
602
+ func.greatest(
603
+ HttpBridgeRetryCircuit.consecutive_failures + 1,
604
+ excluded.consecutive_failures,
605
+ ),
606
+ ),
607
+ else_=HttpBridgeRetryCircuit.consecutive_failures,
608
+ )
609
+ merged_updated_at = func.greatest(
610
+ HttpBridgeRetryCircuit.updated_at_epoch,
611
+ excluded.updated_at_epoch,
612
+ )
613
+ merged_cooldown = case(
614
+ (
615
+ conflict_failures >= threshold,
616
+ func.greatest(
617
+ cooldown_floor,
618
+ merged_updated_at + cooldown_for_failure_count(conflict_failures, excluded.last_detail),
619
+ ),
620
+ ),
621
+ else_=0.0,
622
+ )
623
+ statement = insert_statement.on_conflict_do_update(
624
+ index_elements=[
625
+ HttpBridgeRetryCircuit.session_key_kind,
626
+ HttpBridgeRetryCircuit.session_key_hash,
627
+ HttpBridgeRetryCircuit.api_key_scope,
628
+ ],
629
+ set_={
630
+ "consecutive_failures": conflict_failures,
631
+ "cooldown_until_epoch": case(
632
+ (
633
+ failure_from_current_row,
634
+ func.greatest(
635
+ HttpBridgeRetryCircuit.cooldown_until_epoch,
636
+ excluded.cooldown_until_epoch,
637
+ merged_cooldown,
638
+ ),
639
+ ),
640
+ else_=HttpBridgeRetryCircuit.cooldown_until_epoch,
641
+ ),
642
+ "last_detail": case(
643
+ (
644
+ and_(failure_from_current_row, ~sticky_poison_detail, ~sticky_tombstone),
645
+ excluded.last_detail,
646
+ ),
647
+ else_=HttpBridgeRetryCircuit.last_detail,
648
+ ),
649
+ "updated_at_epoch": case(
650
+ (failure_from_current_row, merged_updated_at),
651
+ else_=HttpBridgeRetryCircuit.updated_at_epoch,
652
+ ),
653
+ },
654
+ )
655
+ elif dialect == "sqlite":
656
+ insert_statement = sqlite_insert(HttpBridgeRetryCircuit).values(**values)
657
+ excluded = insert_statement.excluded
658
+ # ``updated_at_epoch`` is an observation timestamp, not a
659
+ # concurrency version. Equality with the loaded base is the CAS
660
+ # match, even when a replica's wall clock lags it; the failure
661
+ # count guard still rejects an older snapshot that was loaded
662
+ # from the same row after a newer failure had already been
663
+ # merged. Every base-mismatched write is stale — its episode was
664
+ # settled, replaced, or outrun while the write waited — and
665
+ # drops, leaving the row byte-identical so in-flight fences and
666
+ # bases stay valid; the writer reconciles from the returned row.
667
+ # Wall-clock recency cannot stand in for lineage: a delayed
668
+ # write always carries a newer timestamp than the base it
669
+ # loaded, so admitting "newer than base" writes would merge a
670
+ # finished episode's count into whatever lineage owns the row
671
+ # now, including a reset row another replica has already
672
+ # re-struck. A concurrent same-lineage strike that loses this
673
+ # race undercounts by exactly its own strike, which is the
674
+ # accepted direction: the circuit opens at most one failure
675
+ # later, and a stale count can never reopen a settled episode's
676
+ # cooldown against a fresh lineage.
677
+ failure_from_current_row = and_(
678
+ HttpBridgeRetryCircuit.updated_at_epoch == base_updated_at_epoch,
679
+ excluded.consecutive_failures >= HttpBridgeRetryCircuit.consecutive_failures,
680
+ )
681
+ # An at-threshold poison detail is sticky against non-poison
682
+ # strikes: the row is the cross-replica record that a poisoned
683
+ # anchor's clear is still owed, and letting a clean probe
684
+ # failure overwrite it would strand that debt for every worker.
685
+ # Only a reset/settle (which rewrites the row wholesale) or
686
+ # another poison-class strike may change it.
687
+ sticky_poison_detail = and_(
688
+ HttpBridgeRetryCircuit.last_detail.in_(
689
+ ("stream_incomplete", "stream_idle_timeout", "bridge_eventless_timeout")
690
+ ),
691
+ # The effective anchor-poison threshold, not the circuit
692
+ # threshold: a configured threshold of one authorizes the
693
+ # abandonment at the first poison strike, and its failed
694
+ # clear leaves a one-failure debt this predicate must keep.
695
+ HttpBridgeRetryCircuit.consecutive_failures >= sticky_threshold,
696
+ excluded.last_detail.notin_(("stream_incomplete", "stream_idle_timeout", "bridge_eventless_timeout")),
697
+ )
698
+ # An abandonment tombstone is sticky against every strike
699
+ # detail: continuity is already gone, the tombstone is what
700
+ # fails anchorless deltas closed on every replica, and only the
701
+ # fenced settle/supersede paths — a completion establishing
702
+ # fresh continuity — may rewrite it.
703
+ # NULL-safe: a NULL detail must make this term FALSE (so its
704
+ # negation stays TRUE), not NULL — three-valued logic would
705
+ # otherwise freeze the detail of every reset row.
706
+ sticky_tombstone = and_(
707
+ HttpBridgeRetryCircuit.last_detail.is_not(None),
708
+ HttpBridgeRetryCircuit.last_detail == _RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL,
709
+ )
710
+ conflict_failures = case(
711
+ (
712
+ failure_from_current_row,
713
+ func.max(
714
+ HttpBridgeRetryCircuit.consecutive_failures + 1,
715
+ excluded.consecutive_failures,
716
+ ),
717
+ ),
718
+ else_=HttpBridgeRetryCircuit.consecutive_failures,
719
+ )
720
+ merged_updated_at = func.max(
721
+ HttpBridgeRetryCircuit.updated_at_epoch,
722
+ excluded.updated_at_epoch,
723
+ )
724
+ merged_cooldown = case(
725
+ (
726
+ conflict_failures >= threshold,
727
+ func.max(
728
+ cooldown_floor,
729
+ merged_updated_at + cooldown_for_failure_count(conflict_failures, excluded.last_detail),
730
+ ),
731
+ ),
732
+ else_=0.0,
733
+ )
734
+ statement = insert_statement.on_conflict_do_update(
735
+ index_elements=[
736
+ HttpBridgeRetryCircuit.session_key_kind,
737
+ HttpBridgeRetryCircuit.session_key_hash,
738
+ HttpBridgeRetryCircuit.api_key_scope,
739
+ ],
740
+ set_={
741
+ "consecutive_failures": conflict_failures,
742
+ "cooldown_until_epoch": case(
743
+ (
744
+ failure_from_current_row,
745
+ func.max(
746
+ HttpBridgeRetryCircuit.cooldown_until_epoch,
747
+ excluded.cooldown_until_epoch,
748
+ merged_cooldown,
749
+ ),
750
+ ),
751
+ else_=HttpBridgeRetryCircuit.cooldown_until_epoch,
752
+ ),
753
+ "last_detail": case(
754
+ (
755
+ and_(failure_from_current_row, ~sticky_poison_detail, ~sticky_tombstone),
756
+ excluded.last_detail,
757
+ ),
758
+ else_=HttpBridgeRetryCircuit.last_detail,
759
+ ),
760
+ "updated_at_epoch": case(
761
+ (failure_from_current_row, merged_updated_at),
762
+ else_=HttpBridgeRetryCircuit.updated_at_epoch,
763
+ ),
764
+ },
765
+ )
766
+ else:
767
+ raise RuntimeError(f"DurableBridgeRepository retry circuit upsert unsupported for dialect={dialect!r}")
768
+ async with sqlite_writer_section():
769
+ await self._session.execute(statement)
770
+ await self._session.commit()
771
+
772
+ async def claim_retry_circuit_generation(
773
+ self,
774
+ *,
775
+ session_key_kind: str,
776
+ session_key_value: str,
777
+ api_key_scope: str,
778
+ expected_updated_at_epoch: float | None,
779
+ expected_admission_generation: int,
780
+ expected_consecutive_failures: int,
781
+ expected_cooldown_until_epoch: float,
782
+ ) -> DurableBridgeRetryCircuitSnapshot | None:
783
+ """Linearize replay admission against a retry-circuit generation.
784
+
785
+ ``admission_generation`` is independent from the failure observation
786
+ timestamp. A failure committed first makes this claim fail; a delayed
787
+ failure committed afterward still sees its original ``updated_at``
788
+ baseline and is merged after the already-admitted dispatch.
789
+ """
790
+ values = {
791
+ "session_key_kind": session_key_kind,
792
+ "session_key_hash": durable_bridge_hash(session_key_value),
793
+ "api_key_scope": api_key_scope,
794
+ "consecutive_failures": 0,
795
+ "cooldown_until_epoch": 0.0,
796
+ "last_detail": None,
797
+ "updated_at_epoch": time.time(),
798
+ "admission_generation": 1,
799
+ }
800
+ dialect = self._session.get_bind().dialect.name
801
+ async with sqlite_writer_section():
802
+ if expected_updated_at_epoch is None:
803
+ if expected_admission_generation != 0:
804
+ return None
805
+ if dialect == "postgresql":
806
+ statement = pg_insert(HttpBridgeRetryCircuit).values(**values).on_conflict_do_nothing()
807
+ elif dialect == "sqlite":
808
+ statement = sqlite_insert(HttpBridgeRetryCircuit).values(**values).on_conflict_do_nothing()
809
+ else:
810
+ raise RuntimeError(
811
+ f"DurableBridgeRepository retry circuit claim unsupported for dialect={dialect!r}"
812
+ )
813
+ else:
814
+ statement = (
815
+ update(HttpBridgeRetryCircuit)
816
+ .where(
817
+ HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
818
+ HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
819
+ HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
820
+ HttpBridgeRetryCircuit.admission_generation == expected_admission_generation,
821
+ HttpBridgeRetryCircuit.updated_at_epoch == expected_updated_at_epoch,
822
+ HttpBridgeRetryCircuit.consecutive_failures == expected_consecutive_failures,
823
+ HttpBridgeRetryCircuit.cooldown_until_epoch == expected_cooldown_until_epoch,
824
+ )
825
+ .values(admission_generation=expected_admission_generation + 1)
826
+ )
827
+ result = await self._session.execute(statement)
828
+ if getattr(result, "rowcount", 0) != 1:
829
+ await self._session.rollback()
830
+ return None
831
+ await self._session.commit()
832
+ return await self.get_retry_circuit(
833
+ session_key_kind=session_key_kind,
834
+ session_key_value=session_key_value,
835
+ api_key_scope=api_key_scope,
836
+ )
837
+
838
+ async def supersede_retry_circuit_detail(
839
+ self,
840
+ *,
841
+ session_key_kind: str,
842
+ session_key_value: str,
843
+ api_key_scope: str,
844
+ expected_updated_at_epoch: float,
845
+ expected_consecutive_failures: int,
846
+ expected_last_detail: str | None,
847
+ last_detail: str | None,
848
+ ) -> bool:
849
+ """Rewrite a surviving row's failure detail under its version fence.
850
+
851
+ Detail-only by design: the strike upsert increments the merged count,
852
+ so persisting an anchor supersession through it would charge a
853
+ phantom failure. The version is deliberately left unchanged so a
854
+ concurrent strike still merges onto the row and its failure class
855
+ overwrites this one. The count is part of the fence because a
856
+ lagging-clock strike can merge without moving the version — every
857
+ landed merge increments the count, so a strike that slipped in
858
+ ahead of this rewrite makes it miss instead of being overwritten.
859
+ """
860
+ async with sqlite_writer_section():
861
+ result = await self._session.execute(
862
+ update(HttpBridgeRetryCircuit)
863
+ .where(
864
+ HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
865
+ HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
866
+ HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
867
+ HttpBridgeRetryCircuit.updated_at_epoch == expected_updated_at_epoch,
868
+ HttpBridgeRetryCircuit.consecutive_failures == expected_consecutive_failures,
869
+ # The prior detail is part of the fence: two completions
870
+ # can otherwise both believe they own the supersession of
871
+ # one shared row, and the loser's rollback would destroy
872
+ # the winner's — re-poisoning a freshly registered
873
+ # anchor. With this fence exactly one write owns each
874
+ # transition, forward and back.
875
+ HttpBridgeRetryCircuit.last_detail == expected_last_detail,
876
+ )
877
+ .values(last_detail=last_detail)
878
+ )
879
+ await self._session.commit()
880
+ return bool(getattr(result, "rowcount", 0))
881
+
882
+ async def delete_retry_circuit(
883
+ self,
884
+ *,
885
+ session_key_kind: str,
886
+ session_key_value: str,
887
+ api_key_scope: str,
888
+ expected_updated_at_epoch: float | None = None,
889
+ expected_admission_generation: int | None = None,
890
+ expected_consecutive_failures: int | None = None,
891
+ reset_detail: str | None = None,
892
+ ) -> bool:
893
+ conditions = [
894
+ HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
895
+ HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
896
+ HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
897
+ ]
898
+ if expected_consecutive_failures is not None:
899
+ # A lagging-clock strike merges a higher count without moving
900
+ # the epoch; the count keeps this reset from zeroing a newer
901
+ # episode's cooldown the caller never observed.
902
+ conditions.append(HttpBridgeRetryCircuit.consecutive_failures == expected_consecutive_failures)
903
+ if expected_updated_at_epoch is not None:
904
+ conditions.append(HttpBridgeRetryCircuit.updated_at_epoch == expected_updated_at_epoch)
905
+ if expected_admission_generation is not None:
906
+ # A replay claim bumps only ``admission_generation``, leaving
907
+ # the failure-observation epoch untouched; without this fence
908
+ # a reset authorized before the claim would clear the circuit
909
+ # beneath the admitted replay that depends on it.
910
+ conditions.append(HttpBridgeRetryCircuit.admission_generation == expected_admission_generation)
911
+ async with sqlite_writer_section():
912
+ result = await self._session.execute(
913
+ update(HttpBridgeRetryCircuit)
914
+ .where(*conditions)
915
+ .values(
916
+ consecutive_failures=0,
917
+ cooldown_until_epoch=0.0,
918
+ # An abandonment-driven settle leaves a durable tombstone
919
+ # so restarted workers and other replicas fail
920
+ # delta-only follow-ups closed; a completion's settle
921
+ # writes None, erasing it on real recovery.
922
+ last_detail=reset_detail,
923
+ updated_at_epoch=time.time(),
924
+ )
925
+ )
926
+ await self._session.commit()
927
+ # A fenced reset that matched no row is a CAS miss, not a settlement:
928
+ # another writer moved the row after the caller's lookup.
929
+ return bool(getattr(result, "rowcount", 0))
930
+
931
+ async def purge_retry_circuit(
932
+ self,
933
+ *,
934
+ session_key_kind: str,
935
+ session_key_value: str,
936
+ api_key_scope: str,
937
+ expected_updated_at_epoch: float | None = None,
938
+ expected_admission_generation: int | None = None,
939
+ expected_consecutive_failures: int | None = None,
940
+ fence_last_detail: bool = False,
941
+ expected_last_detail: str | None = None,
942
+ reset_detail: str | None = None,
943
+ ) -> bool:
944
+ conditions = [
945
+ HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
946
+ HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
947
+ HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
948
+ ]
949
+ if expected_consecutive_failures is not None:
950
+ conditions.append(HttpBridgeRetryCircuit.consecutive_failures == expected_consecutive_failures)
951
+ if fence_last_detail:
952
+ # A detail-only rewrite — the transitional tombstone supersede —
953
+ # deliberately moves neither the timestamp nor the admission
954
+ # generation, so the observed detail is the only fence that can
955
+ # see it. NULL-safe: an expected NULL matches only NULL.
956
+ conditions.append(
957
+ HttpBridgeRetryCircuit.last_detail.is_(None)
958
+ if expected_last_detail is None
959
+ else HttpBridgeRetryCircuit.last_detail == expected_last_detail
960
+ )
961
+ if expected_updated_at_epoch is not None:
962
+ conditions.append(HttpBridgeRetryCircuit.updated_at_epoch == expected_updated_at_epoch)
963
+ if expected_admission_generation is not None:
964
+ # A replay claim advances only the admission generation and
965
+ # leaves the timestamp untouched; without this fence a
966
+ # stale-row purge racing the claim would delete the active
967
+ # claimed generation and a later recovery could dispatch a
968
+ # second replay beside the first.
969
+ conditions.append(HttpBridgeRetryCircuit.admission_generation == expected_admission_generation)
970
+ async with sqlite_writer_section():
971
+ result = await self._session.execute(delete(HttpBridgeRetryCircuit).where(*conditions))
972
+ await self._session.commit()
973
+ # A fenced purge that matched nothing is a CAS miss: another replica
974
+ # moved the row after this worker's lookup, and the caller must
975
+ # reconcile against the surviving row instead of assuming deletion.
976
+ return bool(getattr(result, "rowcount", 0)) or expected_updated_at_epoch is None
977
+
978
+ async def get_session_by_id(self, session_id: str) -> DurableBridgeSessionSnapshot | None:
979
+ row = await self._session.get(HttpBridgeSessionRecord, session_id)
980
+ return _to_snapshot(row)
981
+
982
+ async def resolve_alias(
983
+ self,
984
+ *,
985
+ alias_kind: str,
986
+ alias_value: str,
987
+ api_key_scope: str,
988
+ ) -> DurableBridgeSessionSnapshot | None:
989
+ statement = (
990
+ select(HttpBridgeSessionRecord)
991
+ .join(HttpBridgeSessionAlias, HttpBridgeSessionAlias.session_id == HttpBridgeSessionRecord.id)
992
+ .where(
993
+ HttpBridgeSessionAlias.alias_kind == alias_kind,
994
+ HttpBridgeSessionAlias.alias_hash == durable_bridge_hash(alias_value),
995
+ HttpBridgeSessionAlias.api_key_scope == api_key_scope,
996
+ )
997
+ .limit(1)
998
+ )
999
+ result = await self._session.execute(statement)
1000
+ row = result.scalar_one_or_none()
1001
+ return _to_snapshot(row)
1002
+
1003
+ async def find_session_by_latest_turn_state(
1004
+ self,
1005
+ *,
1006
+ turn_state: str,
1007
+ api_key_scope: str,
1008
+ ) -> DurableBridgeSessionSnapshot | None:
1009
+ statement = (
1010
+ select(HttpBridgeSessionRecord)
1011
+ .where(
1012
+ HttpBridgeSessionRecord.latest_turn_state == turn_state,
1013
+ HttpBridgeSessionRecord.api_key_scope == api_key_scope,
1014
+ HttpBridgeSessionRecord.state.in_((HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING)),
1015
+ )
1016
+ .order_by(
1017
+ case((HttpBridgeSessionRecord.state == HttpBridgeSessionState.ACTIVE, 0), else_=1),
1018
+ HttpBridgeSessionRecord.last_seen_at.desc(),
1019
+ HttpBridgeSessionRecord.updated_at.desc(),
1020
+ )
1021
+ .limit(1)
1022
+ )
1023
+ result = await self._session.execute(statement)
1024
+ row = result.scalar_one_or_none()
1025
+ return _to_snapshot(row)
1026
+
1027
+ async def find_session_by_latest_response_id(
1028
+ self,
1029
+ *,
1030
+ response_id: str,
1031
+ api_key_scope: str,
1032
+ ) -> DurableBridgeSessionSnapshot | None:
1033
+ statement = (
1034
+ select(HttpBridgeSessionRecord)
1035
+ .where(
1036
+ HttpBridgeSessionRecord.latest_response_id == response_id,
1037
+ HttpBridgeSessionRecord.api_key_scope == api_key_scope,
1038
+ HttpBridgeSessionRecord.state.in_((HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING)),
1039
+ )
1040
+ .order_by(
1041
+ case((HttpBridgeSessionRecord.state == HttpBridgeSessionState.ACTIVE, 0), else_=1),
1042
+ HttpBridgeSessionRecord.last_seen_at.desc(),
1043
+ HttpBridgeSessionRecord.updated_at.desc(),
1044
+ )
1045
+ .limit(1)
1046
+ )
1047
+ result = await self._session.execute(statement)
1048
+ row = result.scalar_one_or_none()
1049
+ return _to_snapshot(row)
1050
+
1051
+ async def claim_session(
1052
+ self,
1053
+ *,
1054
+ session_key_kind: str,
1055
+ session_key_value: str,
1056
+ api_key_scope: str,
1057
+ instance_id: str,
1058
+ lease_ttl_seconds: float,
1059
+ account_id: str | None,
1060
+ model: str | None,
1061
+ service_tier: str | None,
1062
+ latest_turn_state: str | None,
1063
+ latest_response_id: str | None,
1064
+ allow_takeover: bool,
1065
+ owner_process_epoch: str,
1066
+ force_owner_epoch_advance: bool = False,
1067
+ ) -> DurableBridgeSessionSnapshot:
1068
+ session_key_hash = durable_bridge_hash(session_key_value)
1069
+ # ``allow_takeover`` was decided by the caller against a pre-claim
1070
+ # lookup. Once another claimant has demonstrably written this row under
1071
+ # us (lost CAS, or lost insert race), that decision is stale: the row
1072
+ # we re-read may now carry the winner's live lease, and reusing the
1073
+ # permission would let the loser steal it. Revalidate from the fresh
1074
+ # read instead — a live foreign owner then fails closed exactly like a
1075
+ # non-takeover claim, which surfaces as the correct cross-replica
1076
+ # "retry to reach the correct replica" response.
1077
+ contended = False
1078
+ # Bounded retry budget shared by the insert race (IntegrityError) and
1079
+ # the epoch CAS: every round has a winner, so a loser converges after
1080
+ # at most one fresh read per concurrent claimant.
1081
+ for attempt in range(_CLAIM_CAS_ATTEMPTS):
1082
+ now = utcnow()
1083
+ lease_expires_at = now + timedelta(seconds=max(1.0, lease_ttl_seconds))
1084
+ row = await self._session.execute(
1085
+ select(HttpBridgeSessionRecord)
1086
+ .where(
1087
+ HttpBridgeSessionRecord.session_key_kind == session_key_kind,
1088
+ HttpBridgeSessionRecord.session_key_hash == session_key_hash,
1089
+ HttpBridgeSessionRecord.api_key_scope == api_key_scope,
1090
+ )
1091
+ .with_for_update()
1092
+ )
1093
+ existing = row.scalar_one_or_none()
1094
+ if existing is None:
1095
+ record = HttpBridgeSessionRecord(
1096
+ session_key_kind=session_key_kind,
1097
+ session_key_value=session_key_value,
1098
+ session_key_hash=session_key_hash,
1099
+ api_key_scope=api_key_scope,
1100
+ owner_instance_id=instance_id,
1101
+ owner_process_epoch=owner_process_epoch,
1102
+ owner_epoch=1,
1103
+ lease_expires_at=lease_expires_at,
1104
+ state=HttpBridgeSessionState.ACTIVE,
1105
+ account_id=account_id,
1106
+ model=model,
1107
+ service_tier=service_tier,
1108
+ latest_turn_state=latest_turn_state,
1109
+ latest_response_id=latest_response_id,
1110
+ last_seen_at=now,
1111
+ closed_at=None,
1112
+ )
1113
+ self._session.add(record)
1114
+ try:
1115
+ await self._commit_writer_section()
1116
+ except IntegrityError:
1117
+ await self._session.rollback()
1118
+ if attempt < _CLAIM_CAS_ATTEMPTS - 1:
1119
+ contended = True
1120
+ continue
1121
+ raise
1122
+ # Same reason the CAS path builds its own snapshot: another
1123
+ # same-instance claimant can advance this brand-new row before
1124
+ # a refresh runs, and returning that epoch would hand two
1125
+ # claimants the same fence.
1126
+ inserted_id = record.id
1127
+ return DurableBridgeSessionSnapshot(
1128
+ id=inserted_id,
1129
+ session_key_kind=session_key_kind,
1130
+ session_key_value=session_key_value,
1131
+ session_key_hash=session_key_hash,
1132
+ api_key_scope=api_key_scope,
1133
+ owner_instance_id=instance_id,
1134
+ owner_process_epoch=owner_process_epoch,
1135
+ owner_epoch=1,
1136
+ lease_expires_at=lease_expires_at,
1137
+ state=HttpBridgeSessionState.ACTIVE,
1138
+ account_id=account_id,
1139
+ model=model,
1140
+ service_tier=service_tier,
1141
+ latest_turn_state=latest_turn_state,
1142
+ latest_response_id=latest_response_id,
1143
+ latest_input_item_count=None,
1144
+ latest_input_full_fingerprint=None,
1145
+ latest_pending_tool_calls=None,
1146
+ last_seen_at=now,
1147
+ closed_at=None,
1148
+ )
1149
+
1150
+ state_closed = existing.state == HttpBridgeSessionState.CLOSED
1151
+ owner_absent = existing.owner_instance_id is None
1152
+ account_changed = existing.account_id != account_id
1153
+ # Reclaiming a retired row discards continuity for the same reason
1154
+ # an account change does. ``_to_lookup`` masked the anchors while
1155
+ # the marker stood, so the request that triggered this claim was
1156
+ # planned as an unanchored fresh start. Clearing only the marker
1157
+ # would leave the old response id, turn state, fingerprints and
1158
+ # pending calls behind, and the next lookup would serve that
1159
+ # abandoned anchor as ordinary continuity — including when the
1160
+ # retired account itself recovers and is selected again, where
1161
+ # ``account_changed`` is False.
1162
+ continuity_retired = _bridge_continuity_is_abandoned(
1163
+ existing.continuity_abandoned_at,
1164
+ existing.continuity_abandonment_scope,
1165
+ )
1166
+ owner_changed = existing.owner_instance_id != instance_id
1167
+ if owner_changed:
1168
+ lease_expired = existing.lease_expires_at is None or to_utc_naive(existing.lease_expires_at) <= now
1169
+ live_owned_draining = (
1170
+ existing.state == HttpBridgeSessionState.DRAINING and not lease_expired and not owner_absent
1171
+ )
1172
+ takeover_permitted = allow_takeover and not contended
1173
+ if live_owned_draining or (
1174
+ not takeover_permitted and not lease_expired and not owner_absent and not state_closed
1175
+ ):
1176
+ return _to_snapshot_required(existing)
1177
+ # Every claim advances the owner epoch, including a same-owner
1178
+ # reclaim: claims come only from a successor in-memory session (a
1179
+ # reused session renews instead of claiming), so a live same-owner
1180
+ # row means the predecessor local session is retiring concurrently
1181
+ # and its outstanding fenced release/renewals must no-op rather
1182
+ # than race this claim into a closed, ownerless row (issue #1695).
1183
+ next_epoch = existing.owner_epoch + 1
1184
+
1185
+ # Write through an explicit UPDATE that sets every ownership field
1186
+ # unconditionally. Mutating ORM attributes lets SQLAlchemy omit
1187
+ # fields whose values match this transaction's (possibly stale)
1188
+ # read, so a release committing between the SELECT and this write
1189
+ # survived the claim and the refresh below returned a closed,
1190
+ # ownerless row to a claimant that believed it had succeeded
1191
+ # (issue #1695; SQLite's with_for_update is a no-op).
1192
+ values: dict[str, object] = {
1193
+ "owner_instance_id": instance_id,
1194
+ "owner_process_epoch": owner_process_epoch,
1195
+ "owner_epoch": next_epoch,
1196
+ "lease_expires_at": lease_expires_at,
1197
+ "state": HttpBridgeSessionState.ACTIVE,
1198
+ "account_id": account_id,
1199
+ "model": model,
1200
+ "service_tier": service_tier,
1201
+ "last_seen_at": now,
1202
+ "closed_at": None,
1203
+ # A successful claim re-establishes ownership, so any prior
1204
+ # retirement is over. Clearing both markers here is what makes
1205
+ # retirement reversible: the same row, re-pinned to a healthy
1206
+ # account, becomes ordinary hard ownership again.
1207
+ "continuity_abandoned_at": None,
1208
+ "continuity_abandonment_scope": None,
1209
+ }
1210
+ if account_changed or continuity_retired:
1211
+ values["latest_turn_state"] = latest_turn_state
1212
+ values["latest_response_id"] = latest_response_id
1213
+ values["latest_input_item_count"] = None
1214
+ values["latest_input_full_fingerprint"] = None
1215
+ values["latest_pending_tool_calls_json"] = None
1216
+ else:
1217
+ if latest_turn_state is not None:
1218
+ values["latest_turn_state"] = latest_turn_state
1219
+ if latest_response_id is not None:
1220
+ values["latest_response_id"] = latest_response_id
1221
+ values["latest_input_item_count"] = None
1222
+ values["latest_input_full_fingerprint"] = None
1223
+ values["latest_pending_tool_calls_json"] = None
1224
+ async with sqlite_writer_section():
1225
+ # Compare-and-set on the epoch read above: SQLite's
1226
+ # with_for_update is a no-op, so two successor claims can both
1227
+ # read epoch N; without the guard both would write N+1 and both
1228
+ # believe they own the row with colliding fences. The loser's
1229
+ # update matches zero rows and retries against fresh state.
1230
+ result = await self._session.execute(
1231
+ update(HttpBridgeSessionRecord)
1232
+ .where(
1233
+ HttpBridgeSessionRecord.id == existing.id,
1234
+ HttpBridgeSessionRecord.owner_epoch == existing.owner_epoch,
1235
+ )
1236
+ .values(**values)
1237
+ )
1238
+ if not bool(getattr(result, "rowcount", 0)):
1239
+ await self._session.rollback()
1240
+ if attempt < _CLAIM_CAS_ATTEMPTS - 1:
1241
+ contended = True
1242
+ continue
1243
+ raise RuntimeError("Failed to claim durable bridge session after retry")
1244
+ if account_changed or continuity_retired:
1245
+ await self._clear_aliases_for_session(existing.id)
1246
+ await self._session.commit()
1247
+ # Build the snapshot from the values THIS CAS wrote rather than a
1248
+ # post-commit refresh: another successor can commit its own CAS
1249
+ # between this commit and a refresh, and returning that later epoch
1250
+ # would hand this claimant a fence that collides with the winner's.
1251
+ written_turn_state = values.get("latest_turn_state", existing.latest_turn_state)
1252
+ written_response_id = values.get("latest_response_id", existing.latest_response_id)
1253
+ written_pending_json = values.get("latest_pending_tool_calls_json", existing.latest_pending_tool_calls_json)
1254
+ return DurableBridgeSessionSnapshot(
1255
+ id=existing.id,
1256
+ session_key_kind=existing.session_key_kind,
1257
+ session_key_value=existing.session_key_value,
1258
+ session_key_hash=existing.session_key_hash,
1259
+ api_key_scope=existing.api_key_scope,
1260
+ owner_instance_id=instance_id,
1261
+ owner_process_epoch=owner_process_epoch,
1262
+ owner_epoch=next_epoch,
1263
+ lease_expires_at=lease_expires_at,
1264
+ state=HttpBridgeSessionState.ACTIVE,
1265
+ account_id=account_id,
1266
+ model=model,
1267
+ service_tier=service_tier,
1268
+ latest_turn_state=cast("str | None", written_turn_state),
1269
+ latest_response_id=cast("str | None", written_response_id),
1270
+ latest_input_item_count=cast(
1271
+ "int | None", values.get("latest_input_item_count", existing.latest_input_item_count)
1272
+ ),
1273
+ latest_input_full_fingerprint=cast(
1274
+ "str | None",
1275
+ values.get("latest_input_full_fingerprint", existing.latest_input_full_fingerprint),
1276
+ ),
1277
+ latest_pending_tool_calls=_decode_pending_tool_calls(
1278
+ cast("str | None", written_response_id),
1279
+ cast("str | None", written_pending_json),
1280
+ ),
1281
+ last_seen_at=now,
1282
+ closed_at=None,
1283
+ )
1284
+ raise RuntimeError("Failed to claim durable bridge session after retry")
1285
+
1286
+ async def renew_session(
1287
+ self,
1288
+ *,
1289
+ session_id: str,
1290
+ instance_id: str,
1291
+ owner_epoch: int,
1292
+ lease_ttl_seconds: float,
1293
+ latest_turn_state: str | None = None,
1294
+ latest_response_id: str | None = None,
1295
+ latest_input_item_count: int | None = None,
1296
+ latest_input_full_fingerprint: str | None = None,
1297
+ latest_pending_tool_calls: Mapping[str, str] | None = None,
1298
+ state: HttpBridgeSessionState | None = None,
1299
+ ) -> DurableBridgeSessionSnapshot | None:
1300
+ """Renew the lease with a single fenced UPDATE.
1301
+
1302
+ Fenced-out callers mutate nothing and receive the current owner snapshot.
1303
+ """
1304
+
1305
+ now = utcnow()
1306
+ values: dict[str, object] = {
1307
+ "lease_expires_at": now + timedelta(seconds=max(1.0, lease_ttl_seconds)),
1308
+ "last_seen_at": now,
1309
+ }
1310
+ if latest_turn_state is not None:
1311
+ values["latest_turn_state"] = latest_turn_state
1312
+ if latest_response_id is not None:
1313
+ values["latest_response_id"] = latest_response_id
1314
+ values["latest_pending_tool_calls_json"] = _encode_pending_tool_calls(
1315
+ latest_response_id,
1316
+ latest_pending_tool_calls,
1317
+ )
1318
+ if latest_input_item_count is None or latest_input_full_fingerprint is None:
1319
+ values["latest_input_item_count"] = None
1320
+ values["latest_input_full_fingerprint"] = None
1321
+ if latest_input_item_count is not None and latest_input_full_fingerprint is not None:
1322
+ values["latest_input_item_count"] = latest_input_item_count
1323
+ values["latest_input_full_fingerprint"] = latest_input_full_fingerprint
1324
+ if state is not None:
1325
+ values["state"] = state
1326
+ return await self._execute_fenced_session_update(
1327
+ session_id=session_id,
1328
+ instance_id=instance_id,
1329
+ owner_epoch=owner_epoch,
1330
+ values=values,
1331
+ )
1332
+
1333
+ async def latest_session_continuity(self, *, session_id: str) -> tuple[str | None, str | None] | None:
1334
+ """Read the durable session's current continuity anchors.
1335
+
1336
+ Returns ``(latest_response_id, latest_turn_state)``, or ``None`` when
1337
+ the session row does not exist. Both anchors are read together because
1338
+ a continuity clear removes both: fencing on the response id alone
1339
+ would let the clear match while a concurrently registered turn state
1340
+ still occupies the row, deleting continuity the caller never proved
1341
+ dead.
1342
+ """
1343
+ result = await self._session.execute(
1344
+ select(
1345
+ HttpBridgeSessionRecord.latest_response_id,
1346
+ HttpBridgeSessionRecord.latest_turn_state,
1347
+ ).where(HttpBridgeSessionRecord.id == session_id)
1348
+ )
1349
+ row = result.first()
1350
+ return None if row is None else (row[0], row[1])
1351
+
1352
+ async def rebind_session_account(
1353
+ self,
1354
+ *,
1355
+ session_id: str,
1356
+ instance_id: str,
1357
+ owner_epoch: int,
1358
+ account_id: str,
1359
+ clear_continuity: bool = False,
1360
+ expected_latest_response_id: object = REBIND_ANCHOR_UNFENCED,
1361
+ expected_latest_turn_state: object = REBIND_ANCHOR_UNFENCED,
1362
+ ) -> bool:
1363
+ """Persist a replacement account only while this worker owns the lease.
1364
+
1365
+ ``expected_latest_response_id`` and ``expected_latest_turn_state``
1366
+ fence a continuity clear on the anchors the caller validated: fresh
1367
+ continuity registered by a concurrent completion or turn-state write
1368
+ changes a column, the fenced UPDATE matches zero rows, and the caller
1369
+ observes a failed clear instead of deleting continuity its episode
1370
+ never proved dead.
1371
+ """
1372
+
1373
+ async with sqlite_writer_section():
1374
+ values: dict[str, object] = {"account_id": account_id}
1375
+ conditions = [
1376
+ HttpBridgeSessionRecord.id == session_id,
1377
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
1378
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
1379
+ ]
1380
+ if clear_continuity:
1381
+ values.update(
1382
+ latest_turn_state=None,
1383
+ latest_response_id=None,
1384
+ latest_input_item_count=None,
1385
+ latest_input_full_fingerprint=None,
1386
+ latest_pending_tool_calls_json=None,
1387
+ )
1388
+ if expected_latest_response_id is not REBIND_ANCHOR_UNFENCED:
1389
+ if expected_latest_response_id is None:
1390
+ conditions.append(HttpBridgeSessionRecord.latest_response_id.is_(None))
1391
+ else:
1392
+ conditions.append(HttpBridgeSessionRecord.latest_response_id == expected_latest_response_id)
1393
+ if expected_latest_turn_state is not REBIND_ANCHOR_UNFENCED:
1394
+ if expected_latest_turn_state is None:
1395
+ conditions.append(HttpBridgeSessionRecord.latest_turn_state.is_(None))
1396
+ else:
1397
+ conditions.append(HttpBridgeSessionRecord.latest_turn_state == expected_latest_turn_state)
1398
+ result = await self._session.execute(update(HttpBridgeSessionRecord).where(*conditions).values(**values))
1399
+ if clear_continuity and bool(getattr(result, "rowcount", 0)):
1400
+ await self._clear_aliases_for_session(session_id)
1401
+ await self._session.commit()
1402
+ return bool(getattr(result, "rowcount", 0))
1403
+
1404
+ async def release_session(
1405
+ self,
1406
+ *,
1407
+ session_id: str,
1408
+ instance_id: str,
1409
+ owner_epoch: int,
1410
+ draining: bool,
1411
+ ) -> DurableBridgeSessionSnapshot | None:
1412
+ """Release the lease with a single fenced UPDATE.
1413
+
1414
+ Fenced-out callers mutate nothing and receive the current owner snapshot.
1415
+ """
1416
+
1417
+ now = utcnow()
1418
+ values: dict[str, object] = {
1419
+ "owner_instance_id": None,
1420
+ "lease_expires_at": now,
1421
+ "last_seen_at": now,
1422
+ "state": HttpBridgeSessionState.DRAINING if draining else HttpBridgeSessionState.CLOSED,
1423
+ "closed_at": None if draining else now,
1424
+ }
1425
+ return await self._execute_fenced_session_update(
1426
+ session_id=session_id,
1427
+ instance_id=instance_id,
1428
+ owner_epoch=owner_epoch,
1429
+ values=values,
1430
+ )
1431
+
1432
+ async def clear_latest_response_anchor(
1433
+ self,
1434
+ *,
1435
+ session_id: str,
1436
+ instance_id: str,
1437
+ owner_epoch: int,
1438
+ ) -> DurableBridgeSessionSnapshot | None:
1439
+ """Invalidate a stuck eventless anchor with a single fenced UPDATE.
1440
+
1441
+ Clears only the response-id anchor and the state bound to it
1442
+ (input fingerprint/count, pending tool-call manifest). Leaves
1443
+ ``latest_turn_state`` and aliases untouched so the durable session
1444
+ remains reattachable without the stale anchor. Fenced-out callers
1445
+ mutate nothing and receive the current owner snapshot.
1446
+ """
1447
+
1448
+ values: dict[str, object] = {
1449
+ "latest_response_id": None,
1450
+ "latest_input_item_count": None,
1451
+ "latest_input_full_fingerprint": None,
1452
+ "latest_pending_tool_calls_json": None,
1453
+ }
1454
+ return await self._execute_fenced_session_update(
1455
+ session_id=session_id,
1456
+ instance_id=instance_id,
1457
+ owner_epoch=owner_epoch,
1458
+ values=values,
1459
+ )
1460
+
1461
+ async def clear_latest_response_anchor_if_matches(
1462
+ self,
1463
+ *,
1464
+ session_id: str,
1465
+ api_key_scope: str,
1466
+ instance_id: str,
1467
+ owner_epoch: int,
1468
+ response_id: str,
1469
+ ) -> DurableBridgeSessionSnapshot | None:
1470
+ """Clear one response anchor and its matching alias under the owner fence.
1471
+
1472
+ The response-id predicate makes a concurrent newer completion win over
1473
+ a stale denial. Only the denied response alias is removed; turn-state
1474
+ and other response aliases remain routable.
1475
+ """
1476
+
1477
+ values: dict[str, object] = {
1478
+ "latest_response_id": None,
1479
+ "latest_input_item_count": None,
1480
+ "latest_input_full_fingerprint": None,
1481
+ "latest_pending_tool_calls_json": None,
1482
+ }
1483
+ async with sqlite_writer_section():
1484
+ cleared = await self._session.execute(
1485
+ update(HttpBridgeSessionRecord)
1486
+ .where(
1487
+ HttpBridgeSessionRecord.id == session_id,
1488
+ HttpBridgeSessionRecord.api_key_scope == api_key_scope,
1489
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
1490
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
1491
+ HttpBridgeSessionRecord.latest_response_id == response_id,
1492
+ )
1493
+ .values(**values)
1494
+ .returning(HttpBridgeSessionRecord.id)
1495
+ )
1496
+ if cleared.scalar_one_or_none() is None:
1497
+ await self._session.rollback()
1498
+ return None
1499
+ await self._session.execute(
1500
+ delete(HttpBridgeSessionAlias).where(
1501
+ HttpBridgeSessionAlias.session_id == session_id,
1502
+ HttpBridgeSessionAlias.alias_kind == "previous_response_id",
1503
+ HttpBridgeSessionAlias.alias_hash == durable_bridge_hash(response_id),
1504
+ HttpBridgeSessionAlias.alias_value == response_id,
1505
+ HttpBridgeSessionAlias.api_key_scope == api_key_scope,
1506
+ )
1507
+ )
1508
+ row = await self._session.get(
1509
+ HttpBridgeSessionRecord,
1510
+ session_id,
1511
+ populate_existing=True,
1512
+ )
1513
+ await self._session.commit()
1514
+ return _to_snapshot(row)
1515
+
1516
+ async def record_recovery_attempt(
1517
+ self,
1518
+ *,
1519
+ session_id: str,
1520
+ instance_id: str,
1521
+ owner_epoch: int,
1522
+ request_fingerprint: str,
1523
+ request_id: str,
1524
+ account_id: str | None,
1525
+ model: str | None,
1526
+ replay_safe: bool,
1527
+ ) -> DurableBridgeRecoveryAttemptSnapshot | None:
1528
+ """Record a safe request before dispatch so an ambiguous outcome is recoverable."""
1529
+ async with sqlite_writer_section():
1530
+ # Lock the owner row through the journal write so a takeover
1531
+ # cannot advance the epoch after this check but before dispatch.
1532
+ owner_exists = await self._session.scalar(
1533
+ select(HttpBridgeSessionRecord.id)
1534
+ .where(
1535
+ HttpBridgeSessionRecord.id == session_id,
1536
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
1537
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
1538
+ )
1539
+ .with_for_update()
1540
+ )
1541
+ if owner_exists is None:
1542
+ await self._session.rollback()
1543
+ return None
1544
+ attempt = await self._session.scalar(
1545
+ select(HttpBridgeRecoveryAttemptRecord)
1546
+ .where(HttpBridgeRecoveryAttemptRecord.session_id == session_id)
1547
+ .where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint)
1548
+ .with_for_update()
1549
+ )
1550
+ if attempt is None:
1551
+ attempt = HttpBridgeRecoveryAttemptRecord(
1552
+ session_id=session_id,
1553
+ request_fingerprint=request_fingerprint,
1554
+ request_id=request_id,
1555
+ account_id=account_id,
1556
+ model=model,
1557
+ replay_safe=replay_safe,
1558
+ state=HttpBridgeRecoveryAttemptState.UNKNOWN,
1559
+ )
1560
+ self._session.add(attempt)
1561
+ elif attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED:
1562
+ snapshot = _to_recovery_attempt_snapshot(attempt)
1563
+ await self._session.rollback()
1564
+ return snapshot
1565
+ elif attempt.request_id != request_id:
1566
+ # A different request already owns the UNKNOWN checkpoint.
1567
+ # Do not overwrite it while that request may still be between
1568
+ # admission and dispatch; the caller must fail closed rather
1569
+ # than sharing a journal generation.
1570
+ snapshot = _to_recovery_attempt_snapshot(attempt)
1571
+ await self._session.rollback()
1572
+ return snapshot
1573
+ else:
1574
+ attempt.request_id = request_id
1575
+ attempt.account_id = account_id
1576
+ attempt.model = model
1577
+ attempt.replay_safe = replay_safe
1578
+ attempt.state = HttpBridgeRecoveryAttemptState.UNKNOWN
1579
+ attempt.response_id = None
1580
+ try:
1581
+ await self._session.commit()
1582
+ except IntegrityError:
1583
+ # A concurrent owner may have inserted the same fingerprint
1584
+ # after our initial SELECT (the absent-row case cannot be
1585
+ # locked by SQLite). Re-read the winner and use its durable
1586
+ # state instead of surfacing a transient uniqueness failure.
1587
+ await self._session.rollback()
1588
+ owner_exists = await self._session.scalar(
1589
+ select(HttpBridgeSessionRecord.id)
1590
+ .where(
1591
+ HttpBridgeSessionRecord.id == session_id,
1592
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
1593
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
1594
+ )
1595
+ .with_for_update()
1596
+ )
1597
+ if owner_exists is None:
1598
+ await self._session.rollback()
1599
+ return None
1600
+ attempt = await self._session.scalar(
1601
+ select(HttpBridgeRecoveryAttemptRecord)
1602
+ .where(HttpBridgeRecoveryAttemptRecord.session_id == session_id)
1603
+ .where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint)
1604
+ )
1605
+ if attempt is None:
1606
+ raise
1607
+ if attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED:
1608
+ snapshot = _to_recovery_attempt_snapshot(attempt)
1609
+ await self._session.rollback()
1610
+ return snapshot
1611
+ if attempt.request_id != request_id:
1612
+ snapshot = _to_recovery_attempt_snapshot(attempt)
1613
+ await self._session.rollback()
1614
+ return snapshot
1615
+ attempt.request_id = request_id
1616
+ attempt.account_id = account_id
1617
+ attempt.model = model
1618
+ attempt.replay_safe = replay_safe
1619
+ attempt.state = HttpBridgeRecoveryAttemptState.UNKNOWN
1620
+ attempt.response_id = None
1621
+ await self._session.commit()
1622
+ await self._session.refresh(attempt)
1623
+ return _to_recovery_attempt_snapshot(attempt)
1624
+
1625
+ async def lookup_recovery_attempt(
1626
+ self,
1627
+ *,
1628
+ session_id: str,
1629
+ request_fingerprint: str,
1630
+ ) -> DurableBridgeRecoveryAttemptSnapshot | None:
1631
+ attempt = await self._session.scalar(
1632
+ select(HttpBridgeRecoveryAttemptRecord)
1633
+ .where(HttpBridgeRecoveryAttemptRecord.session_id == session_id)
1634
+ .where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint)
1635
+ .where(HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN)
1636
+ .where(HttpBridgeRecoveryAttemptRecord.replay_safe.is_(True))
1637
+ )
1638
+ return _to_recovery_attempt_snapshot(attempt) if attempt is not None else None
1639
+
1640
+ async def mark_recovery_attempt_replayed(
1641
+ self,
1642
+ *,
1643
+ session_id: str,
1644
+ instance_id: str,
1645
+ owner_epoch: int,
1646
+ request_fingerprint: str,
1647
+ response_id: str | None = None,
1648
+ ) -> bool:
1649
+ async with sqlite_writer_section():
1650
+ # Keep the owner fence and journal transition in one transaction.
1651
+ # PostgreSQL's row lock prevents a concurrent takeover from
1652
+ # advancing the epoch between the check and the state update;
1653
+ # sqlite_writer_section provides the equivalent writer
1654
+ # serialization for SQLite.
1655
+ owner_exists = await self._session.scalar(
1656
+ select(HttpBridgeSessionRecord.id)
1657
+ .where(
1658
+ HttpBridgeSessionRecord.id == session_id,
1659
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
1660
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
1661
+ )
1662
+ .with_for_update()
1663
+ )
1664
+ if owner_exists is None:
1665
+ await self._session.rollback()
1666
+ return False
1667
+ values: dict[str, object] = {"state": HttpBridgeRecoveryAttemptState.REPLAYED}
1668
+ if response_id is not None:
1669
+ values["response_id"] = response_id
1670
+ # A claim authorizes one replay and must only transition UNKNOWN
1671
+ # rows. Settlement (which supplies response_id) remains idempotent
1672
+ # for a REPLAYED row after the replay completes.
1673
+ claimable_states = (
1674
+ (HttpBridgeRecoveryAttemptState.UNKNOWN,)
1675
+ if response_id is None
1676
+ else (HttpBridgeRecoveryAttemptState.UNKNOWN, HttpBridgeRecoveryAttemptState.REPLAYED)
1677
+ )
1678
+ result = await self._session.execute(
1679
+ update(HttpBridgeRecoveryAttemptRecord)
1680
+ .where(
1681
+ HttpBridgeRecoveryAttemptRecord.session_id == session_id,
1682
+ HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint,
1683
+ HttpBridgeRecoveryAttemptRecord.state.in_(claimable_states),
1684
+ )
1685
+ .values(**values)
1686
+ )
1687
+ await self._session.commit()
1688
+ return bool(getattr(result, "rowcount", 0))
1689
+
1690
+ async def rollback_recovery_attempt_replayed(
1691
+ self,
1692
+ *,
1693
+ session_id: str,
1694
+ instance_id: str,
1695
+ owner_epoch: int,
1696
+ request_fingerprint: str,
1697
+ ) -> bool:
1698
+ """Return a pre-dispatch replay claim to UNKNOWN under the owner fence."""
1699
+ async with sqlite_writer_section():
1700
+ owner_exists = await self._session.scalar(
1701
+ select(HttpBridgeSessionRecord.id)
1702
+ .where(
1703
+ HttpBridgeSessionRecord.id == session_id,
1704
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
1705
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
1706
+ )
1707
+ .with_for_update()
1708
+ )
1709
+ if owner_exists is None:
1710
+ await self._session.rollback()
1711
+ return False
1712
+ result = await self._session.execute(
1713
+ update(HttpBridgeRecoveryAttemptRecord)
1714
+ .where(
1715
+ HttpBridgeRecoveryAttemptRecord.session_id == session_id,
1716
+ HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint,
1717
+ HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.REPLAYED,
1718
+ HttpBridgeRecoveryAttemptRecord.response_id.is_(None),
1719
+ )
1720
+ .values(state=HttpBridgeRecoveryAttemptState.UNKNOWN)
1721
+ )
1722
+ await self._session.commit()
1723
+ return bool(getattr(result, "rowcount", 0))
1724
+
1725
+ async def rollback_recovery_attempt_before_dispatch(
1726
+ self,
1727
+ *,
1728
+ session_id: str,
1729
+ instance_id: str,
1730
+ owner_epoch: int,
1731
+ request_fingerprint: str,
1732
+ ) -> bool:
1733
+ """Delete an UNKNOWN checkpoint proven not to have reached upstream."""
1734
+ async with sqlite_writer_section():
1735
+ owner_exists = await self._session.scalar(
1736
+ select(HttpBridgeSessionRecord.id)
1737
+ .where(
1738
+ HttpBridgeSessionRecord.id == session_id,
1739
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
1740
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
1741
+ )
1742
+ .with_for_update()
1743
+ )
1744
+ if owner_exists is None:
1745
+ await self._session.rollback()
1746
+ return False
1747
+ result = await self._session.execute(
1748
+ delete(HttpBridgeRecoveryAttemptRecord).where(
1749
+ HttpBridgeRecoveryAttemptRecord.session_id == session_id,
1750
+ HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint,
1751
+ HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN,
1752
+ )
1753
+ )
1754
+ await self._session.commit()
1755
+ return bool(getattr(result, "rowcount", 0))
1756
+
1757
+ async def record_operation(
1758
+ self,
1759
+ *,
1760
+ operation_id: str,
1761
+ session_id: str,
1762
+ instance_id: str,
1763
+ owner_epoch: int,
1764
+ request_fingerprint: str,
1765
+ account_id: str | None,
1766
+ model: str | None,
1767
+ parent_response_id: str | None,
1768
+ api_key_scope: str | None = None,
1769
+ request_text: str | None = None,
1770
+ recovery_attempt_session_id: str | None = None,
1771
+ recovery_attempt_owner_epoch: int | None = None,
1772
+ recovery_attempt_fingerprint: str | None = None,
1773
+ recovery_attempt_consumed: bool = False,
1774
+ ) -> DurableBridgeOperationSnapshot | None:
1775
+ """Create a fenced operation identity, or return the existing one."""
1776
+ async with sqlite_writer_section():
1777
+ owner_exists = await self._session.scalar(
1778
+ select(HttpBridgeSessionRecord.id)
1779
+ .where(
1780
+ HttpBridgeSessionRecord.id == session_id,
1781
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
1782
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
1783
+ )
1784
+ .with_for_update()
1785
+ )
1786
+ if owner_exists is None:
1787
+ await self._session.rollback()
1788
+ return None
1789
+ operation = await self._session.scalar(
1790
+ select(HttpBridgeOperationRecord)
1791
+ .where(HttpBridgeOperationRecord.operation_id == operation_id)
1792
+ .with_for_update()
1793
+ )
1794
+ if operation is None:
1795
+ fingerprint_statement = select(HttpBridgeOperationRecord).where(
1796
+ HttpBridgeOperationRecord.request_fingerprint == request_fingerprint
1797
+ )
1798
+ if api_key_scope is not None:
1799
+ fingerprint_statement = fingerprint_statement.join(
1800
+ HttpBridgeSessionRecord,
1801
+ HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
1802
+ ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope)
1803
+ operation = await self._session.scalar(fingerprint_statement.with_for_update())
1804
+ if operation is not None:
1805
+ rebound_from_session_id = operation.session_id if operation.state == "failed" else None
1806
+ rebound_from_account_id = operation.account_id if operation.state == "failed" else None
1807
+ rebound_from_model = operation.model if operation.state == "failed" else None
1808
+ rebound_from_parent_response_id = operation.parent_response_id if operation.state == "failed" else None
1809
+ if operation.state == "abandoned":
1810
+ # Abandonment is a terminal duplicate-suppression fence.
1811
+ # A later continuation must receive the public recovery
1812
+ # signal rather than rebinding or resetting this row.
1813
+ snapshot = _to_operation_snapshot(operation)
1814
+ await self._session.rollback()
1815
+ return snapshot
1816
+ if recovery_attempt_consumed:
1817
+ # A REPLAYED recovery checkpoint is immutable. Return the
1818
+ # existing row for safe transcript replay or fail-closed
1819
+ # handling; never rebind a failed row and clear its spool.
1820
+ snapshot = _to_operation_snapshot(operation)
1821
+ await self._session.rollback()
1822
+ return snapshot
1823
+ rebound = False
1824
+ handoff_allowed = True
1825
+ if operation.session_id != session_id and operation.state not in {"completed", "incomplete"}:
1826
+ # A global fingerprint can outlive the durable session
1827
+ # that first recorded it. Do not steal an operation from
1828
+ # a still-live owner: its stream may still be dispatching
1829
+ # the turn, and rebinding would fence its writes while a
1830
+ # second owner sends a duplicate upstream request.
1831
+ previous_session = await self._session.scalar(
1832
+ select(HttpBridgeSessionRecord)
1833
+ .where(HttpBridgeSessionRecord.id == operation.session_id)
1834
+ .with_for_update()
1835
+ )
1836
+ now = utcnow()
1837
+ recovery_handoff_allowed = False
1838
+ if (
1839
+ previous_session is not None
1840
+ and recovery_attempt_session_id == operation.session_id
1841
+ and recovery_attempt_owner_epoch is not None
1842
+ and recovery_attempt_fingerprint is not None
1843
+ and previous_session.owner_instance_id == instance_id
1844
+ and previous_session.owner_epoch == recovery_attempt_owner_epoch
1845
+ ):
1846
+ # A fresh account-neutral replay has already fenced
1847
+ # the one-shot journal on the origin session. That
1848
+ # journal owner must remain fenced until settlement,
1849
+ # but the operation itself must move to the
1850
+ # replacement owner so its transcript and outcome
1851
+ # writes are accepted there. This is the only
1852
+ # cross-session handoff allowed while the origin
1853
+ # lease is still active.
1854
+ recovery_attempt = await self._session.scalar(
1855
+ select(HttpBridgeRecoveryAttemptRecord)
1856
+ .where(
1857
+ HttpBridgeRecoveryAttemptRecord.session_id == recovery_attempt_session_id,
1858
+ HttpBridgeRecoveryAttemptRecord.request_fingerprint == recovery_attempt_fingerprint,
1859
+ HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.REPLAYED,
1860
+ HttpBridgeRecoveryAttemptRecord.response_id.is_(None),
1861
+ )
1862
+ .with_for_update()
1863
+ )
1864
+ recovery_handoff_allowed = recovery_attempt is not None
1865
+ handoff_allowed = (
1866
+ recovery_handoff_allowed
1867
+ or previous_session is None
1868
+ or not (
1869
+ previous_session.owner_instance_id is not None
1870
+ and previous_session.lease_expires_at is not None
1871
+ # PostgreSQL returns timestamptz values with an
1872
+ # attached UTC offset, while ``utcnow`` is a
1873
+ # naive UTC value used by the durable layer.
1874
+ # Normalize before comparing so cross-session
1875
+ # recovery remains database-backend agnostic.
1876
+ and to_utc_naive(previous_session.lease_expires_at) > now
1877
+ )
1878
+ )
1879
+ if handoff_allowed:
1880
+ # Transfer only nonterminal operations to the currently
1881
+ # fenced owner before the caller resets the attempt
1882
+ # spool; completed transcripts remain attached to
1883
+ # their original session for replay.
1884
+ operation.session_id = session_id
1885
+ operation.account_id = account_id
1886
+ operation.model = model
1887
+ operation.parent_response_id = parent_response_id
1888
+ if request_text is not None and operation.request_text is None:
1889
+ operation.request_text = request_text
1890
+ operation.updated_at = now
1891
+ if operation.state == "failed" and handoff_allowed:
1892
+ # An explicit upstream failure is retryable. Rebind the
1893
+ # durable operation to the current owner while preserving
1894
+ # its global identity; concurrent reconnects will see the
1895
+ # submitted state and remain fenced.
1896
+ operation.session_id = session_id
1897
+ operation.account_id = account_id
1898
+ operation.model = model
1899
+ operation.parent_response_id = parent_response_id
1900
+ if request_text is not None and operation.request_text is None:
1901
+ operation.request_text = request_text
1902
+ operation.state = "submitted"
1903
+ operation.response_id = None
1904
+ # A failed attempt is a new replay attempt. Remove the
1905
+ # previous attempt's SSE spool atomically so a later
1906
+ # successful retry cannot replay a stale response.failed
1907
+ # event before its fresh response.created sequence.
1908
+ await self._delete_operation_spool_material((operation.operation_id,))
1909
+ operation.event_bytes = 0
1910
+ operation.event_spool_complete = False
1911
+ operation.terminal_append_phase = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_PENDING
1912
+ operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1
1913
+ operation.updated_at = utcnow()
1914
+ rebound = True
1915
+ if request_text is not None and operation.request_text is None:
1916
+ operation.request_text = request_text
1917
+ operation.updated_at = utcnow()
1918
+ snapshot = _to_operation_snapshot(
1919
+ operation,
1920
+ rebound=rebound,
1921
+ rebound_from_session_id=rebound_from_session_id,
1922
+ rebound_from_account_id=rebound_from_account_id,
1923
+ rebound_from_model=rebound_from_model,
1924
+ rebound_from_parent_response_id=rebound_from_parent_response_id,
1925
+ )
1926
+ await self._session.commit()
1927
+ return snapshot
1928
+ operation = HttpBridgeOperationRecord(
1929
+ operation_id=operation_id,
1930
+ session_id=session_id,
1931
+ request_fingerprint=request_fingerprint,
1932
+ account_id=account_id,
1933
+ model=model,
1934
+ parent_response_id=parent_response_id,
1935
+ request_text=request_text,
1936
+ state="submitted",
1937
+ # A transcript is replayable only after the event batcher has
1938
+ # drained and finalized it. Set this explicitly rather than
1939
+ # relying on a backend-specific schema default (notably the
1940
+ # pre-existing SQLite default on migrated databases).
1941
+ event_spool_complete=False,
1942
+ spool_format=HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
1943
+ )
1944
+ self._session.add(operation)
1945
+ try:
1946
+ await self._session.commit()
1947
+ except IntegrityError:
1948
+ await self._session.rollback()
1949
+ operation = await self._session.scalar(
1950
+ select(HttpBridgeOperationRecord).where(HttpBridgeOperationRecord.operation_id == operation_id)
1951
+ )
1952
+ if operation is None:
1953
+ # A reconnect may derive a different session-scoped
1954
+ # operation ID for the same anchored request. The global
1955
+ # fingerprint fence makes that race resolve to the
1956
+ # already-recorded operation instead of dispatching a
1957
+ # duplicate.
1958
+ fingerprint_statement = select(HttpBridgeOperationRecord).where(
1959
+ HttpBridgeOperationRecord.request_fingerprint == request_fingerprint
1960
+ )
1961
+ if api_key_scope is not None:
1962
+ fingerprint_statement = fingerprint_statement.join(
1963
+ HttpBridgeSessionRecord,
1964
+ HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
1965
+ ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope)
1966
+ operation = await self._session.scalar(fingerprint_statement)
1967
+ if operation is None:
1968
+ raise
1969
+ return _to_operation_snapshot(operation)
1970
+ await self._session.refresh(operation)
1971
+ return _to_operation_snapshot(operation, created=True)
1972
+
1973
+ async def get_operation(self, *, operation_id: str) -> DurableBridgeOperationSnapshot | None:
1974
+ operation = await self._session.scalar(
1975
+ select(HttpBridgeOperationRecord).where(HttpBridgeOperationRecord.operation_id == operation_id)
1976
+ )
1977
+ return _to_operation_snapshot(operation) if operation is not None else None
1978
+
1979
+ async def abandon_stale_operations(
1980
+ self,
1981
+ *,
1982
+ cutoff: datetime,
1983
+ lease_expired_before: datetime,
1984
+ protected_operation_ids: Collection[str] = (),
1985
+ batch_size: int = _PURGE_CLOSED_BATCH_SIZE,
1986
+ scan_cursor: DurableBridgeOperationAbandonmentScanCursor | None = None,
1987
+ ) -> DurableBridgeOperationAbandonmentSweep:
1988
+ """Fence stale ownerless ambiguous operations without replaying them.
1989
+
1990
+ The candidate read and the conditional update run in one writer
1991
+ transaction. PostgreSQL locks the operation and owning session on the
1992
+ normal bounded-predicate path while SQLite is serialized by
1993
+ ``sqlite_writer_section``. An oversized protection snapshot is read in
1994
+ finite pages without an expanding predicate; the caller resumes from
1995
+ the returned keyset cursor on the next sweep. Its final updates still
1996
+ compare every value that can change the abandonment decision, including
1997
+ durable event-spool progress, so a recovery claim, owner renewal, or
1998
+ status event that wins the race leaves the row untouched.
1999
+ """
2000
+ now = utcnow()
2001
+ if batch_size <= 0:
2002
+ return DurableBridgeOperationAbandonmentSweep(abandonments=(), next_cursor=None)
2003
+ protected_ids = tuple(
2004
+ dict.fromkeys(str(operation_id) for operation_id in protected_operation_ids if operation_id)
2005
+ )
2006
+ protected_id_set = frozenset(protected_ids)
2007
+ # A large snapshot cannot be represented by one expanding ``NOT IN``
2008
+ # predicate safely; the bounded-page path below filters it in memory.
2009
+ use_bounded_protection = len(protected_ids) > _PROTECTED_OPERATION_ID_SAFE_LIMIT
2010
+ ambiguous_states = ("unknown", "acknowledged")
2011
+ stale_owner = or_(
2012
+ HttpBridgeSessionRecord.lease_expires_at.is_(None),
2013
+ HttpBridgeSessionRecord.lease_expires_at <= lease_expired_before,
2014
+ )
2015
+ candidate_filter = [
2016
+ HttpBridgeOperationRecord.state.in_(ambiguous_states),
2017
+ HttpBridgeOperationRecord.updated_at < cutoff,
2018
+ stale_owner,
2019
+ ]
2020
+ if protected_ids and not use_bounded_protection:
2021
+ candidate_filter.append(~HttpBridgeOperationRecord.operation_id.in_(protected_ids))
2022
+
2023
+ abandoned: list[DurableBridgeOperationAbandonment] = []
2024
+ next_cursor: DurableBridgeOperationAbandonmentScanCursor | None = None
2025
+ async with sqlite_writer_section():
2026
+ candidates = []
2027
+ cursor = scan_cursor if use_bounded_protection else None
2028
+ scanned_rows = 0
2029
+ while len(candidates) < batch_size and (
2030
+ not use_bounded_protection or scanned_rows < _PROTECTED_OPERATION_SCAN_BUDGET
2031
+ ):
2032
+ page_filter = list(candidate_filter)
2033
+ if cursor is not None:
2034
+ # Best-effort keyset resume: on SQLite a row whose
2035
+ # ``updated_at`` was stamped by ``onupdate`` shares the
2036
+ # cursor row's second but not its text form, so it is
2037
+ # neither ``>`` nor ``==`` the bound cursor value and is
2038
+ # skipped until the cursor wraps to ``None``. The sweep
2039
+ # still converges because every pass restarts from the
2040
+ # oldest unprotected row once the scan reaches the end.
2041
+ page_filter.append(
2042
+ or_(
2043
+ HttpBridgeOperationRecord.updated_at > cursor.updated_at,
2044
+ and_(
2045
+ HttpBridgeOperationRecord.updated_at == cursor.updated_at,
2046
+ HttpBridgeOperationRecord.operation_id > cursor.operation_id,
2047
+ ),
2048
+ )
2049
+ )
2050
+ page_size = batch_size
2051
+ if use_bounded_protection:
2052
+ page_size = min(
2053
+ _PROTECTED_OPERATION_ID_SAFE_LIMIT,
2054
+ _PROTECTED_OPERATION_SCAN_BUDGET - scanned_rows,
2055
+ )
2056
+ statement = (
2057
+ select(
2058
+ HttpBridgeOperationRecord,
2059
+ HttpBridgeSessionRecord.owner_instance_id,
2060
+ HttpBridgeSessionRecord.owner_epoch,
2061
+ HttpBridgeOperationRecord.event_bytes,
2062
+ )
2063
+ .join(
2064
+ HttpBridgeSessionRecord,
2065
+ HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
2066
+ )
2067
+ .where(*page_filter)
2068
+ .order_by(
2069
+ HttpBridgeOperationRecord.updated_at.asc(),
2070
+ HttpBridgeOperationRecord.operation_id.asc(),
2071
+ )
2072
+ .limit(page_size)
2073
+ )
2074
+ statement = statement.with_for_update()
2075
+ selected = await self._session.execute(statement)
2076
+ page = list(selected.all())
2077
+ if not page:
2078
+ # Reaching the end after a prior cursor wraps the next
2079
+ # heartbeat to the beginning of the keyset.
2080
+ next_cursor = None
2081
+ break
2082
+ if use_bounded_protection:
2083
+ scanned_rows += len(page)
2084
+ candidates.extend(row for row in page if str(row[0].operation_id) not in protected_id_set)
2085
+ last_operation = page[-1][0]
2086
+ next_cursor = DurableBridgeOperationAbandonmentScanCursor(
2087
+ updated_at=last_operation.updated_at,
2088
+ operation_id=str(last_operation.operation_id),
2089
+ )
2090
+ if len(candidates) >= batch_size:
2091
+ candidates = candidates[:batch_size]
2092
+ break
2093
+ cursor = next_cursor
2094
+ if len(page) < page_size:
2095
+ next_cursor = None
2096
+ break
2097
+ if scanned_rows >= _PROTECTED_OPERATION_SCAN_BUDGET:
2098
+ break
2099
+ else:
2100
+ candidates = page
2101
+ break
2102
+ for operation, owner_instance_id, owner_epoch, candidate_event_bytes in candidates:
2103
+ source_state = str(operation.state)
2104
+ candidate_updated_at = operation.updated_at
2105
+ candidate_event_bytes = int(candidate_event_bytes or 0)
2106
+ owner_predicates = [
2107
+ HttpBridgeSessionRecord.id == operation.session_id,
2108
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
2109
+ ]
2110
+ if owner_instance_id is None:
2111
+ owner_predicates.extend(
2112
+ (
2113
+ HttpBridgeSessionRecord.owner_instance_id.is_(None),
2114
+ or_(
2115
+ HttpBridgeSessionRecord.lease_expires_at.is_(None),
2116
+ HttpBridgeSessionRecord.lease_expires_at <= lease_expired_before,
2117
+ ),
2118
+ )
2119
+ )
2120
+ owner_lease_outcome = "ownerless"
2121
+ else:
2122
+ owner_predicates.extend(
2123
+ (
2124
+ HttpBridgeSessionRecord.owner_instance_id == owner_instance_id,
2125
+ or_(
2126
+ HttpBridgeSessionRecord.lease_expires_at.is_(None),
2127
+ HttpBridgeSessionRecord.lease_expires_at <= lease_expired_before,
2128
+ ),
2129
+ )
2130
+ )
2131
+ owner_lease_outcome = "expired"
2132
+ # The inactivity clock is compared against ``cutoff`` rather
2133
+ # than for equality with the loaded candidate value. SQLite
2134
+ # stores the ``onupdate=func.now()`` timestamp written by the
2135
+ # event appenders as second-precision text while the ORM
2136
+ # binds the loaded datetime back with microseconds, so an
2137
+ # equality predicate never matches the rows this sweep
2138
+ # exists for. Any competing progress, claim, or renewal write
2139
+ # stamps ``updated_at`` at roughly ``now``, which is never
2140
+ # older than ``cutoff``, so the race fence is preserved.
2141
+ compare_and_set = await self._session.execute(
2142
+ update(HttpBridgeOperationRecord)
2143
+ .where(
2144
+ HttpBridgeOperationRecord.operation_id == operation.operation_id,
2145
+ HttpBridgeOperationRecord.state == source_state,
2146
+ HttpBridgeOperationRecord.updated_at < cutoff,
2147
+ HttpBridgeOperationRecord.event_bytes == candidate_event_bytes,
2148
+ exists(select(HttpBridgeSessionRecord.id).where(*owner_predicates)),
2149
+ )
2150
+ .values(state="abandoned", updated_at=now)
2151
+ )
2152
+ if not getattr(compare_and_set, "rowcount", 0):
2153
+ continue
2154
+ age_seconds = max(0.0, (to_utc_naive(now) - to_utc_naive(candidate_updated_at)).total_seconds())
2155
+ abandoned.append(
2156
+ DurableBridgeOperationAbandonment(
2157
+ source_state=source_state,
2158
+ age_seconds=min(age_seconds, float(_ABANDONMENT_LOG_AGE_CAP_SECONDS)),
2159
+ owner_lease_outcome=owner_lease_outcome,
2160
+ session_hash=durable_bridge_hash(operation.session_id)[:16],
2161
+ )
2162
+ )
2163
+ await self._session.commit()
2164
+ return DurableBridgeOperationAbandonmentSweep(
2165
+ abandonments=tuple(abandoned),
2166
+ next_cursor=next_cursor if use_bounded_protection else None,
2167
+ )
2168
+
2169
+ async def reset_operation_event_spool(
2170
+ self,
2171
+ *,
2172
+ operation_id: str,
2173
+ session_id: str,
2174
+ instance_id: str,
2175
+ owner_epoch: int,
2176
+ ) -> bool:
2177
+ """Start a fresh transcript for a server-owned ambiguous retry."""
2178
+ async with sqlite_writer_section():
2179
+ owner_exists = await self._session.scalar(
2180
+ select(HttpBridgeSessionRecord.id)
2181
+ .where(
2182
+ HttpBridgeSessionRecord.id == session_id,
2183
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
2184
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
2185
+ )
2186
+ .with_for_update()
2187
+ )
2188
+ operation = await self._session.scalar(
2189
+ select(HttpBridgeOperationRecord)
2190
+ .where(
2191
+ HttpBridgeOperationRecord.operation_id == operation_id,
2192
+ HttpBridgeOperationRecord.session_id == session_id,
2193
+ HttpBridgeOperationRecord.state.not_in(("completed", "incomplete", "abandoned")),
2194
+ )
2195
+ .with_for_update()
2196
+ )
2197
+ if owner_exists is None or operation is None or operation.state == "abandoned":
2198
+ await self._session.rollback()
2199
+ return False
2200
+ await self._delete_operation_spool_material((operation_id,))
2201
+ operation.event_bytes = 0
2202
+ operation.event_spool_complete = False
2203
+ # The phase fences the terminal write of one attempt only. A fresh
2204
+ # transcript is a fresh attempt, so it must not inherit a previous
2205
+ # attempt's recorded outcome (``failed`` rows are resettable).
2206
+ operation.terminal_append_phase = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_PENDING
2207
+ operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1
2208
+ operation.updated_at = utcnow()
2209
+ await self._session.commit()
2210
+ return True
2211
+
2212
+ async def claim_unknown_operation_for_recovery(
2213
+ self,
2214
+ *,
2215
+ operation_id: str,
2216
+ session_id: str,
2217
+ instance_id: str,
2218
+ owner_epoch: int,
2219
+ max_recovery_dispatches: int | None = None,
2220
+ ) -> bool:
2221
+ """Atomically claim an UNKNOWN operation for one recovery attempt.
2222
+
2223
+ Recovery admission can be reached by multiple reconnects at once. A
2224
+ reset followed by a later state transition leaves a window where each
2225
+ reconnect can observe UNKNOWN and submit the same operation. Keep the
2226
+ owner fence, state transition, and transcript reset in one serialized
2227
+ write so exactly one caller can move UNKNOWN back to SUBMITTED.
2228
+ """
2229
+ async with sqlite_writer_section():
2230
+ owner_exists = await self._session.scalar(
2231
+ select(HttpBridgeSessionRecord.id)
2232
+ .where(
2233
+ HttpBridgeSessionRecord.id == session_id,
2234
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
2235
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
2236
+ )
2237
+ .with_for_update()
2238
+ )
2239
+ operation = await self._session.scalar(
2240
+ select(HttpBridgeOperationRecord)
2241
+ .where(
2242
+ HttpBridgeOperationRecord.operation_id == operation_id,
2243
+ HttpBridgeOperationRecord.session_id == session_id,
2244
+ HttpBridgeOperationRecord.state == "unknown",
2245
+ )
2246
+ .with_for_update()
2247
+ )
2248
+ if owner_exists is None or operation is None or operation.state == "abandoned":
2249
+ await self._session.rollback()
2250
+ return False
2251
+ if max_recovery_dispatches is not None and operation.recovery_dispatch_count >= max_recovery_dispatches:
2252
+ await self._session.rollback()
2253
+ return False
2254
+ await self._delete_operation_spool_material((operation_id,))
2255
+ operation.state = "submitted"
2256
+ operation.response_id = None
2257
+ operation.recovery_dispatch_count += 1
2258
+ operation.event_bytes = 0
2259
+ operation.event_spool_complete = False
2260
+ operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1
2261
+ operation.updated_at = utcnow()
2262
+ await self._session.commit()
2263
+ return True
2264
+
2265
+ async def mark_operation_unknown(
2266
+ self,
2267
+ *,
2268
+ operation_id: str,
2269
+ session_id: str,
2270
+ instance_id: str,
2271
+ owner_epoch: int,
2272
+ restore_recovery_dispatch_claim: bool = False,
2273
+ ) -> bool:
2274
+ """Fence an ambiguously dispatched SUBMITTED operation as UNKNOWN.
2275
+
2276
+ The operation event reader can race the send-failure cleanup. Lock the
2277
+ row before changing it and leave an already acknowledged or terminal
2278
+ operation untouched; those states carry stronger evidence than the
2279
+ transport exception and must never be downgraded to UNKNOWN.
2280
+ """
2281
+ async with sqlite_writer_section():
2282
+ owner_exists = await self._session.scalar(
2283
+ select(HttpBridgeSessionRecord.id)
2284
+ .where(
2285
+ HttpBridgeSessionRecord.id == session_id,
2286
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
2287
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
2288
+ )
2289
+ .with_for_update()
2290
+ )
2291
+ operation = await self._session.scalar(
2292
+ select(HttpBridgeOperationRecord)
2293
+ .where(
2294
+ HttpBridgeOperationRecord.operation_id == operation_id,
2295
+ HttpBridgeOperationRecord.session_id == session_id,
2296
+ )
2297
+ .with_for_update()
2298
+ )
2299
+ if owner_exists is None or operation is None or operation.state == "abandoned":
2300
+ await self._session.rollback()
2301
+ return False
2302
+ if operation.state == "submitted":
2303
+ operation.state = "unknown"
2304
+ if restore_recovery_dispatch_claim and operation.recovery_dispatch_count > 0:
2305
+ operation.recovery_dispatch_count -= 1
2306
+ operation.updated_at = utcnow()
2307
+ elif (
2308
+ restore_recovery_dispatch_claim
2309
+ and operation.state == "unknown"
2310
+ and operation.recovery_dispatch_count > 0
2311
+ ):
2312
+ # A concurrent cleanup may have fenced the row first. The
2313
+ # caller still owns a proven pre-dispatch recovery claim, so
2314
+ # refund exactly that claim while retaining UNKNOWN.
2315
+ operation.recovery_dispatch_count -= 1
2316
+ operation.updated_at = utcnow()
2317
+ await self._session.commit()
2318
+ return True
2319
+
2320
+ async def rollback_operation_before_dispatch(
2321
+ self,
2322
+ *,
2323
+ operation_id: str,
2324
+ session_id: str,
2325
+ instance_id: str,
2326
+ owner_epoch: int,
2327
+ restore_rebound: bool = False,
2328
+ rebound_from_session_id: str | None = None,
2329
+ rebound_from_account_id: str | None = None,
2330
+ rebound_from_model: str | None = None,
2331
+ rebound_from_parent_response_id: str | None = None,
2332
+ ) -> bool:
2333
+ """Undo an operation transition that never reached upstream."""
2334
+ async with sqlite_writer_section():
2335
+ owner_exists = await self._session.scalar(
2336
+ select(HttpBridgeSessionRecord.id)
2337
+ .where(
2338
+ HttpBridgeSessionRecord.id == session_id,
2339
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
2340
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
2341
+ )
2342
+ .with_for_update()
2343
+ )
2344
+ operation = await self._session.scalar(
2345
+ select(HttpBridgeOperationRecord)
2346
+ .where(
2347
+ HttpBridgeOperationRecord.operation_id == operation_id,
2348
+ HttpBridgeOperationRecord.session_id == session_id,
2349
+ HttpBridgeOperationRecord.state == "submitted",
2350
+ HttpBridgeOperationRecord.response_id.is_(None),
2351
+ HttpBridgeOperationRecord.event_bytes == 0,
2352
+ )
2353
+ .with_for_update()
2354
+ )
2355
+ if owner_exists is None or operation is None:
2356
+ await self._session.rollback()
2357
+ return False
2358
+ if await self._operation_has_spool_material(operation_id):
2359
+ await self._session.rollback()
2360
+ return False
2361
+ if restore_rebound:
2362
+ operation.state = "failed"
2363
+ operation.event_spool_complete = False
2364
+ if rebound_from_session_id is not None:
2365
+ operation.session_id = rebound_from_session_id
2366
+ operation.account_id = rebound_from_account_id
2367
+ operation.model = rebound_from_model
2368
+ operation.parent_response_id = rebound_from_parent_response_id
2369
+ operation.updated_at = utcnow()
2370
+ else:
2371
+ await self._session.delete(operation)
2372
+ await self._session.commit()
2373
+ return True
2374
+
2375
+ async def get_operation_by_fingerprint(
2376
+ self,
2377
+ *,
2378
+ request_fingerprint: str,
2379
+ api_key_scope: str | None = None,
2380
+ ) -> DurableBridgeOperationSnapshot | None:
2381
+ statement = select(HttpBridgeOperationRecord).where(
2382
+ HttpBridgeOperationRecord.request_fingerprint == request_fingerprint
2383
+ )
2384
+ if api_key_scope is not None:
2385
+ statement = statement.join(
2386
+ HttpBridgeSessionRecord,
2387
+ HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
2388
+ ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope)
2389
+ operation = await self._session.scalar(statement)
2390
+ return _to_operation_snapshot(operation) if operation is not None else None
2391
+
2392
+ async def get_operation_events(self, *, operation_id: str, max_bytes: int = 8 * 1024 * 1024) -> list[str]:
2393
+ operation = (
2394
+ await self._session.execute(
2395
+ select(
2396
+ HttpBridgeOperationRecord.spool_format,
2397
+ HttpBridgeOperationRecord.event_bytes,
2398
+ ).where(HttpBridgeOperationRecord.operation_id == operation_id)
2399
+ )
2400
+ ).one_or_none()
2401
+ if operation is None:
2402
+ return []
2403
+ spool_format = str(operation.spool_format)
2404
+ if type(operation.event_bytes) is not int:
2405
+ return []
2406
+ expected_event_bytes = operation.event_bytes
2407
+ if expected_event_bytes < 0 or expected_event_bytes > max_bytes:
2408
+ return []
2409
+ if spool_format == HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1:
2410
+ result = await self._session.execute(
2411
+ select(HttpBridgeOperationEvent.event_text)
2412
+ .where(HttpBridgeOperationEvent.operation_id == operation_id)
2413
+ .order_by(HttpBridgeOperationEvent.sequence_number.asc())
2414
+ )
2415
+ events = [str(value) for value in result.scalars().all()]
2416
+ if sum(len(event.encode("utf-8")) for event in events) != expected_event_bytes:
2417
+ return []
2418
+ return events
2419
+ if spool_format != HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2:
2420
+ return []
2421
+
2422
+ chunks = (
2423
+ await self._session.execute(
2424
+ select(HttpBridgeOperationEventChunk)
2425
+ .where(HttpBridgeOperationEventChunk.operation_id == operation_id)
2426
+ .order_by(HttpBridgeOperationEventChunk.first_sequence_number.asc())
2427
+ )
2428
+ ).scalars()
2429
+ expected_sequence = 1
2430
+ total_event_count = 0
2431
+ remaining_bytes = expected_event_bytes
2432
+ events: list[str] = []
2433
+ for chunk in chunks:
2434
+ if type(chunk.event_count) is not int or not isinstance(chunk.payload, bytes):
2435
+ return []
2436
+ if chunk.first_sequence_number != expected_sequence:
2437
+ return []
2438
+ total_event_count += chunk.event_count
2439
+ if total_event_count > DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS:
2440
+ return []
2441
+ try:
2442
+ decoded = decode_durable_bridge_transcript_chunk(
2443
+ codec=chunk.codec,
2444
+ payload=bytes(chunk.payload),
2445
+ event_count=chunk.event_count,
2446
+ uncompressed_bytes=chunk.uncompressed_bytes,
2447
+ payload_sha256=chunk.payload_sha256,
2448
+ max_uncompressed_bytes=remaining_bytes + 4 * chunk.event_count,
2449
+ )
2450
+ except DurableBridgeTranscriptDecodeError:
2451
+ return []
2452
+ decoded_bytes = sum(len(event.encode("utf-8")) for event in decoded)
2453
+ if decoded_bytes > remaining_bytes:
2454
+ return []
2455
+ events.extend(decoded)
2456
+ remaining_bytes -= decoded_bytes
2457
+ expected_sequence += chunk.event_count
2458
+ return events if remaining_bytes == 0 else []
2459
+
2460
+ async def get_operation_by_response_id(self, *, response_id: str) -> DurableBridgeOperationSnapshot | None:
2461
+ operation = await self._session.scalar(
2462
+ select(HttpBridgeOperationRecord).where(
2463
+ HttpBridgeOperationRecord.response_id == response_id,
2464
+ HttpBridgeOperationRecord.state.in_(("completed", "incomplete")),
2465
+ )
2466
+ )
2467
+ return _to_operation_snapshot(operation) if operation is not None else None
2468
+
2469
+ async def get_replayable_transcript(
2470
+ self,
2471
+ *,
2472
+ response_id: str,
2473
+ max_turns: int = 128,
2474
+ max_bytes: int = 8 * 1024 * 1024,
2475
+ ) -> list[DurableBridgeTranscriptTurn] | None:
2476
+ """Return a complete parent-response chain, newest turn last.
2477
+
2478
+ Missing request bodies, truncated event spools, or a broken parent
2479
+ chain make the transcript ineligible for reconstruction.
2480
+ """
2481
+ turns: list[DurableBridgeTranscriptTurn] = []
2482
+ visited: set[str] = set()
2483
+ total_bytes = 0
2484
+ current_response_id: str | None = response_id
2485
+ while current_response_id is not None:
2486
+ if current_response_id in visited or len(turns) >= max_turns:
2487
+ return None
2488
+ visited.add(current_response_id)
2489
+ operation = await self.get_operation_by_response_id(response_id=current_response_id)
2490
+ if operation is None or operation.request_text is None or not operation.event_spool_complete:
2491
+ return None
2492
+ remaining_event_bytes = max_bytes - total_bytes - len(operation.request_text.encode("utf-8"))
2493
+ if remaining_event_bytes < 0:
2494
+ return None
2495
+ events = await self.get_operation_events(
2496
+ operation_id=operation.operation_id,
2497
+ max_bytes=remaining_event_bytes,
2498
+ )
2499
+ if not events or not any(
2500
+ "response.completed" in event or "response.incomplete" in event for event in events
2501
+ ):
2502
+ return None
2503
+ turn_bytes = len(operation.request_text.encode("utf-8")) + sum(
2504
+ len(event.encode("utf-8")) for event in events
2505
+ )
2506
+ total_bytes += turn_bytes
2507
+ if total_bytes > max_bytes:
2508
+ return None
2509
+ turns.append(DurableBridgeTranscriptTurn(operation=operation, events=tuple(events)))
2510
+ current_response_id = operation.parent_response_id
2511
+ turns.reverse()
2512
+ return turns
2513
+
2514
+ async def purge_operation_spool_batch(
2515
+ self,
2516
+ *,
2517
+ cutoff: datetime,
2518
+ batch_size: int = DURABLE_BRIDGE_OPERATION_SPOOL_PURGE_BATCH_SIZE,
2519
+ ) -> DurableBridgeOperationPurgeBatchResult:
2520
+ """Delete eligible transcript material past retention.
2521
+
2522
+ Nonterminal rows are purgeable only after their owning session is
2523
+ ownerless or its lease has expired. Recheck that predicate in the
2524
+ delete transaction so an in-flight operation cannot lose its
2525
+ duplicate-suppression ledger between selection and deletion.
2526
+ """
2527
+ terminal_states = ("completed", "incomplete", "failed", "abandoned")
2528
+ # UNKNOWN is an ambiguous, still-live operation while its owner lease
2529
+ # is active. Treat it like the other nonterminal states so retention
2530
+ # cannot delete the duplicate-suppression fence during a long-running
2531
+ # server-indefinite recovery attempt.
2532
+ nonterminal_states = ("submitted", "acknowledged", "unknown")
2533
+ stale_owner = or_(
2534
+ HttpBridgeSessionRecord.owner_instance_id.is_(None),
2535
+ HttpBridgeSessionRecord.lease_expires_at.is_(None),
2536
+ HttpBridgeSessionRecord.lease_expires_at < utcnow(),
2537
+ )
2538
+ stale_nonterminal = and_(
2539
+ HttpBridgeOperationRecord.state.in_(nonterminal_states),
2540
+ exists(
2541
+ select(HttpBridgeSessionRecord.id)
2542
+ .where(
2543
+ HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
2544
+ stale_owner,
2545
+ )
2546
+ .correlate(HttpBridgeOperationRecord)
2547
+ ),
2548
+ )
2549
+ purgeable = or_(HttpBridgeOperationRecord.state.in_(terminal_states), stale_nonterminal)
2550
+ async with sqlite_writer_section():
2551
+ selected = await self._session.execute(
2552
+ select(HttpBridgeOperationRecord)
2553
+ .join(
2554
+ HttpBridgeSessionRecord,
2555
+ HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
2556
+ )
2557
+ .where(HttpBridgeOperationRecord.updated_at < cutoff, purgeable)
2558
+ .order_by(HttpBridgeOperationRecord.updated_at.asc())
2559
+ .limit(batch_size)
2560
+ .with_for_update()
2561
+ )
2562
+ # The joined FOR UPDATE locks both the operation and owning
2563
+ # session on PostgreSQL, serializing retention deletion with
2564
+ # claim_session() on the same continuity row.
2565
+ operation_ids = [str(operation.operation_id) for operation in selected.scalars().all()]
2566
+ if not operation_ids:
2567
+ await self._session.commit()
2568
+ return DurableBridgeOperationPurgeBatchResult(
2569
+ selected_operations=0,
2570
+ deleted_operations=0,
2571
+ )
2572
+ deleted = await self._session.execute(
2573
+ delete(HttpBridgeOperationRecord)
2574
+ .where(
2575
+ HttpBridgeOperationRecord.operation_id.in_(operation_ids),
2576
+ HttpBridgeOperationRecord.updated_at < cutoff,
2577
+ purgeable,
2578
+ )
2579
+ .returning(HttpBridgeOperationRecord.operation_id)
2580
+ )
2581
+ deleted_ids = [str(value) for value in deleted.scalars().all()]
2582
+ if deleted_ids:
2583
+ await self._delete_operation_spool_material(deleted_ids)
2584
+ await self._session.commit()
2585
+ return DurableBridgeOperationPurgeBatchResult(
2586
+ selected_operations=len(operation_ids),
2587
+ deleted_operations=len(deleted_ids),
2588
+ )
2589
+
2590
+ async def purge_operation_spool(
2591
+ self,
2592
+ *,
2593
+ cutoff: datetime,
2594
+ batch_size: int = DURABLE_BRIDGE_OPERATION_SPOOL_PURGE_BATCH_SIZE,
2595
+ ) -> int:
2596
+ """Delete one eligible transcript batch and return actual deletes."""
2597
+ result = await self.purge_operation_spool_batch(cutoff=cutoff, batch_size=batch_size)
2598
+ return result.deleted_operations
2599
+
2600
+ async def append_operation_event_chunk(
2601
+ self,
2602
+ *,
2603
+ events: Sequence[DurableBridgeOperationEventInput],
2604
+ max_bytes: int,
2605
+ ) -> bool:
2606
+ """Append one compressed v2 chunk under the durable owner fence."""
2607
+ if not events:
2608
+ return True
2609
+ first = events[0]
2610
+ if any(
2611
+ event.operation_id != first.operation_id
2612
+ or event.session_id != first.session_id
2613
+ or event.instance_id != first.instance_id
2614
+ or event.owner_epoch != first.owner_epoch
2615
+ for event in events
2616
+ ):
2617
+ return False
2618
+ event_texts = [event.event_text for event in events]
2619
+ event_bytes = sum(len(event_text.encode("utf-8")) for event_text in event_texts)
2620
+ if event_bytes > max_bytes:
2621
+ return False
2622
+ if not await self._chunk_append_preflight_allows_encoding(
2623
+ operation_id=first.operation_id,
2624
+ session_id=first.session_id,
2625
+ instance_id=first.instance_id,
2626
+ owner_epoch=first.owner_epoch,
2627
+ event_bytes=event_bytes,
2628
+ event_count=len(event_texts),
2629
+ max_bytes=max_bytes,
2630
+ ):
2631
+ return False
2632
+ try:
2633
+ encoded = await asyncio.to_thread(encode_durable_bridge_transcript_chunk, event_texts)
2634
+ except ValueError:
2635
+ return False
2636
+ async with sqlite_writer_section():
2637
+ locked_operation = await self._lock_operation_for_chunk_append(
2638
+ operation_id=first.operation_id,
2639
+ session_id=first.session_id,
2640
+ instance_id=first.instance_id,
2641
+ owner_epoch=first.owner_epoch,
2642
+ )
2643
+ if locked_operation is None:
2644
+ return False
2645
+ operation, append_allowed = locked_operation
2646
+ if not append_allowed:
2647
+ await self._session.rollback()
2648
+ return False
2649
+ if int(operation.event_bytes or 0) + event_bytes > max_bytes:
2650
+ operation.event_spool_complete = False
2651
+ await self._session.commit()
2652
+ return False
2653
+ first_sequence_number = await self._next_operation_chunk_sequence(first.operation_id)
2654
+ if first_sequence_number - 1 + len(event_texts) > DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS:
2655
+ operation.event_spool_complete = False
2656
+ await self._session.commit()
2657
+ return False
2658
+ self._session.add(
2659
+ HttpBridgeOperationEventChunk(
2660
+ operation_id=first.operation_id,
2661
+ first_sequence_number=first_sequence_number,
2662
+ event_count=encoded.event_count,
2663
+ codec=encoded.codec,
2664
+ uncompressed_bytes=encoded.uncompressed_bytes,
2665
+ payload=encoded.payload,
2666
+ payload_sha256=encoded.payload_sha256,
2667
+ )
2668
+ )
2669
+ operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2
2670
+ operation.event_bytes = int(operation.event_bytes or 0) + event_bytes
2671
+ await self._session.commit()
2672
+ return True
2673
+
2674
+ async def append_terminal_operation_chunk(
2675
+ self,
2676
+ *,
2677
+ operation_id: str,
2678
+ session_id: str,
2679
+ instance_id: str,
2680
+ owner_epoch: int,
2681
+ event_text: str,
2682
+ max_bytes: int,
2683
+ state: str,
2684
+ expected_recovery_dispatch_count: int | None = None,
2685
+ response_id: str | None = None,
2686
+ complete_spool: bool = True,
2687
+ ) -> bool:
2688
+ """Append a terminal v2 chunk and expose its outcome atomically."""
2689
+ event_bytes = len(event_text.encode("utf-8"))
2690
+ preflight_allows_encoding = await self._chunk_append_preflight_allows_encoding(
2691
+ operation_id=operation_id,
2692
+ session_id=session_id,
2693
+ instance_id=instance_id,
2694
+ owner_epoch=owner_epoch,
2695
+ event_bytes=event_bytes,
2696
+ event_count=1,
2697
+ max_bytes=max_bytes,
2698
+ )
2699
+ try:
2700
+ encoded = (
2701
+ await asyncio.to_thread(encode_durable_bridge_transcript_chunk, (event_text,))
2702
+ if preflight_allows_encoding
2703
+ else None
2704
+ )
2705
+ except ValueError:
2706
+ encoded = None
2707
+ async with sqlite_writer_section():
2708
+ locked_operation = await self._lock_operation_for_chunk_append(
2709
+ operation_id=operation_id,
2710
+ session_id=session_id,
2711
+ instance_id=instance_id,
2712
+ owner_epoch=owner_epoch,
2713
+ expected_recovery_dispatch_count=expected_recovery_dispatch_count,
2714
+ )
2715
+ if locked_operation is None:
2716
+ return False
2717
+ operation, append_allowed = locked_operation
2718
+ if not append_allowed:
2719
+ if operation.terminal_append_phase in _HTTP_BRIDGE_TERMINAL_APPEND_RECORDED_PHASES:
2720
+ # A terminal outcome is already recorded for this dispatch:
2721
+ # keep it verbatim instead of restamping ``state`` from a
2722
+ # duplicate or late terminal write.
2723
+ await self._session.rollback()
2724
+ return False
2725
+ operation.event_spool_complete = False
2726
+ operation.state = state
2727
+ if response_id is not None:
2728
+ operation.response_id = response_id
2729
+ operation.updated_at = utcnow()
2730
+ await self._session.commit()
2731
+ return False
2732
+ if int(operation.event_bytes or 0) + event_bytes > max_bytes or encoded is None:
2733
+ operation.event_spool_complete = False
2734
+ operation.state = state
2735
+ if response_id is not None:
2736
+ operation.response_id = response_id
2737
+ operation.updated_at = utcnow()
2738
+ await self._session.commit()
2739
+ return False
2740
+ first_sequence_number = await self._next_operation_chunk_sequence(operation_id)
2741
+ if first_sequence_number > DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS:
2742
+ operation.event_spool_complete = False
2743
+ operation.state = state
2744
+ if response_id is not None:
2745
+ operation.response_id = response_id
2746
+ operation.updated_at = utcnow()
2747
+ await self._session.commit()
2748
+ return False
2749
+ self._session.add(
2750
+ HttpBridgeOperationEventChunk(
2751
+ operation_id=operation_id,
2752
+ first_sequence_number=first_sequence_number,
2753
+ event_count=encoded.event_count,
2754
+ codec=encoded.codec,
2755
+ uncompressed_bytes=encoded.uncompressed_bytes,
2756
+ payload=encoded.payload,
2757
+ payload_sha256=encoded.payload_sha256,
2758
+ )
2759
+ )
2760
+ operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2
2761
+ operation.event_bytes = int(operation.event_bytes or 0) + event_bytes
2762
+ operation.state = state
2763
+ if response_id is not None:
2764
+ operation.response_id = response_id
2765
+ operation.event_spool_complete = complete_spool
2766
+ # The terminal transcript block is committed with the outcome.
2767
+ # Recording the phase here is what makes a duplicate or late
2768
+ # terminal write observe a finished dispatch and refuse, while a
2769
+ # deferred ``complete_spool=False`` finalization is still allowed.
2770
+ operation.terminal_append_phase = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_APPENDED
2771
+ operation.updated_at = utcnow()
2772
+ await self._session.commit()
2773
+ return True
2774
+
2775
+ async def append_operation_event(
2776
+ self,
2777
+ *,
2778
+ operation_id: str,
2779
+ session_id: str,
2780
+ instance_id: str,
2781
+ owner_epoch: int,
2782
+ event_text: str,
2783
+ max_bytes: int,
2784
+ ) -> bool:
2785
+ """Append one replayable SSE block under the durable owner fence."""
2786
+ async with sqlite_writer_section():
2787
+ owner_exists = await self._session.scalar(
2788
+ select(HttpBridgeSessionRecord.id)
2789
+ .where(
2790
+ HttpBridgeSessionRecord.id == session_id,
2791
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
2792
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
2793
+ )
2794
+ .with_for_update()
2795
+ )
2796
+ operation = await self._session.scalar(
2797
+ select(HttpBridgeOperationRecord)
2798
+ .where(
2799
+ HttpBridgeOperationRecord.operation_id == operation_id,
2800
+ HttpBridgeOperationRecord.session_id == session_id,
2801
+ HttpBridgeOperationRecord.spool_format == HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
2802
+ )
2803
+ .with_for_update()
2804
+ )
2805
+ if owner_exists is None or operation is None or operation.state == "abandoned":
2806
+ await self._session.rollback()
2807
+ return False
2808
+ event_size = len(event_text.encode("utf-8"))
2809
+ if event_size > max_bytes or int(operation.event_bytes or 0) + event_size > max_bytes:
2810
+ operation.event_spool_complete = False
2811
+ await self._session.commit()
2812
+ return False
2813
+ next_sequence = await self._session.scalar(
2814
+ select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where(
2815
+ HttpBridgeOperationEvent.operation_id == operation_id,
2816
+ )
2817
+ )
2818
+ sequence = int(next_sequence or 1)
2819
+ self._session.add(
2820
+ HttpBridgeOperationEvent(
2821
+ operation_id=operation_id,
2822
+ sequence_number=sequence,
2823
+ # Include occurrence position so identical downstream
2824
+ # blocks remain distinct in replay transcripts.
2825
+ event_fingerprint=durable_bridge_hash(f"{sequence}:{event_text}"),
2826
+ event_text=event_text,
2827
+ )
2828
+ )
2829
+ operation.event_bytes = int(operation.event_bytes or 0) + event_size
2830
+ await self._session.commit()
2831
+ return True
2832
+
2833
+ async def append_terminal_operation_event(
2834
+ self,
2835
+ *,
2836
+ operation_id: str,
2837
+ session_id: str,
2838
+ instance_id: str,
2839
+ owner_epoch: int,
2840
+ event_text: str,
2841
+ max_bytes: int,
2842
+ state: str,
2843
+ expected_recovery_dispatch_count: int | None = None,
2844
+ response_id: str | None = None,
2845
+ complete_spool: bool = True,
2846
+ ) -> bool:
2847
+ """Append a terminal event and expose its operation state atomically."""
2848
+ async with sqlite_writer_section():
2849
+ owner_exists = await self._session.scalar(
2850
+ select(HttpBridgeSessionRecord.id)
2851
+ .where(
2852
+ HttpBridgeSessionRecord.id == session_id,
2853
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
2854
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
2855
+ )
2856
+ .with_for_update()
2857
+ )
2858
+ # ``expected_recovery_dispatch_count`` is opt-in: a caller that
2859
+ # claimed a recovery dispatch pins the generation it observed, and
2860
+ # a caller that never claimed one passes nothing rather than a
2861
+ # literal 0, so an operation retained from an older release with a
2862
+ # non-zero counter still settles.
2863
+ terminal_event_statement = select(HttpBridgeOperationRecord).where(
2864
+ HttpBridgeOperationRecord.operation_id == operation_id,
2865
+ HttpBridgeOperationRecord.session_id == session_id,
2866
+ HttpBridgeOperationRecord.spool_format == HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
2867
+ )
2868
+ if expected_recovery_dispatch_count is not None:
2869
+ terminal_event_statement = terminal_event_statement.where(
2870
+ HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count
2871
+ )
2872
+ operation = await self._session.scalar(terminal_event_statement.with_for_update())
2873
+ if owner_exists is None or operation is None or operation.state == "abandoned":
2874
+ await self._session.rollback()
2875
+ return False
2876
+ # Scoped to rows whose terminal outcome is already recorded: an
2877
+ # ordinary terminal append always observes its own row as
2878
+ # ``state = completed`` with an incomplete spool, because the relay
2879
+ # updates the operation state before appending.
2880
+ if operation.terminal_append_phase in _HTTP_BRIDGE_TERMINAL_APPEND_RECORDED_PHASES:
2881
+ await self._session.rollback()
2882
+ return False
2883
+ event_size = len(event_text.encode("utf-8"))
2884
+ persisted = event_size <= max_bytes and int(operation.event_bytes or 0) + event_size <= max_bytes
2885
+ if persisted:
2886
+ next_sequence = await self._session.scalar(
2887
+ select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where(
2888
+ HttpBridgeOperationEvent.operation_id == operation_id,
2889
+ )
2890
+ )
2891
+ sequence = int(next_sequence or 1)
2892
+ self._session.add(
2893
+ HttpBridgeOperationEvent(
2894
+ operation_id=operation_id,
2895
+ sequence_number=sequence,
2896
+ event_fingerprint=durable_bridge_hash(f"{sequence}:{event_text}"),
2897
+ event_text=event_text,
2898
+ )
2899
+ )
2900
+ operation.event_bytes = int(operation.event_bytes or 0) + event_size
2901
+ else:
2902
+ operation.event_spool_complete = False
2903
+ # The terminal outcome is still authoritative even when the
2904
+ # transcript block cannot fit in the bounded spool. Expose
2905
+ # the failed state so an identical retry does not remain
2906
+ # fenced as an in-flight operation until retention expires.
2907
+ operation.state = state
2908
+ if response_id is not None:
2909
+ operation.response_id = response_id
2910
+ operation.updated_at = utcnow()
2911
+ await self._session.commit()
2912
+ return False
2913
+ operation.state = state
2914
+ if response_id is not None:
2915
+ operation.response_id = response_id
2916
+ operation.event_spool_complete = complete_spool
2917
+ # Reached only when the terminal block was actually spooled (the
2918
+ # over-cap branch above returns early), so the dispatch's terminal
2919
+ # transcript outcome is now recorded.
2920
+ operation.terminal_append_phase = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_APPENDED
2921
+ operation.updated_at = utcnow()
2922
+ await self._session.commit()
2923
+ return persisted
2924
+
2925
+ async def append_operation_events(
2926
+ self,
2927
+ *,
2928
+ events: Sequence[DurableBridgeOperationEventInput],
2929
+ max_bytes: int,
2930
+ ) -> bool:
2931
+ """Append a batch of SSE blocks with one fenced transaction."""
2932
+ if not events:
2933
+ return True
2934
+ first = events[0]
2935
+ if any(
2936
+ event.operation_id != first.operation_id
2937
+ or event.session_id != first.session_id
2938
+ or event.instance_id != first.instance_id
2939
+ or event.owner_epoch != first.owner_epoch
2940
+ for event in events
2941
+ ):
2942
+ return False
2943
+ async with sqlite_writer_section():
2944
+ owner_exists = await self._session.scalar(
2945
+ select(HttpBridgeSessionRecord.id)
2946
+ .where(
2947
+ HttpBridgeSessionRecord.id == first.session_id,
2948
+ HttpBridgeSessionRecord.owner_instance_id == first.instance_id,
2949
+ HttpBridgeSessionRecord.owner_epoch == first.owner_epoch,
2950
+ )
2951
+ .with_for_update()
2952
+ )
2953
+ operation = await self._session.scalar(
2954
+ select(HttpBridgeOperationRecord)
2955
+ .where(
2956
+ HttpBridgeOperationRecord.operation_id == first.operation_id,
2957
+ HttpBridgeOperationRecord.session_id == first.session_id,
2958
+ HttpBridgeOperationRecord.spool_format == HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
2959
+ )
2960
+ .with_for_update()
2961
+ )
2962
+ if owner_exists is None or operation is None or operation.state == "abandoned":
2963
+ await self._session.rollback()
2964
+ return False
2965
+ next_sequence = await self._session.scalar(
2966
+ select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where(
2967
+ HttpBridgeOperationEvent.operation_id == first.operation_id,
2968
+ )
2969
+ )
2970
+ sequence = int(next_sequence or 1)
2971
+ pending: list[tuple[str, int, str, int]] = []
2972
+ total_bytes = int(operation.event_bytes or 0)
2973
+ for event in events:
2974
+ event_size = len(event.event_text.encode("utf-8"))
2975
+ if total_bytes + event_size > max_bytes:
2976
+ operation.event_spool_complete = False
2977
+ await self._session.commit()
2978
+ return False
2979
+ total_bytes += event_size
2980
+ pending.append(
2981
+ (
2982
+ event.event_text,
2983
+ sequence,
2984
+ durable_bridge_hash(f"{sequence}:{event.event_text}"),
2985
+ event_size,
2986
+ )
2987
+ )
2988
+ sequence += 1
2989
+ if pending:
2990
+ for event_text, sequence_number, fingerprint, event_size in pending:
2991
+ self._session.add(
2992
+ HttpBridgeOperationEvent(
2993
+ operation_id=first.operation_id,
2994
+ sequence_number=sequence_number,
2995
+ event_fingerprint=fingerprint,
2996
+ event_text=event_text,
2997
+ )
2998
+ )
2999
+ operation.event_bytes = total_bytes
3000
+ await self._session.commit()
3001
+ return True
3002
+
3003
+ async def finalize_operation_event_spool(
3004
+ self,
3005
+ *,
3006
+ operation_id: str,
3007
+ session_id: str,
3008
+ instance_id: str,
3009
+ owner_epoch: int,
3010
+ expected_state: str | None = None,
3011
+ ) -> bool:
3012
+ """Mark a terminal operation replay-complete after its queue drained."""
3013
+ if expected_state is not None and expected_state not in {"completed", "incomplete", "failed"}:
3014
+ return False
3015
+ async with sqlite_writer_section():
3016
+ owner_exists = await self._session.scalar(
3017
+ select(HttpBridgeSessionRecord.id)
3018
+ .where(
3019
+ HttpBridgeSessionRecord.id == session_id,
3020
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
3021
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
3022
+ )
3023
+ .with_for_update()
3024
+ )
3025
+ predicates = [
3026
+ HttpBridgeOperationRecord.operation_id == operation_id,
3027
+ HttpBridgeOperationRecord.session_id == session_id,
3028
+ HttpBridgeOperationRecord.event_spool_complete.is_(False),
3029
+ # A settled row was published without a confirmed append, so it
3030
+ # is not replayable no matter which attempt reaches finalization
3031
+ # (a terminal append can commit just as its bound expires, which
3032
+ # runs settlement and leaves a finalize in flight).
3033
+ HttpBridgeOperationRecord.terminal_append_phase != HTTP_BRIDGE_TERMINAL_APPEND_PHASE_SETTLED,
3034
+ ]
3035
+ predicates.append(
3036
+ HttpBridgeOperationRecord.state == expected_state
3037
+ if expected_state is not None
3038
+ else HttpBridgeOperationRecord.state.in_(("completed", "incomplete"))
3039
+ )
3040
+ result = await self._session.execute(
3041
+ update(HttpBridgeOperationRecord)
3042
+ .where(*predicates)
3043
+ .values(event_spool_complete=True, updated_at=utcnow())
3044
+ )
3045
+ if owner_exists is None:
3046
+ await self._session.rollback()
3047
+ return False
3048
+ await self._session.commit()
3049
+ return bool(getattr(result, "rowcount", 0))
3050
+
3051
+ async def get_latest_completed_operation(
3052
+ self,
3053
+ *,
3054
+ session_id: str,
3055
+ parent_response_id: str,
3056
+ request_fingerprint: str | None = None,
3057
+ ) -> DurableBridgeOperationSnapshot | None:
3058
+ predicates = [
3059
+ HttpBridgeOperationRecord.session_id == session_id,
3060
+ HttpBridgeOperationRecord.parent_response_id == parent_response_id,
3061
+ HttpBridgeOperationRecord.state == "completed",
3062
+ HttpBridgeOperationRecord.response_id.is_not(None),
3063
+ ]
3064
+ if request_fingerprint is not None:
3065
+ predicates.append(HttpBridgeOperationRecord.request_fingerprint == request_fingerprint)
3066
+ operation = await self._session.scalar(
3067
+ select(HttpBridgeOperationRecord)
3068
+ .where(*predicates)
3069
+ .order_by(HttpBridgeOperationRecord.updated_at.desc())
3070
+ .limit(1)
3071
+ )
3072
+ return _to_operation_snapshot(operation) if operation is not None else None
3073
+
3074
+ async def get_latest_completed_operation_any_session(
3075
+ self,
3076
+ *,
3077
+ parent_response_id: str,
3078
+ api_key_scope: str | None = None,
3079
+ request_fingerprint: str | None = None,
3080
+ ) -> DurableBridgeOperationSnapshot | None:
3081
+ statement = select(HttpBridgeOperationRecord)
3082
+ if api_key_scope is not None:
3083
+ statement = statement.join(
3084
+ HttpBridgeSessionRecord,
3085
+ HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
3086
+ ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope)
3087
+ operation = await self._session.scalar(
3088
+ statement.where(
3089
+ HttpBridgeOperationRecord.parent_response_id == parent_response_id,
3090
+ HttpBridgeOperationRecord.state == "completed",
3091
+ HttpBridgeOperationRecord.response_id.is_not(None),
3092
+ *(
3093
+ [HttpBridgeOperationRecord.request_fingerprint == request_fingerprint]
3094
+ if request_fingerprint is not None
3095
+ else []
3096
+ ),
3097
+ )
3098
+ .order_by(HttpBridgeOperationRecord.updated_at.desc())
3099
+ .limit(1)
3100
+ )
3101
+ return _to_operation_snapshot(operation) if operation is not None else None
3102
+
3103
+ async def settle_terminal_append_failure(
3104
+ self,
3105
+ *,
3106
+ operation_id: str,
3107
+ session_id: str,
3108
+ instance_id: str,
3109
+ owner_epoch: int,
3110
+ state: str,
3111
+ expected_response_id: str | None,
3112
+ expected_recovery_dispatch_count: int | None = None,
3113
+ alternate_expected_response_id: str | None = None,
3114
+ response_id: str | None = None,
3115
+ ) -> bool:
3116
+ """Settle only the terminal attempt whose append outcome was ambiguous."""
3117
+ async with sqlite_writer_section():
3118
+ owner_exists = await self._session.scalar(
3119
+ select(HttpBridgeSessionRecord.id)
3120
+ .where(
3121
+ HttpBridgeSessionRecord.id == session_id,
3122
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
3123
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
3124
+ )
3125
+ .with_for_update()
3126
+ )
3127
+ if owner_exists is None:
3128
+ await self._session.rollback()
3129
+ return False
3130
+ acknowledged_response_matches = (
3131
+ HttpBridgeOperationRecord.response_id == expected_response_id
3132
+ if expected_response_id is not None
3133
+ else HttpBridgeOperationRecord.response_id.is_(None)
3134
+ )
3135
+ if alternate_expected_response_id is not None:
3136
+ acknowledged_response_matches = or_(
3137
+ acknowledged_response_matches,
3138
+ HttpBridgeOperationRecord.response_id == alternate_expected_response_id,
3139
+ )
3140
+ terminal_response_matches = (
3141
+ HttpBridgeOperationRecord.response_id == response_id
3142
+ if response_id is not None
3143
+ else HttpBridgeOperationRecord.response_id.is_(None)
3144
+ )
3145
+ values: dict[str, object] = {
3146
+ "state": state,
3147
+ "event_spool_complete": False,
3148
+ "updated_at": utcnow(),
3149
+ }
3150
+ if state in _HTTP_BRIDGE_TERMINAL_OPERATION_STATES:
3151
+ # A terminal outcome is now published without a confirmed
3152
+ # transcript append. SETTLED makes the row final for this
3153
+ # dispatch so the append that lost the race can neither rewrite
3154
+ # the state nor flip the spool back to replayable.
3155
+ #
3156
+ # Only genuinely terminal settlements set it. A continuity
3157
+ # failure after acknowledgement settles back to
3158
+ # ``acknowledged``, where the operation is still live and its
3159
+ # later appends must keep working.
3160
+ values["terminal_append_phase"] = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_SETTLED
3161
+ if response_id is not None:
3162
+ values["response_id"] = response_id
3163
+ # Opt-in recovery-dispatch fence, as in
3164
+ # ``append_terminal_operation_event``. The newer-attempt rejection
3165
+ # below does not depend on it: a retry that reset the row to
3166
+ # ``submitted`` matches neither ``acknowledged`` nor the terminal
3167
+ # state being settled, so the update touches no row.
3168
+ settlement_statement = update(HttpBridgeOperationRecord).where(
3169
+ HttpBridgeOperationRecord.operation_id == operation_id,
3170
+ HttpBridgeOperationRecord.session_id == session_id,
3171
+ HttpBridgeOperationRecord.state != "abandoned",
3172
+ or_(
3173
+ and_(HttpBridgeOperationRecord.state == "acknowledged", acknowledged_response_matches),
3174
+ and_(
3175
+ HttpBridgeOperationRecord.state == state,
3176
+ or_(acknowledged_response_matches, terminal_response_matches),
3177
+ ),
3178
+ ),
3179
+ )
3180
+ if expected_recovery_dispatch_count is not None:
3181
+ settlement_statement = settlement_statement.where(
3182
+ HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count
3183
+ )
3184
+ result = await self._session.execute(settlement_statement.values(**values))
3185
+ await self._session.commit()
3186
+ return bool(getattr(result, "rowcount", 0))
3187
+
3188
+ async def update_operation(
3189
+ self,
3190
+ *,
3191
+ operation_id: str,
3192
+ session_id: str,
3193
+ instance_id: str,
3194
+ owner_epoch: int,
3195
+ state: str,
3196
+ response_id: str | None = None,
3197
+ ) -> bool:
3198
+ async with sqlite_writer_section():
3199
+ owner_exists = await self._session.scalar(
3200
+ select(HttpBridgeSessionRecord.id)
3201
+ .where(
3202
+ HttpBridgeSessionRecord.id == session_id,
3203
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
3204
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
3205
+ )
3206
+ .with_for_update()
3207
+ )
3208
+ if owner_exists is None:
3209
+ await self._session.rollback()
3210
+ return False
3211
+ values: dict[str, object] = {"state": state, "updated_at": utcnow()}
3212
+ if response_id is not None:
3213
+ values["response_id"] = response_id
3214
+ result = await self._session.execute(
3215
+ update(HttpBridgeOperationRecord)
3216
+ .where(
3217
+ HttpBridgeOperationRecord.operation_id == operation_id,
3218
+ HttpBridgeOperationRecord.session_id == session_id,
3219
+ HttpBridgeOperationRecord.state != "abandoned",
3220
+ )
3221
+ .values(**values)
3222
+ )
3223
+ await self._session.commit()
3224
+ return bool(getattr(result, "rowcount", 0))
3225
+
3226
+ async def _execute_fenced_session_update(
3227
+ self,
3228
+ *,
3229
+ session_id: str,
3230
+ instance_id: str,
3231
+ owner_epoch: int,
3232
+ values: dict[str, object],
3233
+ ) -> DurableBridgeSessionSnapshot | None:
3234
+ async with sqlite_writer_section():
3235
+ result = await self._session.execute(
3236
+ update(HttpBridgeSessionRecord)
3237
+ .where(
3238
+ HttpBridgeSessionRecord.id == session_id,
3239
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
3240
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
3241
+ )
3242
+ .values(**values)
3243
+ .returning(*_SNAPSHOT_COLUMNS)
3244
+ )
3245
+ updated_row = result.one_or_none()
3246
+ await self._session.commit()
3247
+ if updated_row is not None:
3248
+ return _returned_row_to_snapshot(updated_row)
3249
+ current = await self._session.get(HttpBridgeSessionRecord, session_id, populate_existing=True)
3250
+ return _to_snapshot(current)
3251
+
3252
+ async def get_sessions_by_ids(
3253
+ self,
3254
+ session_ids: Sequence[str],
3255
+ *,
3256
+ chunk_size: int = _SESSION_ID_LOOKUP_CHUNK_SIZE,
3257
+ ) -> list[DurableBridgeSessionSnapshot]:
3258
+ unique_ids = list(dict.fromkeys(session_ids))
3259
+ if not unique_ids:
3260
+ return []
3261
+ snapshots: list[DurableBridgeSessionSnapshot] = []
3262
+ for start in range(0, len(unique_ids), chunk_size):
3263
+ chunk = unique_ids[start : start + chunk_size]
3264
+ result = await self._session.execute(
3265
+ select(HttpBridgeSessionRecord).where(HttpBridgeSessionRecord.id.in_(chunk))
3266
+ )
3267
+ snapshots.extend(_to_snapshot_required(row) for row in result.scalars().all())
3268
+ return snapshots
3269
+
3270
+ async def mark_owner_draining(self, *, instance_id: str) -> int:
3271
+ result = await self._session.execute(
3272
+ select(HttpBridgeSessionRecord).where(
3273
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
3274
+ HttpBridgeSessionRecord.state == HttpBridgeSessionState.ACTIVE,
3275
+ )
3276
+ )
3277
+ rows = list(result.scalars().all())
3278
+ now = utcnow()
3279
+ for row in rows:
3280
+ row.state = HttpBridgeSessionState.DRAINING
3281
+ row.last_seen_at = now
3282
+ await self._commit_writer_section()
3283
+ return len(rows)
3284
+
3285
+ async def purge_owned_sessions_on_startup(
3286
+ self,
3287
+ *,
3288
+ instance_id: str,
3289
+ owner_process_epoch: str | None = None,
3290
+ ownerless_cutoff: datetime | None = None,
3291
+ batch_size: int = _PURGE_CLOSED_BATCH_SIZE,
3292
+ ) -> int:
3293
+ """Remove durable bridge rows left by the previous process instance.
3294
+
3295
+ Ownerless ACTIVE/DRAINING rows are preserved by default: a graceful
3296
+ drain release intentionally clears ownership while keeping continuity
3297
+ aliases reusable until the full bridge idle-retention window. Callers
3298
+ that already computed that retention cutoff may pass ``ownerless_cutoff``
3299
+ to piggyback that abandoned-row cleanup onto startup.
3300
+ """
3301
+
3302
+ deleted_count = 0
3303
+ while True:
3304
+ now = utcnow()
3305
+ if owner_process_epoch is None:
3306
+ owned_restart_filter = HttpBridgeSessionRecord.owner_instance_id == instance_id
3307
+ else:
3308
+ owned_restart_filter = and_(
3309
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
3310
+ or_(
3311
+ HttpBridgeSessionRecord.owner_process_epoch.is_(None),
3312
+ HttpBridgeSessionRecord.owner_process_epoch != owner_process_epoch,
3313
+ ),
3314
+ )
3315
+ purge_predicates = [owned_restart_filter]
3316
+ if ownerless_cutoff is not None:
3317
+ purge_predicates.append(
3318
+ and_(
3319
+ HttpBridgeSessionRecord.owner_instance_id.is_(None),
3320
+ HttpBridgeSessionRecord.state.in_(
3321
+ (HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING),
3322
+ ),
3323
+ or_(
3324
+ HttpBridgeSessionRecord.lease_expires_at.is_(None),
3325
+ HttpBridgeSessionRecord.lease_expires_at < now,
3326
+ ),
3327
+ HttpBridgeSessionRecord.last_seen_at < ownerless_cutoff,
3328
+ )
3329
+ )
3330
+ startup_purge_filter = or_(*purge_predicates)
3331
+ result = await self._session.execute(
3332
+ select(
3333
+ HttpBridgeSessionRecord.id,
3334
+ HttpBridgeSessionRecord.session_key_kind,
3335
+ HttpBridgeSessionRecord.session_key_value,
3336
+ HttpBridgeSessionRecord.owner_instance_id,
3337
+ HttpBridgeSessionRecord.owner_process_epoch,
3338
+ HttpBridgeSessionRecord.last_seen_at,
3339
+ )
3340
+ .where(startup_purge_filter)
3341
+ .order_by(HttpBridgeSessionRecord.last_seen_at.asc())
3342
+ .limit(batch_size)
3343
+ )
3344
+ candidates = list(result.all())
3345
+ session_ids = [candidate.id for candidate in candidates]
3346
+ if not session_ids:
3347
+ return deleted_count
3348
+ # Operation rows are the durable recovery ledger. Never cascade
3349
+ # delete a session that still owns a retained operation, including
3350
+ # completed replayable transcripts; detach it so the next instance
3351
+ # can inspect and take over without losing continuity history.
3352
+ operation_session_ids = set(
3353
+ await self._session.scalars(
3354
+ select(HttpBridgeOperationRecord.session_id).where(
3355
+ HttpBridgeOperationRecord.session_id.in_(session_ids),
3356
+ )
3357
+ )
3358
+ )
3359
+ retained_recovery_ids = {
3360
+ candidate.id
3361
+ for candidate in candidates
3362
+ if candidate.id in operation_session_ids
3363
+ or (
3364
+ candidate.owner_instance_id == instance_id
3365
+ and getattr(candidate, "owner_process_epoch", None) == owner_process_epoch
3366
+ and (
3367
+ ownerless_cutoff is None
3368
+ or to_utc_naive(candidate.last_seen_at) >= to_utc_naive(ownerless_cutoff)
3369
+ )
3370
+ and is_http_bridge_account_neutral_replay(
3371
+ kind=candidate.session_key_kind,
3372
+ key=candidate.session_key_value,
3373
+ )
3374
+ )
3375
+ }
3376
+ async with sqlite_writer_section():
3377
+ ownerless_operation_ids = {
3378
+ candidate.id
3379
+ for candidate in candidates
3380
+ if candidate.id in retained_recovery_ids
3381
+ and candidate.id in operation_session_ids
3382
+ and candidate.owner_instance_id is None
3383
+ }
3384
+ if ownerless_operation_ids:
3385
+ # The ownerless-cutoff predicate is part of the same
3386
+ # startup query. Refresh retained rows so the bounded
3387
+ # loop cannot select them forever while their operation
3388
+ # transcript is awaiting normal retention cleanup.
3389
+ await self._session.execute(
3390
+ update(HttpBridgeSessionRecord)
3391
+ .where(
3392
+ HttpBridgeSessionRecord.id.in_(ownerless_operation_ids),
3393
+ HttpBridgeSessionRecord.owner_instance_id.is_(None),
3394
+ )
3395
+ .values(last_seen_at=now, lease_expires_at=now)
3396
+ )
3397
+ if retained_recovery_ids:
3398
+ # A process can die after recording a submitted
3399
+ # operation but before upstream acknowledges it. Once
3400
+ # startup has fenced and detached that owner's session,
3401
+ # classify those rows as UNKNOWN so the replacement can
3402
+ # enter the normal proof-gated recovery path.
3403
+ operation_retained_session_ids = retained_recovery_ids & operation_session_ids
3404
+ if operation_retained_session_ids:
3405
+ eligible_operation_sessions = set(
3406
+ await self._session.scalars(
3407
+ select(HttpBridgeSessionRecord.id)
3408
+ .where(
3409
+ HttpBridgeSessionRecord.id.in_(operation_retained_session_ids),
3410
+ startup_purge_filter,
3411
+ )
3412
+ .with_for_update()
3413
+ )
3414
+ )
3415
+ await self._session.execute(
3416
+ update(HttpBridgeOperationRecord)
3417
+ .where(
3418
+ HttpBridgeOperationRecord.session_id.in_(eligible_operation_sessions),
3419
+ HttpBridgeOperationRecord.state == "submitted",
3420
+ )
3421
+ .values(state="unknown", updated_at=now)
3422
+ )
3423
+ await self._session.execute(
3424
+ update(HttpBridgeSessionRecord)
3425
+ .where(
3426
+ HttpBridgeSessionRecord.id.in_(retained_recovery_ids),
3427
+ # Detach the rows selected as belonging to the
3428
+ # previous process. With an explicit new epoch,
3429
+ # matching the new epoch here would leave old
3430
+ # retained rows selected forever on every loop.
3431
+ startup_purge_filter,
3432
+ )
3433
+ .values(
3434
+ owner_instance_id=None,
3435
+ lease_expires_at=now,
3436
+ state=HttpBridgeSessionState.DRAINING,
3437
+ closed_at=None,
3438
+ )
3439
+ )
3440
+ deletable_ids = [session_id for session_id in session_ids if session_id not in retained_recovery_ids]
3441
+ if deletable_ids:
3442
+ if owner_process_epoch is None:
3443
+ deleted = await self._session.execute(
3444
+ delete(HttpBridgeSessionRecord)
3445
+ .where(HttpBridgeSessionRecord.id.in_(deletable_ids))
3446
+ .where(startup_purge_filter)
3447
+ .returning(HttpBridgeSessionRecord.id)
3448
+ )
3449
+ deleted_ids = list(deleted.scalars().all())
3450
+ else:
3451
+ previous_process_ids = [
3452
+ candidate.id for candidate in candidates if candidate.owner_instance_id == instance_id
3453
+ ]
3454
+ ownerless_ids = [
3455
+ candidate.id
3456
+ for candidate in candidates
3457
+ if candidate.owner_instance_id is None and candidate.id not in retained_recovery_ids
3458
+ ]
3459
+ retired_ids: list[str] = []
3460
+ if previous_process_ids:
3461
+ retired = await self._session.execute(
3462
+ update(HttpBridgeSessionRecord)
3463
+ .where(HttpBridgeSessionRecord.id.in_(previous_process_ids))
3464
+ .where(
3465
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
3466
+ or_(
3467
+ HttpBridgeSessionRecord.owner_process_epoch.is_(None),
3468
+ HttpBridgeSessionRecord.owner_process_epoch != owner_process_epoch,
3469
+ ),
3470
+ )
3471
+ .values(
3472
+ owner_instance_id=None,
3473
+ lease_expires_at=None,
3474
+ state=HttpBridgeSessionState.CLOSED,
3475
+ closed_at=now,
3476
+ last_seen_at=now,
3477
+ latest_turn_state=None,
3478
+ latest_response_id=None,
3479
+ latest_input_item_count=None,
3480
+ latest_input_full_fingerprint=None,
3481
+ latest_pending_tool_calls_json=None,
3482
+ )
3483
+ .returning(HttpBridgeSessionRecord.id)
3484
+ )
3485
+ retired_ids = list(retired.scalars().all())
3486
+ deleted_ownerless_ids: list[str] = []
3487
+ if ownerless_ids:
3488
+ deleted_ownerless = await self._session.execute(
3489
+ delete(HttpBridgeSessionRecord)
3490
+ .where(HttpBridgeSessionRecord.id.in_(ownerless_ids))
3491
+ .where(
3492
+ HttpBridgeSessionRecord.owner_instance_id.is_(None),
3493
+ HttpBridgeSessionRecord.state.in_(
3494
+ (HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING),
3495
+ ),
3496
+ or_(
3497
+ HttpBridgeSessionRecord.lease_expires_at.is_(None),
3498
+ HttpBridgeSessionRecord.lease_expires_at < now,
3499
+ ),
3500
+ HttpBridgeSessionRecord.last_seen_at < ownerless_cutoff
3501
+ if ownerless_cutoff is not None
3502
+ else true(),
3503
+ )
3504
+ .returning(HttpBridgeSessionRecord.id)
3505
+ )
3506
+ deleted_ownerless_ids = list(deleted_ownerless.scalars().all())
3507
+ deleted_ids = retired_ids + deleted_ownerless_ids
3508
+ else:
3509
+ deleted_ids = []
3510
+ if deleted_ids:
3511
+ await self._session.execute(
3512
+ delete(HttpBridgeSessionAlias).where(HttpBridgeSessionAlias.session_id.in_(deleted_ids))
3513
+ )
3514
+ await self._session.commit()
3515
+ deleted_count += len(deleted_ids)
3516
+
3517
+ async def purge_closed_before(self, cutoff: datetime, *, batch_size: int = _PURGE_CLOSED_BATCH_SIZE) -> int:
3518
+ deleted_count = 0
3519
+ while True:
3520
+ result = await self._session.execute(
3521
+ select(HttpBridgeSessionRecord.id)
3522
+ .where(
3523
+ HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED,
3524
+ HttpBridgeSessionRecord.last_seen_at < cutoff,
3525
+ ~exists(
3526
+ select(HttpBridgeOperationRecord.operation_id).where(
3527
+ HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
3528
+ )
3529
+ ),
3530
+ )
3531
+ .order_by(HttpBridgeSessionRecord.last_seen_at.asc())
3532
+ .limit(batch_size)
3533
+ )
3534
+ session_ids = list(result.scalars().all())
3535
+ if not session_ids:
3536
+ return deleted_count
3537
+ async with sqlite_writer_section():
3538
+ await self._session.execute(
3539
+ delete(HttpBridgeSessionAlias).where(
3540
+ HttpBridgeSessionAlias.session_id.in_(
3541
+ select(HttpBridgeSessionRecord.id).where(
3542
+ HttpBridgeSessionRecord.id.in_(session_ids),
3543
+ HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED,
3544
+ HttpBridgeSessionRecord.last_seen_at < cutoff,
3545
+ ~exists(
3546
+ select(HttpBridgeOperationRecord.operation_id).where(
3547
+ HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
3548
+ )
3549
+ ),
3550
+ )
3551
+ )
3552
+ )
3553
+ )
3554
+ deleted = await self._session.execute(
3555
+ delete(HttpBridgeSessionRecord)
3556
+ .where(HttpBridgeSessionRecord.id.in_(session_ids))
3557
+ .where(HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED)
3558
+ .where(HttpBridgeSessionRecord.last_seen_at < cutoff)
3559
+ .where(
3560
+ ~exists(
3561
+ select(HttpBridgeOperationRecord.operation_id).where(
3562
+ HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
3563
+ )
3564
+ )
3565
+ )
3566
+ .returning(HttpBridgeSessionRecord.id)
3567
+ )
3568
+ await self._session.commit()
3569
+ deleted_count += len(deleted.scalars().all())
3570
+
3571
+ async def retire_continuity_owner_if_unavailable(
3572
+ self,
3573
+ session_id: str,
3574
+ *,
3575
+ expected_account_id: str,
3576
+ recovery_deadline_epoch: int,
3577
+ ) -> bool:
3578
+ """Retire one row's owner now, when it cannot return before the deadline.
3579
+
3580
+ The scheduled sweep frees a thread six hours after its last turn. This
3581
+ is the request-path counterpart, and it asks a narrower question: can
3582
+ this owner come back before the request that is waiting on it gives up?
3583
+ ``reset_at`` is the answer for a rate or quota limit; ``paused``,
3584
+ ``reauth_required`` and ``deactivated`` carry no horizon at all, so the
3585
+ answer for them is always no.
3586
+
3587
+ Mirrors ``StickySessionsRepository.abandon_legacy_session_header_owner_if_unavailable``,
3588
+ including why the account row is locked first: PostgreSQL evaluates the
3589
+ status subquery from the UPDATE's snapshot, so without the lock a
3590
+ concurrent recovery can commit while this statement waits on the
3591
+ session row and the stale snapshot would still authorize a retirement.
3592
+ The status predicate stays inside the UPDATE as a second, database-level
3593
+ invariant so a later refactor cannot turn a prior observation into an
3594
+ unconditional write.
3595
+
3596
+ Writes the scope marker alone and leaves the timestamp NULL, so a
3597
+ replica running the previous build keeps treating ``account_id`` as
3598
+ hard ownership for the rest of a rolling deploy.
3599
+ """
3600
+ if not session_id or not expected_account_id:
3601
+ return False
3602
+ owner_status_lock = (
3603
+ select(Account.status, Account.reset_at).where(Account.id == expected_account_id).with_for_update()
3604
+ )
3605
+ unavailable_owner = select(Account.id).where(
3606
+ Account.id == expected_account_id,
3607
+ Account.status.in_(HARD_OWNER_UNAVAILABLE_STATUSES),
3608
+ or_(Account.reset_at.is_(None), Account.reset_at >= recovery_deadline_epoch),
3609
+ )
3610
+ statement = (
3611
+ update(HttpBridgeSessionRecord)
3612
+ .where(
3613
+ HttpBridgeSessionRecord.id == session_id,
3614
+ HttpBridgeSessionRecord.account_id == expected_account_id,
3615
+ HttpBridgeSessionRecord.continuity_abandoned_at.is_(None),
3616
+ HttpBridgeSessionRecord.continuity_abandonment_scope.is_(None),
3617
+ HttpBridgeSessionRecord.account_id.in_(unavailable_owner),
3618
+ )
3619
+ .values(continuity_abandonment_scope=_REQUEST_PATH_ABANDONMENT_SCOPE)
3620
+ .returning(HttpBridgeSessionRecord.id)
3621
+ )
3622
+ async with sqlite_writer_section():
3623
+ locked = (await self._session.execute(owner_status_lock)).one_or_none()
3624
+ if locked is None or locked[0] not in HARD_OWNER_UNAVAILABLE_STATUSES:
3625
+ await self._session.commit()
3626
+ return False
3627
+ owner_reset_at = locked[1]
3628
+ if owner_reset_at is not None and owner_reset_at < recovery_deadline_epoch:
3629
+ # The owner is expected back inside the window the caller is
3630
+ # willing to wait, so waiting keeps the upstream prompt cache
3631
+ # instead of forcing a full resend onto another account.
3632
+ await self._session.commit()
3633
+ return False
3634
+ result = await self._session.execute(statement)
3635
+ await self._session.commit()
3636
+ return result.scalar_one_or_none() is not None
3637
+
3638
+ async def retire_stale_unavailable_bridge_owners(self, cutoff: datetime, *, now: datetime) -> int:
3639
+ """Retire continuity owners that have been unroutable since ``cutoff``.
3640
+
3641
+ The durable-bridge counterpart of
3642
+ ``StickySessionsRepository.purge_stale_hard_codex_session_mappings``,
3643
+ and it exists for the same reason. Deleting the row outright would be
3644
+ indistinguishable from "this key was never seen", which leaves the
3645
+ anchored lookup failing closed forever; a tombstone instead says the
3646
+ owner was deliberately abandoned, so a later request may pick a fresh
3647
+ one. Two phases:
3648
+
3649
+ 1. Tombstone a row whose owner has sat in
3650
+ :data:`HARD_OWNER_UNAVAILABLE_STATUSES` past its reset horizon while
3651
+ the row itself went untouched for the whole grace window.
3652
+ 2. Delete a tombstone that then sat another full window with nobody
3653
+ claiming it, as long as it owns no operation rows.
3654
+
3655
+ Unlike the row deletes elsewhere in this module, phase 1 is **not**
3656
+ gated on ``~exists(operation)``. That guard protects the durable
3657
+ recovery ledger from a ``CASCADE``, which retirement does not trigger:
3658
+ it writes two columns and keeps every operation row intact. Gating it
3659
+ would reproduce the 2026-09-04 outage, where every poisoned row
3660
+ happened to own operations and so was never reachable by cleanup.
3661
+ """
3662
+ now_naive = to_utc_naive(now)
3663
+ cutoff_naive = to_utc_naive(cutoff)
3664
+ now_epoch = naive_utc_to_epoch(now_naive)
3665
+ unavailable_account_ids = select(Account.id).where(
3666
+ Account.status.in_(HARD_OWNER_UNAVAILABLE_STATUSES),
3667
+ or_(Account.reset_at.is_(None), Account.reset_at < now_epoch),
3668
+ )
3669
+ tombstone_stmt = (
3670
+ update(HttpBridgeSessionRecord)
3671
+ .where(
3672
+ HttpBridgeSessionRecord.continuity_abandoned_at.is_(None),
3673
+ HttpBridgeSessionRecord.last_seen_at < cutoff_naive,
3674
+ or_(
3675
+ # A request-path retirement wrote the scope marker alone.
3676
+ # Promote it once the row goes stale, exactly as the sticky
3677
+ # sweep promotes its own younger scoped marker: without
3678
+ # this, such a row satisfies neither phase — phase 1 wants
3679
+ # both columns NULL and phase 2 wants a timestamp — and
3680
+ # would sit in the table forever.
3681
+ HttpBridgeSessionRecord.continuity_abandonment_scope.is_not(None),
3682
+ and_(
3683
+ HttpBridgeSessionRecord.continuity_abandonment_scope.is_(None),
3684
+ HttpBridgeSessionRecord.account_id.is_not(None),
3685
+ HttpBridgeSessionRecord.account_id.in_(unavailable_account_ids),
3686
+ ),
3687
+ ),
3688
+ )
3689
+ # Timestamp with NULL scope is the global form: this sweep has no
3690
+ # per-source question to answer, and a replica that predates the
3691
+ # scope column still understands the timestamp.
3692
+ .values(continuity_abandoned_at=now_naive, continuity_abandonment_scope=None)
3693
+ .returning(HttpBridgeSessionRecord.id)
3694
+ )
3695
+ expired_tombstone_filter = (
3696
+ HttpBridgeSessionRecord.continuity_abandoned_at.is_not(None),
3697
+ HttpBridgeSessionRecord.continuity_abandonment_scope.is_(None),
3698
+ HttpBridgeSessionRecord.continuity_abandoned_at < cutoff_naive,
3699
+ ~exists(
3700
+ select(HttpBridgeOperationRecord.operation_id).where(
3701
+ HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
3702
+ )
3703
+ ),
3704
+ )
3705
+ async with sqlite_writer_section():
3706
+ tombstoned = len((await self._session.execute(tombstone_stmt)).scalars().all())
3707
+ # Aliases first, matching ``purge_closed_before``: the FK cascade
3708
+ # would cover it, but SQLite builds without foreign-key enforcement
3709
+ # would leave orphans that later resolve to a deleted session.
3710
+ await self._session.execute(
3711
+ delete(HttpBridgeSessionAlias).where(
3712
+ HttpBridgeSessionAlias.session_id.in_(
3713
+ select(HttpBridgeSessionRecord.id).where(*expired_tombstone_filter)
3714
+ )
3715
+ )
3716
+ )
3717
+ deleted = len(
3718
+ (
3719
+ await self._session.execute(
3720
+ delete(HttpBridgeSessionRecord)
3721
+ .where(*expired_tombstone_filter)
3722
+ .returning(HttpBridgeSessionRecord.id)
3723
+ )
3724
+ )
3725
+ .scalars()
3726
+ .all()
3727
+ )
3728
+ await self._session.commit()
3729
+ return tombstoned + deleted
3730
+
3731
+ async def purge_abandoned_before(self, cutoff: datetime, *, batch_size: int = _PURGE_CLOSED_BATCH_SIZE) -> int:
3732
+ """Purge ACTIVE/DRAINING rows whose lease expired and whose activity predates the cutoff."""
3733
+
3734
+ deleted_count = 0
3735
+ while True:
3736
+ now = utcnow()
3737
+ abandoned_filter = (
3738
+ HttpBridgeSessionRecord.state.in_((HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING)),
3739
+ or_(
3740
+ HttpBridgeSessionRecord.lease_expires_at.is_(None),
3741
+ HttpBridgeSessionRecord.lease_expires_at < now,
3742
+ ),
3743
+ HttpBridgeSessionRecord.last_seen_at < cutoff,
3744
+ ~exists(
3745
+ select(HttpBridgeOperationRecord.operation_id).where(
3746
+ HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
3747
+ )
3748
+ ),
3749
+ )
3750
+ result = await self._session.execute(
3751
+ select(HttpBridgeSessionRecord.id)
3752
+ .where(*abandoned_filter)
3753
+ .order_by(HttpBridgeSessionRecord.last_seen_at.asc())
3754
+ .limit(batch_size)
3755
+ )
3756
+ session_ids = list(result.scalars().all())
3757
+ if not session_ids:
3758
+ return deleted_count
3759
+ async with sqlite_writer_section():
3760
+ await self._session.execute(
3761
+ delete(HttpBridgeSessionAlias).where(
3762
+ HttpBridgeSessionAlias.session_id.in_(
3763
+ select(HttpBridgeSessionRecord.id).where(
3764
+ HttpBridgeSessionRecord.id.in_(session_ids),
3765
+ *abandoned_filter,
3766
+ )
3767
+ )
3768
+ )
3769
+ )
3770
+ deleted = await self._session.execute(
3771
+ delete(HttpBridgeSessionRecord)
3772
+ .where(HttpBridgeSessionRecord.id.in_(session_ids))
3773
+ .where(*abandoned_filter)
3774
+ .returning(HttpBridgeSessionRecord.id)
3775
+ )
3776
+ await self._session.commit()
3777
+ deleted_count += len(deleted.scalars().all())
3778
+
3779
+ async def purge_retry_circuits_before(
3780
+ self,
3781
+ cutoff_epoch: float,
3782
+ *,
3783
+ tombstone_cutoff_epoch: float | None = None,
3784
+ batch_size: int = _PURGE_CLOSED_BATCH_SIZE,
3785
+ ) -> int:
3786
+ # An anchor_abandoned tombstone guards continuity that outlives the
3787
+ # circuit TTL: reaping it on the circuit schedule would let the
3788
+ # unanchored-delta gate dispatch a context-free request. Tombstones
3789
+ # are purged only past the caller's bridge-retention cutoff, and
3790
+ # preserved entirely when no cutoff is given.
3791
+ non_tombstone = HttpBridgeRetryCircuit.last_detail.is_(None) | (
3792
+ HttpBridgeRetryCircuit.last_detail != _RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL
3793
+ )
3794
+ # A replay claim advances only the admission generation and
3795
+ # deliberately leaves the timestamp unchanged, so a claimed
3796
+ # generation near the TTL could be reaped while its replay
3797
+ # is still in flight. Ever-claimed rows get one extra TTL of
3798
+ # grace before the scheduled purge takes them.
3799
+ stale_predicate = (
3800
+ non_tombstone
3801
+ & (HttpBridgeRetryCircuit.updated_at_epoch < cutoff_epoch)
3802
+ & (
3803
+ (HttpBridgeRetryCircuit.admission_generation == 0)
3804
+ | (
3805
+ HttpBridgeRetryCircuit.updated_at_epoch
3806
+ < cutoff_epoch - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
3807
+ )
3808
+ )
3809
+ )
3810
+ if tombstone_cutoff_epoch is not None:
3811
+ # A crash between a poison settle and its registration leaves a
3812
+ # live session storing the poisoned continuity while delta-only
3813
+ # requests keep refreshing its lease; the tombstone's own
3814
+ # updated_at_epoch stays fixed, so an age cutoff alone would
3815
+ # reap the fail-closed fence out from under that live session.
3816
+ # The tombstone falls only when no session still resolves the
3817
+ # key with continuity to protect.
3818
+ session_continuity_exists = (
3819
+ select(HttpBridgeSessionRecord.id)
3820
+ .where(
3821
+ HttpBridgeSessionRecord.session_key_kind == HttpBridgeRetryCircuit.session_key_kind,
3822
+ HttpBridgeSessionRecord.session_key_hash == HttpBridgeRetryCircuit.session_key_hash,
3823
+ HttpBridgeSessionRecord.api_key_scope == HttpBridgeRetryCircuit.api_key_scope,
3824
+ (HttpBridgeSessionRecord.latest_response_id.is_not(None))
3825
+ | (HttpBridgeSessionRecord.latest_turn_state.is_not(None)),
3826
+ )
3827
+ .exists()
3828
+ )
3829
+ alias_continuity_exists = (
3830
+ select(HttpBridgeSessionAlias.id)
3831
+ .join(
3832
+ HttpBridgeSessionRecord,
3833
+ HttpBridgeSessionAlias.session_id == HttpBridgeSessionRecord.id,
3834
+ )
3835
+ .where(
3836
+ HttpBridgeSessionAlias.alias_kind == HttpBridgeRetryCircuit.session_key_kind,
3837
+ HttpBridgeSessionAlias.alias_hash == HttpBridgeRetryCircuit.session_key_hash,
3838
+ HttpBridgeSessionAlias.api_key_scope == HttpBridgeRetryCircuit.api_key_scope,
3839
+ (HttpBridgeSessionRecord.latest_response_id.is_not(None))
3840
+ | (HttpBridgeSessionRecord.latest_turn_state.is_not(None)),
3841
+ )
3842
+ .exists()
3843
+ )
3844
+ live_continuity_exists = session_continuity_exists | alias_continuity_exists
3845
+ stale_predicate = stale_predicate | (
3846
+ (HttpBridgeRetryCircuit.last_detail == _RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL)
3847
+ & (HttpBridgeRetryCircuit.updated_at_epoch < tombstone_cutoff_epoch)
3848
+ & ~live_continuity_exists
3849
+ )
3850
+ deleted_count = 0
3851
+ while True:
3852
+ result = await self._session.execute(
3853
+ select(
3854
+ HttpBridgeRetryCircuit.session_key_kind,
3855
+ HttpBridgeRetryCircuit.session_key_hash,
3856
+ HttpBridgeRetryCircuit.api_key_scope,
3857
+ )
3858
+ .where(stale_predicate)
3859
+ .limit(batch_size)
3860
+ )
3861
+ keys = [tuple(row) for row in result.fetchall()]
3862
+ if not keys:
3863
+ return deleted_count
3864
+ batch_deleted_count = 0
3865
+ async with sqlite_writer_section():
3866
+ for session_key_kind, session_key_hash, api_key_scope in keys:
3867
+ deleted = await self._session.execute(
3868
+ delete(HttpBridgeRetryCircuit)
3869
+ .where(HttpBridgeRetryCircuit.session_key_kind == session_key_kind)
3870
+ .where(HttpBridgeRetryCircuit.session_key_hash == session_key_hash)
3871
+ .where(HttpBridgeRetryCircuit.api_key_scope == api_key_scope)
3872
+ .where(stale_predicate)
3873
+ .returning(HttpBridgeRetryCircuit.session_key_hash)
3874
+ )
3875
+ batch_deleted_count += len(deleted.scalars().all())
3876
+ await self._session.commit()
3877
+ if batch_deleted_count == 0:
3878
+ return deleted_count
3879
+ deleted_count += batch_deleted_count
3880
+
3881
+ async def upsert_alias(
3882
+ self,
3883
+ *,
3884
+ session_id: str,
3885
+ alias_kind: str,
3886
+ alias_value: str,
3887
+ api_key_scope: str,
3888
+ ) -> None:
3889
+ async with sqlite_writer_section():
3890
+ await self._execute_alias_upsert(
3891
+ session_id=session_id,
3892
+ alias_kind=alias_kind,
3893
+ alias_value=alias_value,
3894
+ api_key_scope=api_key_scope,
3895
+ )
3896
+ await self._session.commit()
3897
+
3898
+ async def register_owned_alias(
3899
+ self,
3900
+ *,
3901
+ session_id: str,
3902
+ api_key_scope: str,
3903
+ instance_id: str,
3904
+ owner_epoch: int,
3905
+ alias_kind: str,
3906
+ alias_value: str,
3907
+ lease_ttl_seconds: float,
3908
+ latest_turn_state: str | None = None,
3909
+ latest_response_id: str | None = None,
3910
+ latest_input_item_count: int | None = None,
3911
+ latest_input_full_fingerprint: str | None = None,
3912
+ latest_pending_tool_calls: Mapping[str, str] | None = None,
3913
+ ) -> DurableBridgeAliasRegistration:
3914
+ """Register continuity only while the caller still owns the durable row."""
3915
+
3916
+ async with sqlite_writer_section():
3917
+ now = utcnow()
3918
+ session_values: dict[str, object] = {
3919
+ "lease_expires_at": now + timedelta(seconds=max(1.0, lease_ttl_seconds)),
3920
+ "last_seen_at": now,
3921
+ }
3922
+ if latest_turn_state is not None:
3923
+ session_values["latest_turn_state"] = latest_turn_state
3924
+ if latest_response_id is not None:
3925
+ session_values["latest_response_id"] = latest_response_id
3926
+ session_values["latest_input_item_count"] = latest_input_item_count
3927
+ session_values["latest_input_full_fingerprint"] = latest_input_full_fingerprint
3928
+ session_values["latest_pending_tool_calls_json"] = _encode_pending_tool_calls(
3929
+ latest_response_id,
3930
+ latest_pending_tool_calls,
3931
+ )
3932
+ elif latest_input_item_count is not None and latest_input_full_fingerprint is not None:
3933
+ session_values["latest_input_item_count"] = latest_input_item_count
3934
+ session_values["latest_input_full_fingerprint"] = latest_input_full_fingerprint
3935
+
3936
+ fenced_update = await self._session.execute(
3937
+ update(HttpBridgeSessionRecord)
3938
+ .where(
3939
+ HttpBridgeSessionRecord.id == session_id,
3940
+ HttpBridgeSessionRecord.api_key_scope == api_key_scope,
3941
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
3942
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
3943
+ )
3944
+ .values(**session_values)
3945
+ .returning(
3946
+ HttpBridgeSessionRecord.id,
3947
+ HttpBridgeSessionRecord.session_key_kind,
3948
+ HttpBridgeSessionRecord.session_key_value,
3949
+ )
3950
+ )
3951
+ target = fenced_update.one_or_none()
3952
+ if target is None:
3953
+ return DurableBridgeAliasRegistration.OWNER_FENCED
3954
+
3955
+ registered = await self._execute_alias_upsert(
3956
+ session_id=session_id,
3957
+ alias_kind=alias_kind,
3958
+ alias_value=alias_value,
3959
+ api_key_scope=api_key_scope,
3960
+ target_account_neutral_replay=is_http_bridge_account_neutral_replay(
3961
+ kind=target.session_key_kind,
3962
+ key=target.session_key_value,
3963
+ ),
3964
+ )
3965
+ if not registered:
3966
+ await self._session.rollback()
3967
+ return DurableBridgeAliasRegistration.ALIAS_PROTECTED
3968
+ await self._session.commit()
3969
+ return DurableBridgeAliasRegistration.REGISTERED
3970
+
3971
+ async def register_reversible_turn_state_alias(
3972
+ self,
3973
+ *,
3974
+ session_id: str,
3975
+ api_key_scope: str,
3976
+ instance_id: str,
3977
+ owner_epoch: int,
3978
+ turn_state: str,
3979
+ lease_ttl_seconds: float,
3980
+ ) -> DurableBridgeAliasRegistrationReceipt:
3981
+ """Publish a pre-dispatch turn alias with enough state for exact rollback."""
3982
+
3983
+ alias_kind = "turn_state"
3984
+ async with sqlite_writer_section():
3985
+ now = utcnow()
3986
+ # The first UPDATE both fences ownership and acquires the target-row
3987
+ # lock (and SQLite's writer lock) before prior alias state is read.
3988
+ fenced_lock = await self._session.execute(
3989
+ update(HttpBridgeSessionRecord)
3990
+ .where(
3991
+ HttpBridgeSessionRecord.id == session_id,
3992
+ HttpBridgeSessionRecord.api_key_scope == api_key_scope,
3993
+ HttpBridgeSessionRecord.owner_instance_id == instance_id,
3994
+ HttpBridgeSessionRecord.owner_epoch == owner_epoch,
3995
+ )
3996
+ .values(
3997
+ lease_expires_at=now + timedelta(seconds=max(1.0, lease_ttl_seconds)),
3998
+ last_seen_at=now,
3999
+ )
4000
+ .returning(
4001
+ HttpBridgeSessionRecord.id,
4002
+ HttpBridgeSessionRecord.session_key_kind,
4003
+ HttpBridgeSessionRecord.session_key_value,
4004
+ HttpBridgeSessionRecord.latest_turn_state,
4005
+ )
4006
+ )
4007
+ target = fenced_lock.one_or_none()
4008
+ if target is None:
4009
+ await self._session.rollback()
4010
+ return DurableBridgeAliasRegistrationReceipt(
4011
+ status=DurableBridgeAliasRegistration.OWNER_FENCED,
4012
+ session_id=session_id,
4013
+ api_key_scope=api_key_scope,
4014
+ alias_kind=alias_kind,
4015
+ alias_value=turn_state,
4016
+ instance_id=instance_id,
4017
+ owner_epoch=owner_epoch,
4018
+ previous_alias_session_id=None,
4019
+ previous_alias_owner_epoch=None,
4020
+ previous_alias_account_id=None,
4021
+ previous_latest_turn_state=None,
4022
+ )
4023
+
4024
+ previous_latest_turn_state = target.latest_turn_state
4025
+ previous_alias_session_id = await self._session.scalar(
4026
+ select(HttpBridgeSessionAlias.session_id)
4027
+ .where(
4028
+ HttpBridgeSessionAlias.alias_kind == alias_kind,
4029
+ HttpBridgeSessionAlias.alias_hash == durable_bridge_hash(turn_state),
4030
+ HttpBridgeSessionAlias.alias_value == turn_state,
4031
+ HttpBridgeSessionAlias.api_key_scope == api_key_scope,
4032
+ )
4033
+ .with_for_update()
4034
+ )
4035
+ previous_alias_owner_epoch = None
4036
+ previous_alias_account_id = None
4037
+ if previous_alias_session_id is not None:
4038
+ previous_alias_owner = (
4039
+ await self._session.execute(
4040
+ select(
4041
+ HttpBridgeSessionRecord.owner_epoch,
4042
+ HttpBridgeSessionRecord.account_id,
4043
+ ).where(HttpBridgeSessionRecord.id == previous_alias_session_id)
4044
+ )
4045
+ ).one_or_none()
4046
+ if previous_alias_owner is not None:
4047
+ previous_alias_owner_epoch = previous_alias_owner.owner_epoch
4048
+ previous_alias_account_id = previous_alias_owner.account_id
4049
+ await self._session.execute(
4050
+ update(HttpBridgeSessionRecord)
4051
+ .where(HttpBridgeSessionRecord.id == session_id)
4052
+ .values(latest_turn_state=turn_state)
4053
+ )
4054
+ registered = await self._execute_alias_upsert(
4055
+ session_id=session_id,
4056
+ alias_kind=alias_kind,
4057
+ alias_value=turn_state,
4058
+ api_key_scope=api_key_scope,
4059
+ target_account_neutral_replay=is_http_bridge_account_neutral_replay(
4060
+ kind=target.session_key_kind,
4061
+ key=target.session_key_value,
4062
+ ),
4063
+ )
4064
+ if not registered:
4065
+ await self._session.rollback()
4066
+ return DurableBridgeAliasRegistrationReceipt(
4067
+ status=DurableBridgeAliasRegistration.ALIAS_PROTECTED,
4068
+ session_id=session_id,
4069
+ api_key_scope=api_key_scope,
4070
+ alias_kind=alias_kind,
4071
+ alias_value=turn_state,
4072
+ instance_id=instance_id,
4073
+ owner_epoch=owner_epoch,
4074
+ previous_alias_session_id=previous_alias_session_id,
4075
+ previous_alias_owner_epoch=previous_alias_owner_epoch,
4076
+ previous_alias_account_id=previous_alias_account_id,
4077
+ previous_latest_turn_state=previous_latest_turn_state,
4078
+ )
4079
+ await self._session.commit()
4080
+
4081
+ return DurableBridgeAliasRegistrationReceipt(
4082
+ status=DurableBridgeAliasRegistration.REGISTERED,
4083
+ session_id=session_id,
4084
+ api_key_scope=api_key_scope,
4085
+ alias_kind=alias_kind,
4086
+ alias_value=turn_state,
4087
+ instance_id=instance_id,
4088
+ owner_epoch=owner_epoch,
4089
+ previous_alias_session_id=previous_alias_session_id,
4090
+ previous_alias_owner_epoch=previous_alias_owner_epoch,
4091
+ previous_alias_account_id=previous_alias_account_id,
4092
+ previous_latest_turn_state=previous_latest_turn_state,
4093
+ )
4094
+
4095
+ async def rollback_reversible_turn_state_alias(
4096
+ self,
4097
+ *,
4098
+ receipt: DurableBridgeAliasRegistrationReceipt,
4099
+ ) -> bool:
4100
+ """Undo a registered pre-dispatch alias while the same owner is fenced in."""
4101
+
4102
+ if receipt.status != DurableBridgeAliasRegistration.REGISTERED:
4103
+ return False
4104
+
4105
+ async with sqlite_writer_section():
4106
+ previous_session_valid = False
4107
+ dialect = self._session.get_bind().dialect.name
4108
+ if dialect == "postgresql":
4109
+ session_ids = {receipt.session_id}
4110
+ if receipt.previous_alias_session_id is not None:
4111
+ session_ids.add(receipt.previous_alias_session_id)
4112
+ locked_records = (
4113
+ (
4114
+ await self._session.execute(
4115
+ select(HttpBridgeSessionRecord)
4116
+ .where(HttpBridgeSessionRecord.id.in_(session_ids))
4117
+ .order_by(HttpBridgeSessionRecord.id)
4118
+ .with_for_update()
4119
+ )
4120
+ )
4121
+ .scalars()
4122
+ .all()
4123
+ )
4124
+ records_by_id = {record.id: record for record in locked_records}
4125
+ target_record = records_by_id.get(receipt.session_id)
4126
+ if (
4127
+ target_record is None
4128
+ or target_record.api_key_scope != receipt.api_key_scope
4129
+ or target_record.owner_instance_id != receipt.instance_id
4130
+ or target_record.owner_epoch != receipt.owner_epoch
4131
+ ):
4132
+ await self._session.rollback()
4133
+ return False
4134
+ previous_record = (
4135
+ records_by_id.get(receipt.previous_alias_session_id)
4136
+ if receipt.previous_alias_session_id is not None
4137
+ else None
4138
+ )
4139
+ previous_session_valid = previous_record is not None and (
4140
+ previous_record.owner_epoch == receipt.previous_alias_owner_epoch
4141
+ and previous_record.account_id == receipt.previous_alias_account_id
4142
+ )
4143
+
4144
+ fenced_restore = await self._session.execute(
4145
+ update(HttpBridgeSessionRecord)
4146
+ .where(
4147
+ HttpBridgeSessionRecord.id == receipt.session_id,
4148
+ HttpBridgeSessionRecord.api_key_scope == receipt.api_key_scope,
4149
+ HttpBridgeSessionRecord.owner_instance_id == receipt.instance_id,
4150
+ HttpBridgeSessionRecord.owner_epoch == receipt.owner_epoch,
4151
+ )
4152
+ .values(
4153
+ latest_turn_state=case(
4154
+ (
4155
+ HttpBridgeSessionRecord.latest_turn_state == receipt.alias_value,
4156
+ receipt.previous_latest_turn_state,
4157
+ ),
4158
+ else_=HttpBridgeSessionRecord.latest_turn_state,
4159
+ )
4160
+ )
4161
+ .returning(HttpBridgeSessionRecord.id)
4162
+ )
4163
+ if fenced_restore.scalar_one_or_none() is None:
4164
+ await self._session.rollback()
4165
+ return False
4166
+
4167
+ if dialect != "postgresql" and receipt.previous_alias_session_id is not None:
4168
+ previous_record = (
4169
+ await self._session.execute(
4170
+ select(
4171
+ HttpBridgeSessionRecord.owner_epoch,
4172
+ HttpBridgeSessionRecord.account_id,
4173
+ ).where(HttpBridgeSessionRecord.id == receipt.previous_alias_session_id)
4174
+ )
4175
+ ).one_or_none()
4176
+ previous_session_valid = previous_record is not None and (
4177
+ previous_record.owner_epoch == receipt.previous_alias_owner_epoch
4178
+ and previous_record.account_id == receipt.previous_alias_account_id
4179
+ )
4180
+
4181
+ alias_predicate = (
4182
+ HttpBridgeSessionAlias.session_id == receipt.session_id,
4183
+ HttpBridgeSessionAlias.alias_kind == receipt.alias_kind,
4184
+ HttpBridgeSessionAlias.alias_hash == durable_bridge_hash(receipt.alias_value),
4185
+ HttpBridgeSessionAlias.alias_value == receipt.alias_value,
4186
+ HttpBridgeSessionAlias.api_key_scope == receipt.api_key_scope,
4187
+ )
4188
+ current_alias_session_id = await self._session.scalar(
4189
+ select(HttpBridgeSessionAlias.session_id).where(*alias_predicate).with_for_update()
4190
+ )
4191
+ if current_alias_session_id == receipt.session_id:
4192
+ previous_session_id = receipt.previous_alias_session_id
4193
+ if previous_session_id is None:
4194
+ await self._session.execute(delete(HttpBridgeSessionAlias).where(*alias_predicate))
4195
+ elif previous_session_id != receipt.session_id:
4196
+ if not previous_session_valid:
4197
+ await self._session.execute(delete(HttpBridgeSessionAlias).where(*alias_predicate))
4198
+ else:
4199
+ await self._session.execute(
4200
+ update(HttpBridgeSessionAlias)
4201
+ .where(*alias_predicate)
4202
+ .values(session_id=previous_session_id, updated_at=utcnow())
4203
+ )
4204
+ await self._session.commit()
4205
+ return True
4206
+
4207
+ async def _execute_alias_upsert(
4208
+ self,
4209
+ *,
4210
+ session_id: str,
4211
+ alias_kind: str,
4212
+ alias_value: str,
4213
+ api_key_scope: str,
4214
+ target_account_neutral_replay: bool | None = None,
4215
+ ) -> bool:
4216
+ dialect = self._session.get_bind().dialect.name
4217
+ now = utcnow()
4218
+ values = {
4219
+ "session_id": session_id,
4220
+ "alias_kind": alias_kind,
4221
+ "alias_value": alias_value,
4222
+ "alias_hash": durable_bridge_hash(alias_value),
4223
+ "api_key_scope": api_key_scope,
4224
+ }
4225
+ existing_target_is_account_neutral_replay = HttpBridgeSessionAlias.session_id.in_(
4226
+ select(HttpBridgeSessionRecord.id).where(
4227
+ HttpBridgeSessionRecord.session_key_kind == HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KIND,
4228
+ HttpBridgeSessionRecord.session_key_value.like(f"{HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX}%"),
4229
+ HttpBridgeSessionRecord.session_key_value != HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX,
4230
+ )
4231
+ )
4232
+ existing_target_is_rebindable = HttpBridgeSessionAlias.session_id.in_(
4233
+ select(HttpBridgeSessionRecord.id).where(
4234
+ HttpBridgeSessionRecord.session_key_kind.in_(HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_REBINDABLE_KINDS),
4235
+ )
4236
+ )
4237
+ existing_target_is_replaceable_recovery = HttpBridgeSessionAlias.session_id.in_(
4238
+ select(HttpBridgeSessionRecord.id).where(
4239
+ HttpBridgeSessionRecord.session_key_kind == HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KIND,
4240
+ HttpBridgeSessionRecord.session_key_value.like(f"{HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX}%"),
4241
+ HttpBridgeSessionRecord.session_key_value != HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX,
4242
+ or_(
4243
+ HttpBridgeSessionRecord.owner_instance_id.is_(None),
4244
+ HttpBridgeSessionRecord.lease_expires_at.is_(None),
4245
+ HttpBridgeSessionRecord.lease_expires_at <= now,
4246
+ ),
4247
+ )
4248
+ )
4249
+ conflict_where = None
4250
+ if target_account_neutral_replay is True:
4251
+ conflict_where = or_(
4252
+ HttpBridgeSessionAlias.session_id == session_id,
4253
+ existing_target_is_replaceable_recovery,
4254
+ existing_target_is_rebindable,
4255
+ )
4256
+ elif target_account_neutral_replay is False:
4257
+ conflict_where = or_(
4258
+ HttpBridgeSessionAlias.session_id == session_id,
4259
+ ~existing_target_is_account_neutral_replay,
4260
+ )
4261
+ if dialect == "postgresql":
4262
+ statement = (
4263
+ pg_insert(HttpBridgeSessionAlias)
4264
+ .values(**values)
4265
+ .on_conflict_do_update(
4266
+ index_elements=[
4267
+ HttpBridgeSessionAlias.alias_kind,
4268
+ HttpBridgeSessionAlias.alias_hash,
4269
+ HttpBridgeSessionAlias.api_key_scope,
4270
+ ],
4271
+ set_={
4272
+ "session_id": session_id,
4273
+ "alias_value": alias_value,
4274
+ "updated_at": now,
4275
+ },
4276
+ where=conflict_where,
4277
+ )
4278
+ .returning(HttpBridgeSessionAlias.session_id)
4279
+ )
4280
+ elif dialect == "sqlite":
4281
+ statement = (
4282
+ sqlite_insert(HttpBridgeSessionAlias)
4283
+ .values(**values)
4284
+ .on_conflict_do_update(
4285
+ index_elements=[
4286
+ HttpBridgeSessionAlias.alias_kind,
4287
+ HttpBridgeSessionAlias.alias_hash,
4288
+ HttpBridgeSessionAlias.api_key_scope,
4289
+ ],
4290
+ set_={
4291
+ "session_id": session_id,
4292
+ "alias_value": alias_value,
4293
+ "updated_at": now,
4294
+ },
4295
+ where=conflict_where,
4296
+ )
4297
+ .returning(HttpBridgeSessionAlias.session_id)
4298
+ )
4299
+ else:
4300
+ raise RuntimeError(f"DurableBridgeRepository alias upsert unsupported for dialect={dialect!r}")
4301
+ result = await self._session.execute(statement)
4302
+ return result.scalar_one_or_none() is not None
4303
+
4304
+ async def _clear_aliases_for_session(self, session_id: str) -> None:
4305
+ await self._session.execute(
4306
+ delete(HttpBridgeSessionAlias).where(HttpBridgeSessionAlias.session_id == session_id)
4307
+ )
4308
+
4309
+
4310
+ async def missing_durable_bridge_tables(session: AsyncSession) -> tuple[str, ...]:
4311
+ dialect = session.get_bind().dialect.name
4312
+ expected = set(REQUIRED_DURABLE_BRIDGE_TABLES)
4313
+ if dialect == "sqlite":
4314
+ result = await session.execute(
4315
+ text(
4316
+ "SELECT name FROM sqlite_master "
4317
+ "WHERE type = 'table' "
4318
+ "AND name IN ('http_bridge_sessions', 'http_bridge_session_aliases', 'http_bridge_retry_circuits', "
4319
+ "'http_bridge_recovery_attempts', 'http_bridge_operations', 'http_bridge_operation_events', "
4320
+ "'http_bridge_operation_event_chunks')"
4321
+ )
4322
+ )
4323
+ else:
4324
+ result = await session.execute(
4325
+ text(
4326
+ "SELECT table_name FROM information_schema.tables "
4327
+ "WHERE table_schema = 'public' "
4328
+ "AND table_name IN ("
4329
+ "'http_bridge_sessions', 'http_bridge_session_aliases', 'http_bridge_retry_circuits', "
4330
+ "'http_bridge_recovery_attempts', 'http_bridge_operations', 'http_bridge_operation_events', "
4331
+ "'http_bridge_operation_event_chunks'"
4332
+ ")"
4333
+ )
4334
+ )
4335
+ present = {str(row[0]) for row in result.fetchall()}
4336
+ return tuple(sorted(expected - present))
4337
+
4338
+
4339
+ _SNAPSHOT_COLUMNS = (
4340
+ HttpBridgeSessionRecord.id,
4341
+ HttpBridgeSessionRecord.session_key_kind,
4342
+ HttpBridgeSessionRecord.session_key_value,
4343
+ HttpBridgeSessionRecord.session_key_hash,
4344
+ HttpBridgeSessionRecord.api_key_scope,
4345
+ HttpBridgeSessionRecord.owner_instance_id,
4346
+ HttpBridgeSessionRecord.owner_process_epoch,
4347
+ HttpBridgeSessionRecord.owner_epoch,
4348
+ HttpBridgeSessionRecord.lease_expires_at,
4349
+ HttpBridgeSessionRecord.state,
4350
+ HttpBridgeSessionRecord.account_id,
4351
+ HttpBridgeSessionRecord.model,
4352
+ HttpBridgeSessionRecord.service_tier,
4353
+ HttpBridgeSessionRecord.latest_turn_state,
4354
+ HttpBridgeSessionRecord.latest_response_id,
4355
+ HttpBridgeSessionRecord.latest_input_item_count,
4356
+ HttpBridgeSessionRecord.latest_input_full_fingerprint,
4357
+ HttpBridgeSessionRecord.latest_pending_tool_calls_json,
4358
+ HttpBridgeSessionRecord.last_seen_at,
4359
+ HttpBridgeSessionRecord.closed_at,
4360
+ HttpBridgeSessionRecord.continuity_abandoned_at,
4361
+ HttpBridgeSessionRecord.continuity_abandonment_scope,
4362
+ )
4363
+
4364
+
4365
+ def _returned_row_to_snapshot(row: Row[tuple[object, ...]]) -> DurableBridgeSessionSnapshot:
4366
+ mapping = row._mapping
4367
+ continuity_abandoned = _bridge_continuity_is_abandoned(
4368
+ mapping[HttpBridgeSessionRecord.continuity_abandoned_at],
4369
+ mapping[HttpBridgeSessionRecord.continuity_abandonment_scope],
4370
+ )
4371
+ return DurableBridgeSessionSnapshot(
4372
+ id=mapping[HttpBridgeSessionRecord.id],
4373
+ session_key_kind=mapping[HttpBridgeSessionRecord.session_key_kind],
4374
+ session_key_value=mapping[HttpBridgeSessionRecord.session_key_value],
4375
+ session_key_hash=mapping[HttpBridgeSessionRecord.session_key_hash],
4376
+ api_key_scope=mapping[HttpBridgeSessionRecord.api_key_scope],
4377
+ owner_instance_id=mapping[HttpBridgeSessionRecord.owner_instance_id],
4378
+ owner_process_epoch=mapping[HttpBridgeSessionRecord.owner_process_epoch],
4379
+ owner_epoch=mapping[HttpBridgeSessionRecord.owner_epoch],
4380
+ lease_expires_at=mapping[HttpBridgeSessionRecord.lease_expires_at],
4381
+ state=mapping[HttpBridgeSessionRecord.state],
4382
+ account_id=mapping[HttpBridgeSessionRecord.account_id],
4383
+ model=mapping[HttpBridgeSessionRecord.model],
4384
+ service_tier=mapping[HttpBridgeSessionRecord.service_tier],
4385
+ latest_turn_state=mapping[HttpBridgeSessionRecord.latest_turn_state],
4386
+ latest_response_id=mapping[HttpBridgeSessionRecord.latest_response_id],
4387
+ latest_input_item_count=mapping[HttpBridgeSessionRecord.latest_input_item_count],
4388
+ latest_input_full_fingerprint=mapping[HttpBridgeSessionRecord.latest_input_full_fingerprint],
4389
+ latest_pending_tool_calls=_decode_pending_tool_calls(
4390
+ mapping[HttpBridgeSessionRecord.latest_response_id],
4391
+ mapping[HttpBridgeSessionRecord.latest_pending_tool_calls_json],
4392
+ ),
4393
+ last_seen_at=mapping[HttpBridgeSessionRecord.last_seen_at],
4394
+ closed_at=mapping[HttpBridgeSessionRecord.closed_at],
4395
+ continuity_abandoned=continuity_abandoned,
4396
+ abandoned_account_id=(mapping[HttpBridgeSessionRecord.account_id] if continuity_abandoned else None),
4397
+ )
4398
+
4399
+
4400
+ def _bridge_continuity_is_abandoned(
4401
+ abandoned_at: datetime | None,
4402
+ abandonment_scope: str | None,
4403
+ ) -> bool:
4404
+ """Return whether a writer retired this row's continuity owner.
4405
+
4406
+ Mirrors ``sticky_repository._continuity_is_abandoned_for_source``. The
4407
+ bridge has one continuity source, so any scope marker retires the owner;
4408
+ the legacy-timestamp form retires it globally. Scope is written alone so a
4409
+ pre-migration replica, which knows neither column, keeps treating
4410
+ ``account_id`` as hard ownership through a rolling deploy.
4411
+ """
4412
+ return abandonment_scope is not None or abandoned_at is not None
4413
+
4414
+
4415
+ def _to_snapshot(row: HttpBridgeSessionRecord | None) -> DurableBridgeSessionSnapshot | None:
4416
+ if row is None:
4417
+ return None
4418
+ continuity_abandoned = _bridge_continuity_is_abandoned(
4419
+ row.continuity_abandoned_at,
4420
+ row.continuity_abandonment_scope,
4421
+ )
4422
+ return DurableBridgeSessionSnapshot(
4423
+ id=row.id,
4424
+ session_key_kind=row.session_key_kind,
4425
+ session_key_value=row.session_key_value,
4426
+ session_key_hash=row.session_key_hash,
4427
+ api_key_scope=row.api_key_scope,
4428
+ owner_instance_id=row.owner_instance_id,
4429
+ owner_process_epoch=row.owner_process_epoch,
4430
+ owner_epoch=row.owner_epoch,
4431
+ lease_expires_at=row.lease_expires_at,
4432
+ state=row.state,
4433
+ account_id=row.account_id,
4434
+ model=row.model,
4435
+ service_tier=row.service_tier,
4436
+ latest_turn_state=row.latest_turn_state,
4437
+ latest_response_id=row.latest_response_id,
4438
+ latest_input_item_count=row.latest_input_item_count,
4439
+ latest_input_full_fingerprint=row.latest_input_full_fingerprint,
4440
+ latest_pending_tool_calls=_decode_pending_tool_calls(
4441
+ row.latest_response_id,
4442
+ row.latest_pending_tool_calls_json,
4443
+ ),
4444
+ last_seen_at=row.last_seen_at,
4445
+ closed_at=row.closed_at,
4446
+ continuity_abandoned=continuity_abandoned,
4447
+ abandoned_account_id=row.account_id if continuity_abandoned else None,
4448
+ )
4449
+
4450
+
4451
+ def _to_snapshot_required(row: HttpBridgeSessionRecord) -> DurableBridgeSessionSnapshot:
4452
+ snapshot = _to_snapshot(row)
4453
+ if snapshot is None:
4454
+ raise RuntimeError("Expected durable bridge session snapshot")
4455
+ return snapshot
4456
+
4457
+
4458
+ def _to_recovery_attempt_snapshot(
4459
+ row: HttpBridgeRecoveryAttemptRecord,
4460
+ ) -> DurableBridgeRecoveryAttemptSnapshot:
4461
+ return DurableBridgeRecoveryAttemptSnapshot(
4462
+ session_id=row.session_id,
4463
+ request_fingerprint=row.request_fingerprint,
4464
+ request_id=row.request_id,
4465
+ account_id=row.account_id,
4466
+ model=row.model,
4467
+ replay_safe=bool(row.replay_safe),
4468
+ state=row.state,
4469
+ response_id=row.response_id,
4470
+ )
4471
+
4472
+
4473
+ def _to_operation_snapshot(
4474
+ row: HttpBridgeOperationRecord,
4475
+ *,
4476
+ created: bool = False,
4477
+ rebound: bool = False,
4478
+ rebound_from_session_id: str | None = None,
4479
+ rebound_from_account_id: str | None = None,
4480
+ rebound_from_model: str | None = None,
4481
+ rebound_from_parent_response_id: str | None = None,
4482
+ ) -> DurableBridgeOperationSnapshot:
4483
+ return DurableBridgeOperationSnapshot(
4484
+ operation_id=row.operation_id,
4485
+ session_id=row.session_id,
4486
+ request_fingerprint=row.request_fingerprint,
4487
+ account_id=row.account_id,
4488
+ model=row.model,
4489
+ parent_response_id=row.parent_response_id,
4490
+ state=row.state,
4491
+ response_id=row.response_id,
4492
+ recovery_dispatch_count=row.recovery_dispatch_count,
4493
+ request_text=row.request_text,
4494
+ event_spool_complete=bool(row.event_spool_complete),
4495
+ created=created,
4496
+ rebound=rebound,
4497
+ rebound_from_session_id=rebound_from_session_id,
4498
+ rebound_from_account_id=rebound_from_account_id,
4499
+ rebound_from_model=rebound_from_model,
4500
+ rebound_from_parent_response_id=rebound_from_parent_response_id,
4501
+ )
4502
+
4503
+
4504
+ def _to_retry_circuit_snapshot(row: HttpBridgeRetryCircuit | None) -> DurableBridgeRetryCircuitSnapshot | None:
4505
+ if row is None:
4506
+ return None
4507
+ return DurableBridgeRetryCircuitSnapshot(
4508
+ session_key_kind=row.session_key_kind,
4509
+ session_key_hash=row.session_key_hash,
4510
+ api_key_scope=row.api_key_scope,
4511
+ consecutive_failures=row.consecutive_failures,
4512
+ cooldown_until_epoch=row.cooldown_until_epoch,
4513
+ last_detail=row.last_detail,
4514
+ updated_at_epoch=row.updated_at_epoch,
4515
+ admission_generation=row.admission_generation,
4516
+ )