codexLB-up 1.25.0b10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +10 -0
- app/admin_cli.py +494 -0
- app/cli.py +339 -0
- app/codex_sessions_retag.py +487 -0
- app/core/__init__.py +0 -0
- app/core/audit/__init__.py +0 -0
- app/core/audit/service.py +216 -0
- app/core/audit/types.py +94 -0
- app/core/auth/__init__.py +236 -0
- app/core/auth/api_key_cache.py +65 -0
- app/core/auth/dashboard_access.py +429 -0
- app/core/auth/dashboard_mode.py +171 -0
- app/core/auth/dashboard_session_ttl.py +74 -0
- app/core/auth/dashboard_users_cache.py +120 -0
- app/core/auth/dependencies.py +710 -0
- app/core/auth/external_identity.py +78 -0
- app/core/auth/guardian.py +374 -0
- app/core/auth/models.py +49 -0
- app/core/auth/providers/__init__.py +129 -0
- app/core/auth/providers/oidc.py +838 -0
- app/core/auth/providers/registry.py +118 -0
- app/core/auth/refresh.py +394 -0
- app/core/auth/step_up.py +117 -0
- app/core/auth/totp.py +81 -0
- app/core/balancer/__init__.py +71 -0
- app/core/balancer/logic.py +1612 -0
- app/core/balancer/rendezvous_hash.py +21 -0
- app/core/balancer/types.py +21 -0
- app/core/bootstrap.py +172 -0
- app/core/cache/invalidation.py +426 -0
- app/core/clients/__init__.py +0 -0
- app/core/clients/codex.py +996 -0
- app/core/clients/codex_version.py +187 -0
- app/core/clients/codex_version_snapshot.py +2 -0
- app/core/clients/files.py +534 -0
- app/core/clients/headers.py +29 -0
- app/core/clients/http.py +511 -0
- app/core/clients/model_fetcher.py +276 -0
- app/core/clients/native_egress.py +1304 -0
- app/core/clients/oauth.py +477 -0
- app/core/clients/proxy.py +6089 -0
- app/core/clients/proxy_websocket.py +1411 -0
- app/core/clients/rate_limit_reset_credits.py +412 -0
- app/core/clients/stream_errors.py +12 -0
- app/core/clients/thread_cache_identity.py +347 -0
- app/core/clients/usage.py +467 -0
- app/core/clock.py +170 -0
- app/core/config/__init__.py +0 -0
- app/core/config/background_jobs.py +73 -0
- app/core/config/context_window_overrides.py +166 -0
- app/core/config/dashboard_overrides.py +145 -0
- app/core/config/inheritable.py +47 -0
- app/core/config/key_fingerprint.py +115 -0
- app/core/config/settings.py +872 -0
- app/core/config/settings_cache.py +100 -0
- app/core/config/spool_retention.py +217 -0
- app/core/config/tiers.py +174 -0
- app/core/conversation_archive.py +568 -0
- app/core/crypto.py +37 -0
- app/core/draining.py +11 -0
- app/core/errors.py +365 -0
- app/core/exceptions.py +155 -0
- app/core/handlers/__init__.py +3 -0
- app/core/handlers/exceptions.py +478 -0
- app/core/http_protocol.py +176 -0
- app/core/http_protocol_httptools.py +204 -0
- app/core/ingress_limits.py +17 -0
- app/core/metadata_store.py +18 -0
- app/core/metrics/__init__.py +0 -0
- app/core/metrics/middleware.py +74 -0
- app/core/metrics/prometheus.py +604 -0
- app/core/middleware/__init__.py +25 -0
- app/core/middleware/api_firewall.py +111 -0
- app/core/middleware/app_version.py +37 -0
- app/core/middleware/dashboard_auth_proxy.py +55 -0
- app/core/middleware/dashboard_csrf.py +177 -0
- app/core/middleware/dashboard_gzip.py +41 -0
- app/core/middleware/dashboard_overrides.py +59 -0
- app/core/middleware/firewall_cache.py +91 -0
- app/core/middleware/inflight.py +126 -0
- app/core/middleware/multipart_content_encoding.py +129 -0
- app/core/middleware/path_rewrite.py +148 -0
- app/core/middleware/request_body_limit.py +196 -0
- app/core/middleware/request_decompression.py +209 -0
- app/core/middleware/request_id.py +72 -0
- app/core/middleware/required_capability_http.py +116 -0
- app/core/middleware/trusted_proxy_headers.py +36 -0
- app/core/multipart.py +237 -0
- app/core/multipart_fields.py +91 -0
- app/core/openai/__init__.py +0 -0
- app/core/openai/chat_requests.py +493 -0
- app/core/openai/chat_responses.py +931 -0
- app/core/openai/contracts.py +46 -0
- app/core/openai/exceptions.py +24 -0
- app/core/openai/host_models.py +14 -0
- app/core/openai/images.py +413 -0
- app/core/openai/message_coercion.py +376 -0
- app/core/openai/model_refresh_scheduler.py +423 -0
- app/core/openai/model_registry.py +997 -0
- app/core/openai/model_registry_store.py +548 -0
- app/core/openai/models.py +172 -0
- app/core/openai/parsing.py +97 -0
- app/core/openai/requests.py +1980 -0
- app/core/openai/strict_schema.py +199 -0
- app/core/openai/tool_call_safety.py +99 -0
- app/core/openai/v1_requests.py +187 -0
- app/core/plan_types.py +90 -0
- app/core/prestop.py +202 -0
- app/core/rate_limiter/__init__.py +0 -0
- app/core/rate_limiter/db_rate_limiter.py +215 -0
- app/core/request_locality.py +473 -0
- app/core/resilience/__init__.py +0 -0
- app/core/resilience/backpressure.py +57 -0
- app/core/resilience/bulkhead.py +202 -0
- app/core/resilience/circuit_breaker.py +150 -0
- app/core/resilience/degradation.py +55 -0
- app/core/resilience/loop_lag_monitor.py +60 -0
- app/core/resilience/memory_monitor.py +137 -0
- app/core/resilience/network_recovery.py +256 -0
- app/core/resilience/overload.py +98 -0
- app/core/resilience/toggles.py +114 -0
- app/core/retention/__init__.py +0 -0
- app/core/retention/job.py +240 -0
- app/core/retention/scheduler.py +69 -0
- app/core/runtime_logging.py +601 -0
- app/core/scheduling/__init__.py +0 -0
- app/core/scheduling/leader_election.py +993 -0
- app/core/scheduling/leader_election_handle.py +25 -0
- app/core/server.py +140 -0
- app/core/shutdown.py +326 -0
- app/core/socket_peer.py +22 -0
- app/core/startup.py +41 -0
- app/core/timeout_invariants.py +347 -0
- app/core/tracing/__init__.py +0 -0
- app/core/tracing/otel.py +207 -0
- app/core/types.py +6 -0
- app/core/upstream_proxy/__init__.py +13 -0
- app/core/upstream_proxy/cache.py +130 -0
- app/core/upstream_proxy/resolver.py +189 -0
- app/core/upstream_proxy/types.py +104 -0
- app/core/usage/__init__.py +407 -0
- app/core/usage/depletion.py +109 -0
- app/core/usage/live_hub.py +41 -0
- app/core/usage/live_snapshots.py +174 -0
- app/core/usage/logs.py +212 -0
- app/core/usage/metadata_scheduler.py +118 -0
- app/core/usage/models.py +54 -0
- app/core/usage/pricing.py +576 -0
- app/core/usage/pricing_catalog.py +241 -0
- app/core/usage/pricing_snapshot.json +821 -0
- app/core/usage/quota.py +133 -0
- app/core/usage/refresh_policy.py +21 -0
- app/core/usage/refresh_scheduler.py +713 -0
- app/core/usage/reset_credits_refresh_scheduler.py +435 -0
- app/core/usage/types.py +168 -0
- app/core/utils/__init__.py +0 -0
- app/core/utils/json_guards.py +18 -0
- app/core/utils/locks.py +33 -0
- app/core/utils/masking.py +20 -0
- app/core/utils/proxy_env.py +104 -0
- app/core/utils/request_id.py +65 -0
- app/core/utils/retry.py +77 -0
- app/core/utils/shared_future.py +172 -0
- app/core/utils/sse.py +286 -0
- app/core/utils/time.py +24 -0
- app/db/__init__.py +0 -0
- app/db/account_identity_lock.py +56 -0
- app/db/alembic/__init__.py +1 -0
- app/db/alembic/env.py +70 -0
- app/db/alembic/revision_ids.py +43 -0
- app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
- app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
- app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
- app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
- app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
- app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
- app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
- app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
- app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
- app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
- app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
- app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
- app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
- app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
- app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
- app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
- app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
- app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
- app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
- app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
- app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
- app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
- app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
- app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
- app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
- app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
- app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
- app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
- app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
- app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
- app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
- app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
- app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
- app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
- app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
- app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
- app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
- app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
- app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
- app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
- app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
- app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
- app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
- app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
- app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
- app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
- app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
- app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
- app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
- app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
- app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
- app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
- app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
- app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
- app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
- app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
- app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
- app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
- app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
- app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
- app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
- app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
- app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
- app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
- app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
- app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
- app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
- app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
- app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
- app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
- app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
- app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
- app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
- app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
- app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
- app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
- app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
- app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
- app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
- app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
- app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
- app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
- app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
- app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
- app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
- app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
- app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
- app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
- app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
- app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
- app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
- app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
- app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
- app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
- app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
- app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
- app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
- app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
- app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
- app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
- app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
- app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
- app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
- app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
- app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
- app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
- app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
- app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
- app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
- app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
- app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
- app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
- app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
- app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
- app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
- app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
- app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
- app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
- app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
- app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
- app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
- app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
- app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
- app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
- app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
- app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
- app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
- app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
- app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
- app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
- app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
- app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
- app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
- app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
- app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
- app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
- app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
- app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
- app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
- app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
- app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
- app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
- app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
- app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
- app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
- app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
- app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
- app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
- app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
- app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
- app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
- app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
- app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
- app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
- app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
- app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
- app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
- app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
- app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
- app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
- app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
- app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
- app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
- app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
- app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
- app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
- app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
- app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
- app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
- app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
- app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
- app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
- app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
- app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
- app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
- app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
- app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
- app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
- app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
- app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
- app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
- app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
- app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
- app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
- app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
- app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
- app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
- app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
- app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
- app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
- app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
- app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
- app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
- app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
- app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
- app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
- app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
- app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
- app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
- app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
- app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
- app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
- app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
- app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
- app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
- app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
- app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
- app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
- app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
- app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
- app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
- app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
- app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
- app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
- app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
- app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
- app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
- app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
- app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
- app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
- app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
- app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
- app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
- app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
- app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
- app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
- app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
- app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
- app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
- app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
- app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
- app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
- app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
- app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
- app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
- app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
- app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
- app/db/alembic/versions/__init__.py +1 -0
- app/db/backup.py +64 -0
- app/db/migrate.py +1249 -0
- app/db/migration_lock.py +160 -0
- app/db/migration_url.py +18 -0
- app/db/models.py +3100 -0
- app/db/recover.py +244 -0
- app/db/session.py +1295 -0
- app/db/snapshot.py +46 -0
- app/db/sqlite_lock_retry.py +190 -0
- app/db/sqlite_utils.py +549 -0
- app/dependencies.py +462 -0
- app/main.py +1240 -0
- app/modules/__init__.py +0 -0
- app/modules/accounts/__init__.py +0 -0
- app/modules/accounts/api.py +426 -0
- app/modules/accounts/auth_manager.py +1439 -0
- app/modules/accounts/background_repository.py +160 -0
- app/modules/accounts/deletion.py +540 -0
- app/modules/accounts/mappers.py +654 -0
- app/modules/accounts/refresh_claims.py +352 -0
- app/modules/accounts/repository.py +1679 -0
- app/modules/accounts/schemas.py +308 -0
- app/modules/accounts/service.py +809 -0
- app/modules/accounts/usage_rollup.py +486 -0
- app/modules/accounts/usage_rollup_scheduler.py +75 -0
- app/modules/accounts/usage_time_rollup.py +1037 -0
- app/modules/accounts/usage_time_rollup_read.py +522 -0
- app/modules/api_keys/__init__.py +1 -0
- app/modules/api_keys/api.py +344 -0
- app/modules/api_keys/last_used_coalescer.py +216 -0
- app/modules/api_keys/limit_windows.py +41 -0
- app/modules/api_keys/repository.py +1200 -0
- app/modules/api_keys/reset_scheduler.py +85 -0
- app/modules/api_keys/schemas.py +135 -0
- app/modules/api_keys/service.py +2123 -0
- app/modules/audit/__init__.py +0 -0
- app/modules/audit/api.py +88 -0
- app/modules/audit/repository.py +74 -0
- app/modules/audit/schemas.py +33 -0
- app/modules/audit/service.py +71 -0
- app/modules/auth_providers/__init__.py +0 -0
- app/modules/auth_providers/api.py +127 -0
- app/modules/auth_providers/config.py +343 -0
- app/modules/auth_providers/repository.py +101 -0
- app/modules/auth_providers/schemas.py +88 -0
- app/modules/auth_providers/seed.py +105 -0
- app/modules/auth_providers/service.py +289 -0
- app/modules/automations/__init__.py +1 -0
- app/modules/automations/api.py +369 -0
- app/modules/automations/repository.py +1896 -0
- app/modules/automations/scheduler.py +107 -0
- app/modules/automations/schemas.py +158 -0
- app/modules/automations/service.py +2548 -0
- app/modules/cache_isolation_probe/__init__.py +0 -0
- app/modules/cache_isolation_probe/api.py +188 -0
- app/modules/cache_isolation_probe/prefix.py +144 -0
- app/modules/cache_isolation_probe/schemas.py +94 -0
- app/modules/cache_isolation_probe/sender.py +229 -0
- app/modules/cache_isolation_probe/service.py +525 -0
- app/modules/conversation_archive/__init__.py +1 -0
- app/modules/conversation_archive/api.py +101 -0
- app/modules/conversation_archive/schemas.py +28 -0
- app/modules/conversation_archive/service.py +165 -0
- app/modules/dashboard/__init__.py +1 -0
- app/modules/dashboard/api.py +106 -0
- app/modules/dashboard/builders.py +107 -0
- app/modules/dashboard/repository.py +304 -0
- app/modules/dashboard/schemas.py +148 -0
- app/modules/dashboard/service.py +534 -0
- app/modules/dashboard/timeframes.py +19 -0
- app/modules/dashboard/weekly_pace.py +617 -0
- app/modules/dashboard_auth/__init__.py +1 -0
- app/modules/dashboard_auth/api.py +1344 -0
- app/modules/dashboard_auth/oidc_api.py +753 -0
- app/modules/dashboard_auth/oidc_flows.py +330 -0
- app/modules/dashboard_auth/repository.py +452 -0
- app/modules/dashboard_auth/schemas.py +232 -0
- app/modules/dashboard_auth/service.py +1384 -0
- app/modules/dashboard_roles/__init__.py +0 -0
- app/modules/dashboard_roles/api.py +64 -0
- app/modules/dashboard_roles/repository.py +54 -0
- app/modules/dashboard_roles/schemas.py +29 -0
- app/modules/dashboard_roles/seed.py +80 -0
- app/modules/dashboard_roles/service.py +142 -0
- app/modules/dashboard_users/__init__.py +0 -0
- app/modules/dashboard_users/api.py +278 -0
- app/modules/dashboard_users/break_glass.py +169 -0
- app/modules/dashboard_users/credentials.py +18 -0
- app/modules/dashboard_users/identity_resolver.py +665 -0
- app/modules/dashboard_users/repository.py +725 -0
- app/modules/dashboard_users/schemas.py +117 -0
- app/modules/dashboard_users/service.py +1054 -0
- app/modules/firewall/__init__.py +1 -0
- app/modules/firewall/api.py +78 -0
- app/modules/firewall/repository.py +51 -0
- app/modules/firewall/schemas.py +28 -0
- app/modules/firewall/service.py +93 -0
- app/modules/fleet/__init__.py +1 -0
- app/modules/fleet/api.py +177 -0
- app/modules/fleet/mappers.py +56 -0
- app/modules/fleet/observability.py +372 -0
- app/modules/fleet/schemas.py +119 -0
- app/modules/health/__init__.py +0 -0
- app/modules/health/api.py +246 -0
- app/modules/health/schemas.py +27 -0
- app/modules/limit_warmup/__init__.py +1 -0
- app/modules/limit_warmup/repository.py +159 -0
- app/modules/limit_warmup/service.py +1001 -0
- app/modules/model_sources/__init__.py +1 -0
- app/modules/model_sources/api.py +99 -0
- app/modules/model_sources/catalog.py +307 -0
- app/modules/model_sources/forwarding.py +1435 -0
- app/modules/model_sources/projection.py +55 -0
- app/modules/model_sources/repository.py +201 -0
- app/modules/model_sources/schemas.py +79 -0
- app/modules/model_sources/selection.py +166 -0
- app/modules/model_sources/service.py +236 -0
- app/modules/oauth/__init__.py +0 -0
- app/modules/oauth/api.py +101 -0
- app/modules/oauth/repository.py +238 -0
- app/modules/oauth/schemas.py +49 -0
- app/modules/oauth/service.py +1196 -0
- app/modules/oauth/templates/oauth_success.html +122 -0
- app/modules/proxy/__init__.py +0 -0
- app/modules/proxy/_load_balancer/__init__.py +1 -0
- app/modules/proxy/_load_balancer/error_rate.py +116 -0
- app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
- app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
- app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
- app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
- app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
- app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
- app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
- app/modules/proxy/_load_balancer/tunables.py +97 -0
- app/modules/proxy/_load_balancer/types.py +96 -0
- app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
- app/modules/proxy/_service/__init__.py +6 -0
- app/modules/proxy/_service/api_key_usage.py +745 -0
- app/modules/proxy/_service/clock_budget.py +38 -0
- app/modules/proxy/_service/codex_control.py +611 -0
- app/modules/proxy/_service/compact.py +2211 -0
- app/modules/proxy/_service/file_ops.py +749 -0
- app/modules/proxy/_service/http_bridge/__init__.py +3 -0
- app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
- app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
- app/modules/proxy/_service/http_bridge/activity.py +178 -0
- app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
- app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
- app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
- app/modules/proxy/_service/http_bridge/protocol.py +167 -0
- app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
- app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
- app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
- app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
- app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
- app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
- app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
- app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
- app/modules/proxy/_service/observability.py +410 -0
- app/modules/proxy/_service/rate_limit.py +361 -0
- app/modules/proxy/_service/realtime_live.py +655 -0
- app/modules/proxy/_service/refresh.py +154 -0
- app/modules/proxy/_service/request_log.py +585 -0
- app/modules/proxy/_service/response_create.py +930 -0
- app/modules/proxy/_service/streaming/__init__.py +3 -0
- app/modules/proxy/_service/streaming/helpers.py +1317 -0
- app/modules/proxy/_service/streaming/mixin.py +1100 -0
- app/modules/proxy/_service/streaming/protocol.py +32 -0
- app/modules/proxy/_service/streaming/retry.py +3762 -0
- app/modules/proxy/_service/support.py +2122 -0
- app/modules/proxy/_service/transcribe.py +434 -0
- app/modules/proxy/_service/warmup.py +483 -0
- app/modules/proxy/_service/websocket/__init__.py +3 -0
- app/modules/proxy/_service/websocket/helpers.py +2360 -0
- app/modules/proxy/_service/websocket/mixin.py +7285 -0
- app/modules/proxy/_service/websocket/protocol.py +74 -0
- app/modules/proxy/account_cache.py +229 -0
- app/modules/proxy/account_eligibility.py +87 -0
- app/modules/proxy/additional_model_limits.py +40 -0
- app/modules/proxy/affinity.py +830 -0
- app/modules/proxy/affinity_observation.py +101 -0
- app/modules/proxy/api.py +10254 -0
- app/modules/proxy/api_key_usage.py +113 -0
- app/modules/proxy/cap_partitioning.py +280 -0
- app/modules/proxy/capability_lineage.py +65 -0
- app/modules/proxy/capability_lineage_repository.py +82 -0
- app/modules/proxy/capability_routing.py +253 -0
- app/modules/proxy/continuity.py +112 -0
- app/modules/proxy/downstream_delivery.py +277 -0
- app/modules/proxy/durable_bridge_coordinator.py +1217 -0
- app/modules/proxy/durable_bridge_repository.py +4516 -0
- app/modules/proxy/durable_bridge_runtime.py +9 -0
- app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
- app/modules/proxy/fair_share.py +131 -0
- app/modules/proxy/file_pin_repository.py +210 -0
- app/modules/proxy/helpers.py +439 -0
- app/modules/proxy/http_bridge_event_batcher.py +613 -0
- app/modules/proxy/http_bridge_forwarding.py +797 -0
- app/modules/proxy/http_continuation.py +76 -0
- app/modules/proxy/images_observability.py +62 -0
- app/modules/proxy/images_service.py +1054 -0
- app/modules/proxy/load_balancer.py +3016 -0
- app/modules/proxy/rate_limit_cache.py +46 -0
- app/modules/proxy/replay_safety.py +1063 -0
- app/modules/proxy/repo_bundle.py +31 -0
- app/modules/proxy/request_policy.py +1054 -0
- app/modules/proxy/ring_membership.py +251 -0
- app/modules/proxy/schemas.py +345 -0
- app/modules/proxy/selection_errors.py +41 -0
- app/modules/proxy/service.py +2600 -0
- app/modules/proxy/source_admission.py +125 -0
- app/modules/proxy/source_dispatch.py +999 -0
- app/modules/proxy/sticky_repository.py +831 -0
- app/modules/proxy/thread_anchors.py +491 -0
- app/modules/proxy/tool_call_dedupe.py +802 -0
- app/modules/proxy/types.py +54 -0
- app/modules/proxy/work_admission.py +121 -0
- app/modules/quota_planner/__init__.py +1 -0
- app/modules/quota_planner/api.py +276 -0
- app/modules/quota_planner/logic.py +743 -0
- app/modules/quota_planner/repository.py +722 -0
- app/modules/quota_planner/scheduler.py +212 -0
- app/modules/quota_planner/schemas.py +102 -0
- app/modules/quota_planner/warmup.py +693 -0
- app/modules/rate_limit_reset_credits/__init__.py +0 -0
- app/modules/rate_limit_reset_credits/api.py +601 -0
- app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
- app/modules/rate_limit_reset_credits/store.py +114 -0
- app/modules/reports/__init__.py +1 -0
- app/modules/reports/api.py +134 -0
- app/modules/reports/cache.py +74 -0
- app/modules/reports/filters.py +27 -0
- app/modules/reports/repository.py +525 -0
- app/modules/reports/rollup.py +150 -0
- app/modules/reports/rollup_read.py +148 -0
- app/modules/reports/schemas.py +123 -0
- app/modules/reports/service.py +287 -0
- app/modules/reports/thread_identity.py +256 -0
- app/modules/request_logs/__init__.py +0 -0
- app/modules/request_logs/api.py +224 -0
- app/modules/request_logs/cost_backfill.py +154 -0
- app/modules/request_logs/mappers.py +97 -0
- app/modules/request_logs/repository.py +1765 -0
- app/modules/request_logs/schemas.py +145 -0
- app/modules/request_logs/service.py +348 -0
- app/modules/role_mappings/__init__.py +0 -0
- app/modules/role_mappings/api.py +206 -0
- app/modules/role_mappings/matching.py +55 -0
- app/modules/role_mappings/repository.py +106 -0
- app/modules/role_mappings/schemas.py +67 -0
- app/modules/role_mappings/service.py +334 -0
- app/modules/runtime/__init__.py +1 -0
- app/modules/runtime/api.py +18 -0
- app/modules/runtime/schemas.py +14 -0
- app/modules/runtime/service.py +218 -0
- app/modules/scim/__init__.py +1 -0
- app/modules/scim/api.py +194 -0
- app/modules/scim/dependencies.py +112 -0
- app/modules/scim/errors.py +77 -0
- app/modules/scim/management_api.py +156 -0
- app/modules/scim/repository.py +212 -0
- app/modules/scim/schemas.py +145 -0
- app/modules/scim/service.py +786 -0
- app/modules/scim/tokens.py +43 -0
- app/modules/settings/__init__.py +1 -0
- app/modules/settings/api.py +1677 -0
- app/modules/settings/repository.py +606 -0
- app/modules/settings/schemas.py +496 -0
- app/modules/settings/service.py +769 -0
- app/modules/shared/__init__.py +0 -0
- app/modules/shared/schemas.py +22 -0
- app/modules/sticky_sessions/__init__.py +1 -0
- app/modules/sticky_sessions/api.py +118 -0
- app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
- app/modules/sticky_sessions/schemas.py +78 -0
- app/modules/sticky_sessions/service.py +196 -0
- app/modules/telemetry/__init__.py +5 -0
- app/modules/telemetry/api.py +115 -0
- app/modules/telemetry/clients.py +62 -0
- app/modules/telemetry/consent.py +115 -0
- app/modules/telemetry/scheduler.py +85 -0
- app/modules/telemetry/schemas.py +163 -0
- app/modules/telemetry/sender.py +221 -0
- app/modules/telemetry/snapshot.py +490 -0
- app/modules/usage/__init__.py +0 -0
- app/modules/usage/additional_quota_keys.py +379 -0
- app/modules/usage/api.py +45 -0
- app/modules/usage/background_repository.py +147 -0
- app/modules/usage/builders.py +559 -0
- app/modules/usage/depletion_service.py +277 -0
- app/modules/usage/live_ingest.py +356 -0
- app/modules/usage/mappers.py +29 -0
- app/modules/usage/plan_downgrade_observations.py +488 -0
- app/modules/usage/repository.py +1890 -0
- app/modules/usage/schemas.py +68 -0
- app/modules/usage/service.py +102 -0
- app/modules/usage/updater.py +1488 -0
- app/static/assets/Trans-Cj-J9mVA.js +1 -0
- app/static/assets/access-page-DHzZiXZi.js +1 -0
- app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
- app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
- app/static/assets/account-status-DIsU5sUS.js +1 -0
- app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
- app/static/assets/accounts-page-Bbai8ift.js +3 -0
- app/static/assets/api-Dwa7XxPe.js +1 -0
- app/static/assets/api-client-dugX1Pti.js +55 -0
- app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
- app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
- app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
- app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
- app/static/assets/apis-page-BvXz9c6i.js +2 -0
- app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
- app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
- app/static/assets/automations-page-DQgNAJOR.js +3 -0
- app/static/assets/badge-DqMb9es0.js +1 -0
- app/static/assets/blur-email-HEvKagG2.js +1 -0
- app/static/assets/button-BjVmmiGW.js +1 -0
- app/static/assets/checkbox-apsl9vzk.js +1 -0
- app/static/assets/chevron-left-1d_pHVa5.js +1 -0
- app/static/assets/circle-alert-BS0xd_o1.js +1 -0
- app/static/assets/clsx-DB0hHKMi.js +1 -0
- app/static/assets/colors-BBy8GEtr.js +1 -0
- app/static/assets/copy-button-D7nAvOKR.js +1 -0
- app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
- app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
- app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
- app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
- app/static/assets/donut-chart-Bxc69cra.js +1 -0
- app/static/assets/download-kQteIcsB.js +1 -0
- app/static/assets/ellipsis-B_5OgIre.js +1 -0
- app/static/assets/empty-state-JcA7tc8m.js +1 -0
- app/static/assets/es2015-bMH8YKZT.js +41 -0
- app/static/assets/es6-B264ni6W.js +69 -0
- app/static/assets/external-link-UVAOgMQ0.js +1 -0
- app/static/assets/i18n-BWqZ9Ua2.js +1 -0
- app/static/assets/inbox-B_RM2s1J.js +1 -0
- app/static/assets/index-DIpd264Z.js +23 -0
- app/static/assets/index-DMeLKRJx.css +2 -0
- app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
- app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
- app/static/assets/iso-Bpe5Ihbb.js +1 -0
- app/static/assets/label-CGaLmLNf.js +1 -0
- app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
- app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
- app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
- app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
- app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
- app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
- app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
- app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
- app/static/assets/payload-CIYnGZz9.js +1 -0
- app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
- app/static/assets/report-chart-card-BXDliwcc.js +1 -0
- app/static/assets/reports-page-CGCJeMHG.js +3 -0
- app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
- app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
- app/static/assets/search-HfhqwERt.js +1 -0
- app/static/assets/select-Cl33bz-g.js +1 -0
- app/static/assets/settings-page-mZj01HYl.js +2 -0
- app/static/assets/shim-CZwyM4Q7.js +1 -0
- app/static/assets/skeleton-C_EUc4Rp.js +1 -0
- app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
- app/static/assets/status-badge-CHHA5X6c.js +1 -0
- app/static/assets/switch-CzxgokcM.js +1 -0
- app/static/assets/table-DXaNSQ0G.js +1 -0
- app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
- app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
- app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
- app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
- app/static/assets/trash-D3KyWr1h.js +1 -0
- app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
- app/static/assets/use-accounts-PWlVGw3z.js +1 -0
- app/static/assets/use-auth-CuDlW3iC.js +1 -0
- app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
- app/static/assets/use-model-sources-Bul24JBu.js +1 -0
- app/static/assets/use-models-DUCPUuZg.js +1 -0
- app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
- app/static/assets/useTranslation--56nHhlH.js +1 -0
- app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
- app/static/assets/users-uIC3TIlE.js +1 -0
- app/static/assets/vendor-query-_bHZuxEx.js +1 -0
- app/static/assets/vendor-react-BVL0YZPW.js +9 -0
- app/static/favicon.svg +10 -0
- app/static/fonts/GeistSans-Variable.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
- app/static/index.html +29 -0
- codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
- codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
- codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
- codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
- codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
- config/additional_quota_registry.json +11 -0
|
@@ -0,0 +1,4582 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import math
|
|
7
|
+
import random
|
|
8
|
+
from collections import deque
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
from dataclasses import dataclass, replace
|
|
11
|
+
from typing import Any, Literal, Mapping, cast
|
|
12
|
+
from uuid import uuid4
|
|
13
|
+
|
|
14
|
+
from app.core.clients.files import create_file as core_create_file # noqa: F401
|
|
15
|
+
from app.core.clients.files import finalize_file as core_finalize_file # noqa: F401
|
|
16
|
+
from app.core.clients.proxy import ( # noqa: F401
|
|
17
|
+
CODEX_INSTALLATION_ID_HEADER,
|
|
18
|
+
CODEX_TURN_METADATA_HEADER,
|
|
19
|
+
ImageFetchSession,
|
|
20
|
+
ProxyResponseError,
|
|
21
|
+
UpstreamProxyRouteTrace,
|
|
22
|
+
_as_image_fetch_session,
|
|
23
|
+
_finalize_responses_lite_reasoning_context,
|
|
24
|
+
_historical_agent_control_output_occurrences,
|
|
25
|
+
_inline_content_images,
|
|
26
|
+
_inline_input_image_urls,
|
|
27
|
+
_payload_has_responses_lite_websocket_marker,
|
|
28
|
+
_payload_uses_responses_lite,
|
|
29
|
+
_ws_transport_payload_budget_bytes,
|
|
30
|
+
apply_codex_installation_metadata,
|
|
31
|
+
filter_inbound_headers,
|
|
32
|
+
pop_compact_timeout_overrides,
|
|
33
|
+
pop_stream_timeout_overrides,
|
|
34
|
+
pop_transcribe_timeout_overrides,
|
|
35
|
+
push_compact_timeout_overrides,
|
|
36
|
+
push_stream_timeout_overrides,
|
|
37
|
+
push_transcribe_timeout_overrides,
|
|
38
|
+
)
|
|
39
|
+
from app.core.clients.proxy import CodexControlResponse as CodexControlResponse
|
|
40
|
+
from app.core.clients.proxy import codex_control_request as core_codex_control_request # noqa: F401
|
|
41
|
+
from app.core.clients.proxy import compact_responses as core_compact_responses # noqa: F401
|
|
42
|
+
from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401
|
|
43
|
+
from app.core.clients.proxy_websocket import (
|
|
44
|
+
UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE,
|
|
45
|
+
UpstreamWebSocketTransportError,
|
|
46
|
+
is_account_neutral_websocket_error_code,
|
|
47
|
+
)
|
|
48
|
+
from app.core.clients.thread_cache_identity import (
|
|
49
|
+
THREAD_CACHE_IDENTITY_MODE_SHARED,
|
|
50
|
+
ThreadCacheIdentity,
|
|
51
|
+
apply_thread_cache_identity,
|
|
52
|
+
thread_cache_identity_from,
|
|
53
|
+
)
|
|
54
|
+
from app.core.clock import REAL_CLOCK, Clock, clock_for, scheduler_for
|
|
55
|
+
from app.core.config.dashboard_overrides import with_dashboard_overrides
|
|
56
|
+
from app.core.errors import openai_error
|
|
57
|
+
from app.core.openai.parsing import parse_sse_event
|
|
58
|
+
from app.core.openai.requests import (
|
|
59
|
+
ResponsesRequest,
|
|
60
|
+
)
|
|
61
|
+
from app.core.resilience.overload import is_local_overload_error_code
|
|
62
|
+
from app.core.types import JsonObject, JsonValue
|
|
63
|
+
from app.core.utils.locks import fast_lock
|
|
64
|
+
from app.core.utils.request_id import (
|
|
65
|
+
ensure_request_id,
|
|
66
|
+
ensure_request_scope_id,
|
|
67
|
+
get_request_id,
|
|
68
|
+
reset_request_id,
|
|
69
|
+
set_request_id,
|
|
70
|
+
)
|
|
71
|
+
from app.core.utils.sse import format_sse_event, parse_sse_data_json
|
|
72
|
+
from app.db.models import DashboardSettings, StickySessionKind
|
|
73
|
+
from app.modules.api_keys.service import (
|
|
74
|
+
ApiKeyData,
|
|
75
|
+
ApiKeyUsageReservationData,
|
|
76
|
+
)
|
|
77
|
+
from app.modules.proxy._load_balancer.tunables import RoutingTunables
|
|
78
|
+
from app.modules.proxy._service.api_key_usage import (
|
|
79
|
+
_API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS,
|
|
80
|
+
)
|
|
81
|
+
from app.modules.proxy._service.compact import (
|
|
82
|
+
_sticky_key_for_compact_request as _sticky_key_for_compact_request,
|
|
83
|
+
)
|
|
84
|
+
from app.modules.proxy._service.compact import (
|
|
85
|
+
_sticky_key_from_compact_payload as _sticky_key_from_compact_payload,
|
|
86
|
+
)
|
|
87
|
+
from app.modules.proxy._service.http_bridge import helpers as _http_bridge_helpers
|
|
88
|
+
from app.modules.proxy._service.http_bridge.accepted_replay import (
|
|
89
|
+
_claim_websocket_replay_create_gate,
|
|
90
|
+
_http_bridge_accepted_replay_may_exclude_account,
|
|
91
|
+
)
|
|
92
|
+
from app.modules.proxy._service.http_bridge.helpers import (
|
|
93
|
+
_HTTP_BRIDGE_COOLDOWN_SUPPRESSION_ATTR,
|
|
94
|
+
_HTTP_BRIDGE_PRE_SUBMIT_FAILURE_ATTR,
|
|
95
|
+
_await_task_deferring_cancellation,
|
|
96
|
+
_bind_http_bridge_proxy_injected_anchor,
|
|
97
|
+
_build_http_bridge_prewarm_text,
|
|
98
|
+
_http_bridge_abandonment_may_settle_circuit,
|
|
99
|
+
_http_bridge_client_full_history_recovery_error,
|
|
100
|
+
_http_bridge_denied_anchor_fence_advanced,
|
|
101
|
+
_http_bridge_durable_lease_ttl_seconds,
|
|
102
|
+
_http_bridge_is_previous_response_owner_unavailable,
|
|
103
|
+
_http_bridge_key_strength,
|
|
104
|
+
_http_bridge_payload_looks_like_full_resend,
|
|
105
|
+
_http_bridge_precreated_retry_failure_error,
|
|
106
|
+
_http_bridge_prewarm_enabled,
|
|
107
|
+
_http_bridge_request_budget_seconds,
|
|
108
|
+
_http_bridge_request_counts_against_queue,
|
|
109
|
+
_http_bridge_request_state_holds_safe_replay,
|
|
110
|
+
_http_bridge_retry_circuit_attempt_selection_for_pending_requests,
|
|
111
|
+
_log_http_bridge_event,
|
|
112
|
+
_record_continuity_fail_closed,
|
|
113
|
+
_record_http_bridge_prewarm_outcome,
|
|
114
|
+
_register_http_bridge_turn_state_aliases_locked,
|
|
115
|
+
_release_http_bridge_unanchored_handoff,
|
|
116
|
+
)
|
|
117
|
+
from app.modules.proxy._service.http_bridge.quarantine import (
|
|
118
|
+
_http_bridge_session_key_poison_quarantined,
|
|
119
|
+
_record_http_bridge_quarantine_wedged_pending,
|
|
120
|
+
)
|
|
121
|
+
from app.modules.proxy._service.http_bridge.retry_circuit import (
|
|
122
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
|
|
123
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD,
|
|
124
|
+
_POISON_ANCHOR_CAPTURE_UNAVAILABLE,
|
|
125
|
+
_http_bridge_retry_circuit_suppression_message,
|
|
126
|
+
)
|
|
127
|
+
from app.modules.proxy._service.http_bridge.service_stubs import (
|
|
128
|
+
_call_with_supported_optional_kwargs,
|
|
129
|
+
_classify_upstream_close,
|
|
130
|
+
_count_external_image_urls,
|
|
131
|
+
_enforce_response_create_size_limit,
|
|
132
|
+
_estimated_lease_tokens_from_request_usage_budget,
|
|
133
|
+
_fingerprint_input_items,
|
|
134
|
+
_inline_top_level_input_image_urls,
|
|
135
|
+
_normalize_service_tier_value,
|
|
136
|
+
_normalize_session_id,
|
|
137
|
+
_prepare_websocket_request_state_for_account_switch,
|
|
138
|
+
_prepare_websocket_request_state_for_auth_replay,
|
|
139
|
+
_prepare_websocket_request_state_for_visible_output_replay,
|
|
140
|
+
_prewarm_response_timeout_seconds,
|
|
141
|
+
_release_websocket_response_create_gate,
|
|
142
|
+
_response_create_client_metadata,
|
|
143
|
+
_security_work_advisory_event,
|
|
144
|
+
_service_as_image_fetch_session,
|
|
145
|
+
_service_get_settings,
|
|
146
|
+
_service_get_settings_cache,
|
|
147
|
+
_service_inline_input_image_urls,
|
|
148
|
+
_service_lease_http_session,
|
|
149
|
+
_slim_response_create_payload_for_upstream,
|
|
150
|
+
_upstream_response_create_max_bytes,
|
|
151
|
+
_websocket_auth_failure_permanent_code,
|
|
152
|
+
_websocket_auth_failure_requires_reauth,
|
|
153
|
+
_websocket_request_text_is_account_neutral_fresh_replay,
|
|
154
|
+
)
|
|
155
|
+
from app.modules.proxy._service.http_bridge.upstream_events import (
|
|
156
|
+
_abandon_durable_http_bridge_continuity,
|
|
157
|
+
)
|
|
158
|
+
from app.modules.proxy._service.observability import (
|
|
159
|
+
_hash_identifier as _hash_identifier,
|
|
160
|
+
)
|
|
161
|
+
from app.modules.proxy._service.observability import (
|
|
162
|
+
_hash_identifier_or_none as _hash_identifier_or_none,
|
|
163
|
+
)
|
|
164
|
+
from app.modules.proxy._service.observability import (
|
|
165
|
+
_interesting_header_keys as _interesting_header_keys,
|
|
166
|
+
)
|
|
167
|
+
from app.modules.proxy._service.observability import (
|
|
168
|
+
_tools_hash as _tools_hash,
|
|
169
|
+
)
|
|
170
|
+
from app.modules.proxy._service.observability import (
|
|
171
|
+
_truncate_identifier as _truncate_identifier,
|
|
172
|
+
)
|
|
173
|
+
from app.modules.proxy._service.support import (
|
|
174
|
+
_ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE,
|
|
175
|
+
_HARD_HTTP_BRIDGE_AFFINITY_KINDS, # noqa: F401
|
|
176
|
+
_REQUEST_TRANSPORT_WEBSOCKET,
|
|
177
|
+
_WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401
|
|
178
|
+
_api_key_fair_share_threshold_pct_from_settings,
|
|
179
|
+
_clear_websocket_request_error_overrides,
|
|
180
|
+
_copy_websocket_route_metadata_from_session,
|
|
181
|
+
_event_type_from_payload,
|
|
182
|
+
_HTTPBridgeResponseCreateAttempt,
|
|
183
|
+
_HTTPBridgeRetryCircuitAttemptSelection,
|
|
184
|
+
_HTTPBridgeSession,
|
|
185
|
+
_request_log_client_fields,
|
|
186
|
+
_websocket_request_can_replay_before_visible_output,
|
|
187
|
+
_WebSocketRequestState,
|
|
188
|
+
)
|
|
189
|
+
from app.modules.proxy._service.support import (
|
|
190
|
+
_websocket_route_log_kwargs as _websocket_route_log_kwargs,
|
|
191
|
+
)
|
|
192
|
+
from app.modules.proxy._service.warmup import (
|
|
193
|
+
WarmupExecutionData as WarmupExecutionData,
|
|
194
|
+
)
|
|
195
|
+
from app.modules.proxy._service.warmup import (
|
|
196
|
+
WarmupFailedAccountData as WarmupFailedAccountData,
|
|
197
|
+
)
|
|
198
|
+
from app.modules.proxy._service.warmup import (
|
|
199
|
+
WarmupSkippedAccountData as WarmupSkippedAccountData,
|
|
200
|
+
)
|
|
201
|
+
from app.modules.proxy._service.warmup import (
|
|
202
|
+
WarmupSubmittedAccountData as WarmupSubmittedAccountData,
|
|
203
|
+
)
|
|
204
|
+
from app.modules.proxy._service.warmup import (
|
|
205
|
+
_is_warmup_usage_eligible as _is_warmup_usage_eligible,
|
|
206
|
+
)
|
|
207
|
+
from app.modules.proxy._service.warmup import (
|
|
208
|
+
_materialize_warmup_account as _materialize_warmup_account,
|
|
209
|
+
)
|
|
210
|
+
from app.modules.proxy._service.warmup import (
|
|
211
|
+
_snapshot_warmup_account as _snapshot_warmup_account,
|
|
212
|
+
)
|
|
213
|
+
from app.modules.proxy._service.warmup import (
|
|
214
|
+
_WarmupAccountSnapshot as _WarmupAccountSnapshot,
|
|
215
|
+
)
|
|
216
|
+
from app.modules.proxy._service.warmup import (
|
|
217
|
+
_WarmupSubmitResult as _WarmupSubmitResult,
|
|
218
|
+
)
|
|
219
|
+
from app.modules.proxy._service.warmup import (
|
|
220
|
+
_WarmupUsageSnapshot as _WarmupUsageSnapshot,
|
|
221
|
+
)
|
|
222
|
+
from app.modules.proxy.affinity import (
|
|
223
|
+
_AffinityPolicy,
|
|
224
|
+
_extract_model_class,
|
|
225
|
+
_owner_lookup_session_id_from_headers,
|
|
226
|
+
_sticky_key_from_turn_state_header,
|
|
227
|
+
)
|
|
228
|
+
from app.modules.proxy.affinity_observation import AffinityObservation
|
|
229
|
+
from app.modules.proxy.api_key_usage import estimate_api_key_request_usage
|
|
230
|
+
from app.modules.proxy.continuity import is_http_bridge_account_neutral_replay
|
|
231
|
+
from app.modules.proxy.durable_bridge_repository import (
|
|
232
|
+
DurableBridgeAliasRegistration,
|
|
233
|
+
DurableBridgeAliasRegistrationReceipt,
|
|
234
|
+
durable_bridge_api_key_scope,
|
|
235
|
+
durable_bridge_hash,
|
|
236
|
+
durable_bridge_operation_fingerprint,
|
|
237
|
+
durable_bridge_operation_id,
|
|
238
|
+
)
|
|
239
|
+
from app.modules.proxy.fair_share import (
|
|
240
|
+
API_KEY_STREAM_FAIR_SHARE_ERROR_CODE,
|
|
241
|
+
ApiKeyFairShareDenialError,
|
|
242
|
+
)
|
|
243
|
+
from app.modules.proxy.helpers import (
|
|
244
|
+
_normalize_error_code,
|
|
245
|
+
_parse_openai_error,
|
|
246
|
+
)
|
|
247
|
+
from app.modules.proxy.load_balancer import effective_account_concurrency_caps, effective_routing_tunables
|
|
248
|
+
from app.modules.proxy.tool_call_dedupe import (
|
|
249
|
+
dedupe_replayed_side_effect_input_items,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
logger = logging.getLogger("app.modules.proxy.service")
|
|
253
|
+
|
|
254
|
+
_HTTP_BRIDGE_CLEAN_CLOSE_RETRY_MAX_COUNT = 1
|
|
255
|
+
_HTTP_BRIDGE_CLEAN_CLOSE_RETRY_JITTER_MAX_SECONDS = 2.0
|
|
256
|
+
|
|
257
|
+
_REQUEST_TRANSPORT_HTTP = "http"
|
|
258
|
+
_WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE = "account_auth_invalidated"
|
|
259
|
+
_NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE = "no_security_work_authorized_accounts"
|
|
260
|
+
_SECURITY_WORK_NO_AUTHORIZED_ACCOUNTS_MESSAGE = (
|
|
261
|
+
"Upstream flagged this request as possible cybersecurity work, but no account is marked as authorized for "
|
|
262
|
+
"security work. codex-lb is continuing with normal account selection; the upstream request may still fail until "
|
|
263
|
+
"an account with Trusted Access for Cyber is marked as security-work-authorized."
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
@dataclass(frozen=True, slots=True)
|
|
268
|
+
class _HTTPBridgeStaleGateSnapshot:
|
|
269
|
+
pending_states: list[_WebSocketRequestState]
|
|
270
|
+
queued_count: int
|
|
271
|
+
threshold_seconds: float
|
|
272
|
+
stale_request_states: list[_WebSocketRequestState]
|
|
273
|
+
should_retire: bool
|
|
274
|
+
retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _http_bridge_cooldown_suppression_is_replay_safe(request_state: _WebSocketRequestState) -> bool:
|
|
278
|
+
"""Return whether a cooldown-suppressed submission is provably undispatched.
|
|
279
|
+
|
|
280
|
+
The retry-circuit gate runs before any reconnect or send for the current
|
|
281
|
+
submission, so a request that carries no continuation identity (client or
|
|
282
|
+
proxy-injected), no file account pin, and has seen zero send attempts,
|
|
283
|
+
zero upstream response events and zero replays cannot have executed
|
|
284
|
+
upstream. Only those requests may degrade to the raw-HTTP fallback;
|
|
285
|
+
ambiguous continuations keep the bounded 503 because a replay could
|
|
286
|
+
execute the turn twice.
|
|
287
|
+
|
|
288
|
+
A non-empty payload ``conversation`` counts as continuation identity even
|
|
289
|
+
though it is neither an anchor nor a turn-state key: it binds the turn to
|
|
290
|
+
the bridge session's account, and unlike a response anchor it has no owner
|
|
291
|
+
index, so a raw-HTTP replay cannot prove that owner in a multi-account
|
|
292
|
+
pool and would fail the turn closed instead of letting the cooldown expire.
|
|
293
|
+
|
|
294
|
+
``response_create_attempt_count`` is the send marker rather than
|
|
295
|
+
``awaiting_response_created``: the latter is set optimistically when the
|
|
296
|
+
request state is built and is only cleared after a submission is torn
|
|
297
|
+
down, so it is true for every fresh turn and would make this predicate —
|
|
298
|
+
and the fallback it gates — unreachable. The attempt count increments at
|
|
299
|
+
the ``response.create`` send itself and is never reset, so zero proves no
|
|
300
|
+
frame left for upstream.
|
|
301
|
+
"""
|
|
302
|
+
return (
|
|
303
|
+
request_state.previous_response_id is None
|
|
304
|
+
and not request_state.hard_continuity_anchor
|
|
305
|
+
and not request_state.proxy_injected_previous_response_id
|
|
306
|
+
and not request_state.file_required_preferred_account
|
|
307
|
+
and request_state.response_id is None
|
|
308
|
+
and request_state.response_event_count == 0
|
|
309
|
+
and request_state.replay_count == 0
|
|
310
|
+
and request_state.last_downstream_sequence_number is None
|
|
311
|
+
and not request_state.downstream_visible
|
|
312
|
+
and not request_state.payload_conversation_bound
|
|
313
|
+
and request_state.response_create_attempt_count == 0
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _http_bridge_operation_fingerprint(
|
|
318
|
+
*,
|
|
319
|
+
session_id: str,
|
|
320
|
+
api_key_scope: str,
|
|
321
|
+
request_state: _WebSocketRequestState,
|
|
322
|
+
text_data: str,
|
|
323
|
+
) -> str:
|
|
324
|
+
fingerprint_text = _text_without_account_installation_id(text_data)
|
|
325
|
+
return durable_bridge_operation_fingerprint(
|
|
326
|
+
api_key_scope=api_key_scope,
|
|
327
|
+
request_text=fingerprint_text,
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _http_bridge_terminal_hard_turn_response_id(
|
|
332
|
+
request_state: _WebSocketRequestState,
|
|
333
|
+
operation: Any,
|
|
334
|
+
*,
|
|
335
|
+
allow_anchored_continuation: bool = False,
|
|
336
|
+
) -> str | None:
|
|
337
|
+
"""Return a completed hard-turn anchor when this is a new client turn.
|
|
338
|
+
|
|
339
|
+
Hard turn-state requests can omit ``previous_response_id``. Their durable
|
|
340
|
+
operation fingerprint is therefore otherwise identical for repeated
|
|
341
|
+
prompts. A terminal operation with a response id represents the prior turn,
|
|
342
|
+
not an in-flight retry, so the next request must advance from its response
|
|
343
|
+
rather than replaying that transcript. Recovery/rebind states retain the
|
|
344
|
+
operation identity and are intentionally excluded here. Spool completeness
|
|
345
|
+
is required only when replaying the stored transcript.
|
|
346
|
+
"""
|
|
347
|
+
if (
|
|
348
|
+
(request_state.previous_response_id is not None and not allow_anchored_continuation)
|
|
349
|
+
or not request_state.hard_continuity_anchor
|
|
350
|
+
or request_state.operation_id is not None
|
|
351
|
+
or request_state.operation_rebind_required
|
|
352
|
+
or request_state.replay_count != 0
|
|
353
|
+
):
|
|
354
|
+
return None
|
|
355
|
+
operation_state = getattr(operation, "state", None)
|
|
356
|
+
operation_state = getattr(operation_state, "value", operation_state)
|
|
357
|
+
if operation_state != "completed":
|
|
358
|
+
return None
|
|
359
|
+
response_id = getattr(operation, "response_id", None)
|
|
360
|
+
return response_id if isinstance(response_id, str) and response_id else None
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
async def _rollback_http_bridge_recovery_turn_state_registration(
|
|
364
|
+
service: Any,
|
|
365
|
+
receipt: DurableBridgeAliasRegistrationReceipt,
|
|
366
|
+
) -> tuple[bool, asyncio.CancelledError | None]:
|
|
367
|
+
rollback_task = scheduler_for(service).create_task(
|
|
368
|
+
service._durable_bridge.rollback_recovery_turn_state_registration(receipt=receipt)
|
|
369
|
+
)
|
|
370
|
+
return await _await_task_deferring_cancellation(rollback_task)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _bridge_thread_cache_identity(session: Any) -> ThreadCacheIdentity | None:
|
|
374
|
+
"""The mode this bridge session sends under, or ``None`` for a pure no-op.
|
|
375
|
+
|
|
376
|
+
The bridge dispatches ``response.create`` itself rather than going through
|
|
377
|
+
``_stream_responses_with_session``, so without this a thread served partly
|
|
378
|
+
by the bridge and partly by the per-turn bypass would alternate between a
|
|
379
|
+
scoped and an unscoped identity turn by turn — the worst possible outcome,
|
|
380
|
+
because upstream would see two names for one thread on one account.
|
|
381
|
+
|
|
382
|
+
The fleet value is read off the overlaid settings (dashboard over
|
|
383
|
+
environment over default). The per-API-key override is deliberately not
|
|
384
|
+
consulted here: this path holds only the key's id, not the row, and a
|
|
385
|
+
request that reaches the bridge has already had the override applied at
|
|
386
|
+
ingress when it was dispatched through the ordinary transport. Resolving it
|
|
387
|
+
a second time from a different source could disagree with that decision.
|
|
388
|
+
"""
|
|
389
|
+
|
|
390
|
+
mode = getattr(with_dashboard_overrides(_service_get_settings()), "thread_cache_identity_mode", None)
|
|
391
|
+
identity = thread_cache_identity_from(mode, getattr(getattr(session, "account", None), "id", None))
|
|
392
|
+
return identity
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _text_with_thread_cache_identity(text_data: str, identity: ThreadCacheIdentity | None) -> str:
|
|
396
|
+
"""Scope the frame's thread identity for egress only.
|
|
397
|
+
|
|
398
|
+
Send-only, exactly like ``_text_with_operation_id``: the result is never
|
|
399
|
+
written back to ``request_state.request_text`` or
|
|
400
|
+
``fresh_upstream_request_text``, because
|
|
401
|
+
``_http_bridge_operation_fingerprint`` must keep hashing account-neutral
|
|
402
|
+
text. If a scoped value entered the durable fingerprint, every account swap
|
|
403
|
+
would change the operation identity and the spool lookup would miss
|
|
404
|
+
mid-recovery.
|
|
405
|
+
"""
|
|
406
|
+
|
|
407
|
+
if identity is None or identity.mode == THREAD_CACHE_IDENTITY_MODE_SHARED:
|
|
408
|
+
return text_data
|
|
409
|
+
try:
|
|
410
|
+
frame = json.loads(text_data)
|
|
411
|
+
except (TypeError, json.JSONDecodeError):
|
|
412
|
+
return text_data
|
|
413
|
+
if not isinstance(frame, dict):
|
|
414
|
+
return text_data
|
|
415
|
+
apply_thread_cache_identity(frame, identity)
|
|
416
|
+
return json.dumps(frame, ensure_ascii=True, separators=(",", ":"))
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
async def _send_http_bridge_request_text_with_archive_id(
|
|
420
|
+
session: "_HTTPBridgeSession",
|
|
421
|
+
request_state: _WebSocketRequestState,
|
|
422
|
+
text_data: str,
|
|
423
|
+
*,
|
|
424
|
+
on_send_started: Callable[[], None] | None = None,
|
|
425
|
+
clock: Clock = REAL_CLOCK,
|
|
426
|
+
) -> None:
|
|
427
|
+
text_data = _text_with_operation_id(text_data, request_state.operation_id)
|
|
428
|
+
text_data = _text_with_thread_cache_identity(text_data, _bridge_thread_cache_identity(session))
|
|
429
|
+
# Operation metadata is added after the initial payload sizing pass. Check
|
|
430
|
+
# the exact frame that will cross the websocket so the metadata cannot
|
|
431
|
+
# push an otherwise-valid response.create over the upstream limit.
|
|
432
|
+
_enforce_http_bridge_response_create_text_size(request_state, text_data)
|
|
433
|
+
if on_send_started is not None:
|
|
434
|
+
on_send_started()
|
|
435
|
+
token = set_request_id(request_state.archive_request_id)
|
|
436
|
+
try:
|
|
437
|
+
request_state.response_create_attempt_count += 1
|
|
438
|
+
attempt = _HTTPBridgeResponseCreateAttempt(ordinal=request_state.response_create_attempt_count)
|
|
439
|
+
request_state.response_create_attempt = attempt
|
|
440
|
+
request_state.response_create_sent_at = clock.monotonic()
|
|
441
|
+
session.upstream_reader_wakeup.set()
|
|
442
|
+
try:
|
|
443
|
+
await session.upstream.send_text(text_data)
|
|
444
|
+
except BaseException:
|
|
445
|
+
# A failed or cancelled send is settled by its caller. Disarm the
|
|
446
|
+
# owner watchdog before lifecycle ownership is released so the
|
|
447
|
+
# reader cannot race that cleanup and settle the request twice.
|
|
448
|
+
attempt.disarmed = True
|
|
449
|
+
if request_state.response_create_attempt is attempt:
|
|
450
|
+
request_state.response_create_sent_at = None
|
|
451
|
+
session.upstream_reader_wakeup.set()
|
|
452
|
+
raise
|
|
453
|
+
finally:
|
|
454
|
+
reset_request_id(token)
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
async def _settle_claimed_http_bridge_liveness_failure(
|
|
458
|
+
service: Any,
|
|
459
|
+
session: "_HTTPBridgeSession",
|
|
460
|
+
*,
|
|
461
|
+
error_message: str,
|
|
462
|
+
) -> None:
|
|
463
|
+
"""Finish the pending-deque settlement claimed beside a failed send."""
|
|
464
|
+
|
|
465
|
+
if session.liveness_settlement_owner != "send":
|
|
466
|
+
raise RuntimeError("HTTP bridge liveness settlement started without the send claim")
|
|
467
|
+
async with session.lifecycle_lock:
|
|
468
|
+
await service._fail_http_bridge_reader_and_maybe_retire(
|
|
469
|
+
session,
|
|
470
|
+
error_code=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE,
|
|
471
|
+
error_message=error_message,
|
|
472
|
+
penalize_account=False,
|
|
473
|
+
force_retire=True,
|
|
474
|
+
)
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
_CLIENT_METADATA_TAIL_KEY = ',"client_metadata":'
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _dumps_compact(value: JsonValue) -> str:
|
|
481
|
+
return json.dumps(value, ensure_ascii=True, separators=(",", ":"))
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _splice_account_installation_id(text_data: str, codex_installation_id: str | None) -> str | None:
|
|
485
|
+
"""Stamp ``client_metadata`` without decoding the rest of the frame.
|
|
486
|
+
|
|
487
|
+
Compact ``json.dumps`` (fixed separators, ``ensure_ascii``, insertion
|
|
488
|
+
order) is compositional: an object encodes as the concatenation of its
|
|
489
|
+
independently encoded members. For a frame produced by that encoder,
|
|
490
|
+
rewriting only the trailing top-level ``client_metadata`` value therefore
|
|
491
|
+
yields exactly the bytes a full decode/rewrite/encode would. This covers the
|
|
492
|
+
two dominant shapes -- ``client_metadata`` as the last key, or absent -- and
|
|
493
|
+
returns ``None`` for anything else so the caller falls back to the full
|
|
494
|
+
round trip.
|
|
495
|
+
"""
|
|
496
|
+
if len(text_data) < 3 or text_data[0] != "{" or text_data[-1] != "}":
|
|
497
|
+
return None
|
|
498
|
+
index = text_data.rfind(_CLIENT_METADATA_TAIL_KEY)
|
|
499
|
+
if index < 0:
|
|
500
|
+
if '"client_metadata"' in text_data:
|
|
501
|
+
# Leading or nested key: not provably absent at the top level.
|
|
502
|
+
return None
|
|
503
|
+
inserted: dict[str, JsonValue] = {}
|
|
504
|
+
apply_codex_installation_metadata(inserted, codex_installation_id)
|
|
505
|
+
inserted_metadata = inserted.get("client_metadata")
|
|
506
|
+
if inserted_metadata is None:
|
|
507
|
+
return text_data
|
|
508
|
+
return f"{text_data[:-1]}{_CLIENT_METADATA_TAIL_KEY}{_dumps_compact(inserted_metadata)}}}"
|
|
509
|
+
encoded_metadata = text_data[index + len(_CLIENT_METADATA_TAIL_KEY) : -1]
|
|
510
|
+
if not encoded_metadata.startswith("{"):
|
|
511
|
+
return None
|
|
512
|
+
try:
|
|
513
|
+
raw_metadata = json.loads(encoded_metadata)
|
|
514
|
+
except json.JSONDecodeError:
|
|
515
|
+
# Nested match, or a value followed by further keys: not the trailing top-level key.
|
|
516
|
+
return None
|
|
517
|
+
if not isinstance(raw_metadata, dict):
|
|
518
|
+
return None
|
|
519
|
+
container: dict[str, JsonValue] = {"client_metadata": cast(dict[str, JsonValue], raw_metadata)}
|
|
520
|
+
apply_codex_installation_metadata(container, codex_installation_id)
|
|
521
|
+
metadata = container.get("client_metadata")
|
|
522
|
+
if metadata is None:
|
|
523
|
+
return f"{text_data[:index]}}}"
|
|
524
|
+
updated_metadata = _dumps_compact(metadata)
|
|
525
|
+
if updated_metadata == encoded_metadata:
|
|
526
|
+
return text_data
|
|
527
|
+
return f"{text_data[:index]}{_CLIENT_METADATA_TAIL_KEY}{updated_metadata}}}"
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _text_with_account_installation_id(text_data: str, codex_installation_id: str | None) -> str:
|
|
531
|
+
spliced = _splice_account_installation_id(text_data, codex_installation_id)
|
|
532
|
+
if spliced is not None:
|
|
533
|
+
return spliced
|
|
534
|
+
payload = json.loads(text_data)
|
|
535
|
+
if not isinstance(payload, dict):
|
|
536
|
+
return text_data
|
|
537
|
+
apply_codex_installation_metadata(cast(dict[str, JsonValue], payload), codex_installation_id)
|
|
538
|
+
return _dumps_compact(payload)
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def _text_with_operation_id(text_data: str, operation_id: str | None) -> str:
|
|
542
|
+
"""Attach a stable operation identity without changing the request contract."""
|
|
543
|
+
if not operation_id:
|
|
544
|
+
return text_data
|
|
545
|
+
try:
|
|
546
|
+
payload = json.loads(text_data)
|
|
547
|
+
except (TypeError, json.JSONDecodeError):
|
|
548
|
+
return text_data
|
|
549
|
+
if not isinstance(payload, dict):
|
|
550
|
+
return text_data
|
|
551
|
+
raw_metadata = payload.get("client_metadata")
|
|
552
|
+
metadata = dict(raw_metadata) if isinstance(raw_metadata, dict) else {}
|
|
553
|
+
# This namespace is reserved by the bridge; never trust a caller-supplied
|
|
554
|
+
# value to stand in for the durable operation identity.
|
|
555
|
+
metadata["codex_lb_operation_id"] = operation_id
|
|
556
|
+
payload["client_metadata"] = metadata
|
|
557
|
+
return json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _text_without_operation_id(text_data: str) -> str:
|
|
561
|
+
"""Remove caller-supplied bridge identity before durable fingerprinting."""
|
|
562
|
+
try:
|
|
563
|
+
payload = json.loads(text_data)
|
|
564
|
+
except (TypeError, json.JSONDecodeError):
|
|
565
|
+
return text_data
|
|
566
|
+
if not isinstance(payload, dict):
|
|
567
|
+
return text_data
|
|
568
|
+
raw_metadata = payload.get("client_metadata")
|
|
569
|
+
if not isinstance(raw_metadata, dict) or "codex_lb_operation_id" not in raw_metadata:
|
|
570
|
+
return text_data
|
|
571
|
+
metadata = dict(raw_metadata)
|
|
572
|
+
metadata.pop("codex_lb_operation_id", None)
|
|
573
|
+
if metadata:
|
|
574
|
+
payload["client_metadata"] = metadata
|
|
575
|
+
else:
|
|
576
|
+
payload.pop("client_metadata", None)
|
|
577
|
+
return json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _text_without_account_installation_id(text_data: str) -> str:
|
|
581
|
+
"""Normalize account-specific installation metadata out of a fingerprint."""
|
|
582
|
+
try:
|
|
583
|
+
payload = json.loads(text_data)
|
|
584
|
+
except (TypeError, json.JSONDecodeError):
|
|
585
|
+
return text_data
|
|
586
|
+
if not isinstance(payload, dict):
|
|
587
|
+
return text_data
|
|
588
|
+
raw_metadata = payload.get("client_metadata")
|
|
589
|
+
if not isinstance(raw_metadata, dict):
|
|
590
|
+
return text_data
|
|
591
|
+
metadata: dict[str, JsonValue] = {}
|
|
592
|
+
for key, value in raw_metadata.items():
|
|
593
|
+
if not isinstance(key, str) or key.lower() == CODEX_INSTALLATION_ID_HEADER:
|
|
594
|
+
continue
|
|
595
|
+
if key.lower() == CODEX_TURN_METADATA_HEADER and isinstance(value, str):
|
|
596
|
+
try:
|
|
597
|
+
turn_metadata = json.loads(value)
|
|
598
|
+
except json.JSONDecodeError:
|
|
599
|
+
turn_metadata = None
|
|
600
|
+
if isinstance(turn_metadata, dict) and "installation_id" in turn_metadata:
|
|
601
|
+
turn_metadata.pop("installation_id", None)
|
|
602
|
+
value = json.dumps(turn_metadata, ensure_ascii=True, separators=(",", ":"))
|
|
603
|
+
metadata[key] = value
|
|
604
|
+
if metadata:
|
|
605
|
+
payload["client_metadata"] = metadata
|
|
606
|
+
else:
|
|
607
|
+
payload.pop("client_metadata", None)
|
|
608
|
+
return json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def _text_with_previous_response_id(text_data: str, response_id: str | None) -> str:
|
|
612
|
+
if not response_id:
|
|
613
|
+
return text_data
|
|
614
|
+
try:
|
|
615
|
+
payload = json.loads(text_data)
|
|
616
|
+
except (TypeError, json.JSONDecodeError):
|
|
617
|
+
return text_data
|
|
618
|
+
if not isinstance(payload, dict) or not response_id:
|
|
619
|
+
return text_data
|
|
620
|
+
payload["previous_response_id"] = response_id
|
|
621
|
+
return json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def _enforce_http_bridge_response_create_text_size(
|
|
625
|
+
request_state: _WebSocketRequestState,
|
|
626
|
+
text_data: str,
|
|
627
|
+
) -> None:
|
|
628
|
+
original_request_text = request_state.request_text
|
|
629
|
+
request_state.request_text = text_data
|
|
630
|
+
try:
|
|
631
|
+
_enforce_response_create_size_limit(request_state)
|
|
632
|
+
finally:
|
|
633
|
+
request_state.request_text = original_request_text
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def _request_kind_from_headers(headers: Mapping[str, str] | None) -> str:
|
|
637
|
+
if not headers:
|
|
638
|
+
return "normal"
|
|
639
|
+
raw_turn_metadata = headers.get("x-codex-turn-metadata") or headers.get("X-Codex-Turn-Metadata")
|
|
640
|
+
if not isinstance(raw_turn_metadata, str):
|
|
641
|
+
return "normal"
|
|
642
|
+
try:
|
|
643
|
+
turn_metadata = json.loads(raw_turn_metadata)
|
|
644
|
+
except json.JSONDecodeError:
|
|
645
|
+
return "normal"
|
|
646
|
+
if not isinstance(turn_metadata, dict):
|
|
647
|
+
return "normal"
|
|
648
|
+
raw_request_kind = turn_metadata.get("request_kind")
|
|
649
|
+
if not isinstance(raw_request_kind, str):
|
|
650
|
+
return "normal"
|
|
651
|
+
request_kind = raw_request_kind.strip()
|
|
652
|
+
if request_kind in {"normal", "prewarm"}:
|
|
653
|
+
return request_kind
|
|
654
|
+
return "normal"
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _http_bridge_session_unowned_locked(
|
|
658
|
+
session: "_HTTPBridgeSession",
|
|
659
|
+
*,
|
|
660
|
+
ignore_unanchored_reservation: bool = False,
|
|
661
|
+
) -> bool:
|
|
662
|
+
"""Whether no turn owns ``session`` right now; call under ``pending_lock``.
|
|
663
|
+
|
|
664
|
+
A registered admission waiter owns a turn that has not yet been counted
|
|
665
|
+
into the queue (it may be suspended on the pre-lock fair-share resolve,
|
|
666
|
+
issue #1971, or anywhere between the retry-circuit gate and dispatch);
|
|
667
|
+
retiring under it would fail an admitted turn with upstream_unavailable.
|
|
668
|
+
A deferred retirement re-runs on the turn's own drain triggers once it
|
|
669
|
+
proceeds; if the waiter instead fails admission, its cleanup releases the
|
|
670
|
+
registration and the submit finalizer retires the flagged session.
|
|
671
|
+
"""
|
|
672
|
+
has_visible_pending = any(
|
|
673
|
+
_http_bridge_request_counts_against_queue(request_state) for request_state in session.pending_requests
|
|
674
|
+
)
|
|
675
|
+
return (
|
|
676
|
+
not has_visible_pending
|
|
677
|
+
and session.queued_request_count == 0
|
|
678
|
+
and session.admission_waiter_count == 0
|
|
679
|
+
and (ignore_unanchored_reservation or session.unanchored_reservation_id is None)
|
|
680
|
+
and not session.upstream_close_attempted
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
class _HTTPBridgeRequestSubmitMixin:
|
|
685
|
+
@staticmethod
|
|
686
|
+
def _http_bridge_clean_close_retry_max_count() -> int:
|
|
687
|
+
configured = _HTTP_BRIDGE_CLEAN_CLOSE_RETRY_MAX_COUNT
|
|
688
|
+
# Keep this recovery bounded even if an unsafe higher value is supplied.
|
|
689
|
+
return max(0, min(1, configured))
|
|
690
|
+
|
|
691
|
+
@staticmethod
|
|
692
|
+
def _http_bridge_clean_close_retry_jitter_seconds() -> float:
|
|
693
|
+
return random.uniform(0.0, _HTTP_BRIDGE_CLEAN_CLOSE_RETRY_JITTER_MAX_SECONDS)
|
|
694
|
+
|
|
695
|
+
def _prepare_http_bridge_request(
|
|
696
|
+
self: Any,
|
|
697
|
+
payload: ResponsesRequest,
|
|
698
|
+
headers: Mapping[str, str],
|
|
699
|
+
*,
|
|
700
|
+
api_key: ApiKeyData | None,
|
|
701
|
+
api_key_reservation: ApiKeyUsageReservationData | None,
|
|
702
|
+
request_id: str | None = None,
|
|
703
|
+
client_ip: str | None = None,
|
|
704
|
+
enforce_openai_sdk_contract: bool = True,
|
|
705
|
+
preserve_responses_lite_client_metadata: bool = False,
|
|
706
|
+
) -> tuple[_WebSocketRequestState, str]:
|
|
707
|
+
# One dump feeds client-metadata derivation, the frame and the usage
|
|
708
|
+
# budget; ``to_payload`` is deterministic so sharing it is exact.
|
|
709
|
+
base_payload = payload.to_payload()
|
|
710
|
+
request_state, text_data = self._prepare_response_bridge_request_state(
|
|
711
|
+
payload,
|
|
712
|
+
api_key=api_key,
|
|
713
|
+
api_key_reservation=api_key_reservation,
|
|
714
|
+
include_type_field=True,
|
|
715
|
+
attach_event_queue=True,
|
|
716
|
+
transport=_REQUEST_TRANSPORT_HTTP,
|
|
717
|
+
client_metadata=_response_create_client_metadata(
|
|
718
|
+
base_payload,
|
|
719
|
+
headers=headers,
|
|
720
|
+
preserve_existing_responses_lite=preserve_responses_lite_client_metadata,
|
|
721
|
+
),
|
|
722
|
+
headers=headers,
|
|
723
|
+
session_id=_owner_lookup_session_id_from_headers(headers),
|
|
724
|
+
request_log_id=request_id or get_request_id() or ensure_request_id(None),
|
|
725
|
+
enforce_openai_sdk_contract=enforce_openai_sdk_contract,
|
|
726
|
+
upstream_payload_base=base_payload,
|
|
727
|
+
)
|
|
728
|
+
(
|
|
729
|
+
request_state.useragent,
|
|
730
|
+
request_state.useragent_group,
|
|
731
|
+
request_state.conversation_id,
|
|
732
|
+
) = _request_log_client_fields(headers)
|
|
733
|
+
request_state.client_ip = client_ip
|
|
734
|
+
return request_state, text_data
|
|
735
|
+
|
|
736
|
+
def _prepare_response_bridge_request_state(
|
|
737
|
+
self: Any,
|
|
738
|
+
payload: ResponsesRequest,
|
|
739
|
+
*,
|
|
740
|
+
api_key: ApiKeyData | None,
|
|
741
|
+
api_key_reservation: ApiKeyUsageReservationData | None,
|
|
742
|
+
include_type_field: bool,
|
|
743
|
+
attach_event_queue: bool,
|
|
744
|
+
transport: str,
|
|
745
|
+
client_metadata: Mapping[str, JsonValue] | None,
|
|
746
|
+
headers: Mapping[str, str] | None = None,
|
|
747
|
+
session_id: str | None = None,
|
|
748
|
+
request_id: str | None = None,
|
|
749
|
+
request_log_id: str | None = None,
|
|
750
|
+
enforce_openai_sdk_contract: bool = True,
|
|
751
|
+
upstream_payload_base: JsonObject | None = None,
|
|
752
|
+
) -> tuple[_WebSocketRequestState, str]:
|
|
753
|
+
deduped_replayed_input_count: int | None = None
|
|
754
|
+
deduped_replayed_input_fingerprint: str | None = None
|
|
755
|
+
deduped_replayed_tool_call_count = 0
|
|
756
|
+
if payload.previous_response_id is not None and isinstance(payload.input, list):
|
|
757
|
+
replayed_input_items = cast(list[JsonValue], payload.input)
|
|
758
|
+
deduped_input_items, deduped_replayed_tool_call_count = dedupe_replayed_side_effect_input_items(
|
|
759
|
+
replayed_input_items,
|
|
760
|
+
sanitize_missing_outputs=False,
|
|
761
|
+
)
|
|
762
|
+
if deduped_replayed_tool_call_count > 0:
|
|
763
|
+
deduped_replayed_input_count = len(replayed_input_items)
|
|
764
|
+
deduped_replayed_input_fingerprint = _fingerprint_input_items(replayed_input_items)
|
|
765
|
+
payload = payload.model_copy(update={"input": deduped_input_items})
|
|
766
|
+
# The caller's dump describes the un-deduped input; it must not
|
|
767
|
+
# become the forwarded frame or the budget base.
|
|
768
|
+
upstream_payload_base = None
|
|
769
|
+
protected_agent_control_output_occurrences = (
|
|
770
|
+
_historical_agent_control_output_occurrences(cast(list[JsonValue], payload.input))
|
|
771
|
+
if isinstance(payload.input, list)
|
|
772
|
+
else {}
|
|
773
|
+
)
|
|
774
|
+
if upstream_payload_base is None:
|
|
775
|
+
upstream_payload_base = payload.to_payload()
|
|
776
|
+
# Shallow copy: every mutation below rebinds top-level keys only, so
|
|
777
|
+
# ``upstream_payload_base`` stays the pristine dump for the budget.
|
|
778
|
+
upstream_payload = dict(upstream_payload_base)
|
|
779
|
+
upstream_payload.pop("stream", None)
|
|
780
|
+
upstream_payload.pop("background", None)
|
|
781
|
+
if include_type_field:
|
|
782
|
+
upstream_payload["type"] = "response.create"
|
|
783
|
+
if client_metadata:
|
|
784
|
+
upstream_payload["client_metadata"] = client_metadata
|
|
785
|
+
_finalize_responses_lite_reasoning_context(
|
|
786
|
+
upstream_payload,
|
|
787
|
+
responses_lite=(
|
|
788
|
+
_payload_uses_responses_lite(upstream_payload)
|
|
789
|
+
or _payload_has_responses_lite_websocket_marker(upstream_payload)
|
|
790
|
+
),
|
|
791
|
+
)
|
|
792
|
+
forwarded_service_tier = _normalize_service_tier_value(upstream_payload.get("service_tier"))
|
|
793
|
+
input_item_count = 0
|
|
794
|
+
input_full_fingerprint: str | None = None
|
|
795
|
+
payload_input = payload.input
|
|
796
|
+
if isinstance(payload_input, list):
|
|
797
|
+
payload_input_list = cast(list[JsonValue], payload_input)
|
|
798
|
+
input_item_count = len(payload_input_list)
|
|
799
|
+
if input_item_count > 0:
|
|
800
|
+
input_full_fingerprint = _fingerprint_input_items(payload_input_list)
|
|
801
|
+
|
|
802
|
+
resolved_request_id = request_id or f"ws_{uuid4().hex}"
|
|
803
|
+
header_request_kind = _request_kind_from_headers(headers)
|
|
804
|
+
generate_false_prewarm = header_request_kind == "prewarm" and upstream_payload.get("generate") is False
|
|
805
|
+
connection_request_kind = header_request_kind if transport == _REQUEST_TRANSPORT_WEBSOCKET else None
|
|
806
|
+
request_kind = (
|
|
807
|
+
"normal" if connection_request_kind == "prewarm" and not generate_false_prewarm else header_request_kind
|
|
808
|
+
)
|
|
809
|
+
request_state = _WebSocketRequestState(
|
|
810
|
+
request_id=resolved_request_id,
|
|
811
|
+
request_log_id=request_log_id,
|
|
812
|
+
archive_request_id=request_log_id or resolved_request_id,
|
|
813
|
+
model=payload.model,
|
|
814
|
+
service_tier=forwarded_service_tier,
|
|
815
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
816
|
+
api_key_reservation=api_key_reservation,
|
|
817
|
+
started_at=clock_for(self).monotonic(),
|
|
818
|
+
thread_affinity_last_touch_at=clock_for(self).monotonic(),
|
|
819
|
+
api_key_reservation_last_touch_at=clock_for(self).monotonic(),
|
|
820
|
+
requested_service_tier=forwarded_service_tier,
|
|
821
|
+
awaiting_response_created=True,
|
|
822
|
+
event_queue=asyncio.Queue() if attach_event_queue else None,
|
|
823
|
+
transport=transport,
|
|
824
|
+
enforce_openai_sdk_contract=enforce_openai_sdk_contract,
|
|
825
|
+
api_key=api_key,
|
|
826
|
+
request_usage_budget=estimate_api_key_request_usage(payload, upstream_payload=upstream_payload_base),
|
|
827
|
+
previous_response_id=payload.previous_response_id,
|
|
828
|
+
session_id=_normalize_session_id(session_id),
|
|
829
|
+
hard_continuity_anchor=(
|
|
830
|
+
payload.previous_response_id is not None
|
|
831
|
+
or _sticky_key_from_turn_state_header(headers or {}) is not None
|
|
832
|
+
),
|
|
833
|
+
# Hard turn-state operation recovery can inject its anchor later,
|
|
834
|
+
# after the request state is prepared. Preserve the original
|
|
835
|
+
# payload shape so a full resend remains eligible for denied-anchor
|
|
836
|
+
# retirement when that operation path supplies the id.
|
|
837
|
+
proxy_injected_anchor_had_full_resend_payload=_http_bridge_payload_looks_like_full_resend(payload),
|
|
838
|
+
payload_conversation_bound=bool(payload.conversation),
|
|
839
|
+
input_item_count=input_item_count,
|
|
840
|
+
input_full_fingerprint=input_full_fingerprint,
|
|
841
|
+
request_kind=request_kind,
|
|
842
|
+
connection_request_kind=connection_request_kind,
|
|
843
|
+
generate_false_prewarm=generate_false_prewarm,
|
|
844
|
+
)
|
|
845
|
+
if deduped_replayed_input_count is not None:
|
|
846
|
+
request_state.input_item_count = deduped_replayed_input_count
|
|
847
|
+
request_state.input_full_fingerprint = deduped_replayed_input_fingerprint
|
|
848
|
+
logger.warning(
|
|
849
|
+
"%s_replayed_tool_call_input_deduped request_id=%s original_items=%s deduped_to=%s "
|
|
850
|
+
"removed_tool_calls=%s previous_response_id=%s",
|
|
851
|
+
transport,
|
|
852
|
+
request_state.request_id,
|
|
853
|
+
deduped_replayed_input_count,
|
|
854
|
+
input_item_count,
|
|
855
|
+
deduped_replayed_tool_call_count,
|
|
856
|
+
payload.previous_response_id,
|
|
857
|
+
)
|
|
858
|
+
text_data = json.dumps(upstream_payload, ensure_ascii=True, separators=(",", ":"))
|
|
859
|
+
payload_size = len(text_data.encode("utf-8"))
|
|
860
|
+
max_bytes = _upstream_response_create_max_bytes()
|
|
861
|
+
if payload_size > max_bytes:
|
|
862
|
+
slimmed_payload, slim_summary = _slim_response_create_payload_for_upstream(
|
|
863
|
+
upstream_payload,
|
|
864
|
+
max_bytes=max_bytes,
|
|
865
|
+
protected_agent_control_output_occurrences=protected_agent_control_output_occurrences,
|
|
866
|
+
)
|
|
867
|
+
if slim_summary is not None:
|
|
868
|
+
upstream_payload = slimmed_payload
|
|
869
|
+
text_data = json.dumps(upstream_payload, ensure_ascii=True, separators=(",", ":"))
|
|
870
|
+
logger.warning(
|
|
871
|
+
(
|
|
872
|
+
"Slimmed response.create request_id=%s request_log_id=%s transport=%s "
|
|
873
|
+
"original_bytes=%s slimmed_bytes=%s "
|
|
874
|
+
"historical_tool_outputs_slimmed=%s historical_images_slimmed=%s"
|
|
875
|
+
),
|
|
876
|
+
request_state.request_id,
|
|
877
|
+
request_state.request_log_id,
|
|
878
|
+
transport,
|
|
879
|
+
payload_size,
|
|
880
|
+
len(text_data.encode("utf-8")),
|
|
881
|
+
slim_summary["historical_tool_outputs_slimmed"],
|
|
882
|
+
slim_summary["historical_images_slimmed"],
|
|
883
|
+
)
|
|
884
|
+
request_state.request_text = text_data
|
|
885
|
+
_enforce_response_create_size_limit(request_state)
|
|
886
|
+
return request_state, text_data
|
|
887
|
+
|
|
888
|
+
def _http_bridge_text_with_account_installation_id(
|
|
889
|
+
self: Any,
|
|
890
|
+
session: "_HTTPBridgeSession",
|
|
891
|
+
request_state: _WebSocketRequestState,
|
|
892
|
+
text_data: str,
|
|
893
|
+
) -> str:
|
|
894
|
+
codex_installation_id = getattr(session.account, "codex_installation_id", None)
|
|
895
|
+
# The memo holds the exact objects the previous call returned for this
|
|
896
|
+
# installation id; a text rewrite yields a new object and an account
|
|
897
|
+
# swap changes the id, so both miss and take the full stamp below.
|
|
898
|
+
memo_valid = request_state.installation_stamp_installation_id == codex_installation_id
|
|
899
|
+
if memo_valid and (
|
|
900
|
+
text_data is request_state.installation_stamp_text
|
|
901
|
+
or text_data is request_state.installation_stamp_fresh_text
|
|
902
|
+
):
|
|
903
|
+
updated_text = text_data
|
|
904
|
+
else:
|
|
905
|
+
updated_text = _text_with_account_installation_id(text_data, codex_installation_id)
|
|
906
|
+
fresh_text = request_state.fresh_upstream_request_text
|
|
907
|
+
if fresh_text is not None and not (memo_valid and fresh_text is request_state.installation_stamp_fresh_text):
|
|
908
|
+
updated_fresh_text = _text_with_account_installation_id(fresh_text, codex_installation_id)
|
|
909
|
+
_enforce_http_bridge_response_create_text_size(request_state, updated_fresh_text)
|
|
910
|
+
request_state.fresh_upstream_request_text = updated_fresh_text
|
|
911
|
+
if updated_text == text_data:
|
|
912
|
+
updated_text = text_data
|
|
913
|
+
else:
|
|
914
|
+
request_state.request_text = updated_text
|
|
915
|
+
_enforce_response_create_size_limit(request_state)
|
|
916
|
+
request_state.installation_stamp_installation_id = codex_installation_id
|
|
917
|
+
request_state.installation_stamp_text = updated_text
|
|
918
|
+
request_state.installation_stamp_fresh_text = request_state.fresh_upstream_request_text
|
|
919
|
+
return updated_text
|
|
920
|
+
|
|
921
|
+
async def _inline_http_bridge_image_urls(
|
|
922
|
+
self: Any,
|
|
923
|
+
text_data: str,
|
|
924
|
+
request_state: _WebSocketRequestState,
|
|
925
|
+
) -> str:
|
|
926
|
+
"""Inline external ``input_image`` URLs into ``data:`` URLs.
|
|
927
|
+
|
|
928
|
+
The HTTP direct-stream path already does this via
|
|
929
|
+
``_inline_input_image_urls`` in :mod:`app.core.clients.proxy`, but the
|
|
930
|
+
HTTP bridge (WebSocket pool) path was missing the conversion. The
|
|
931
|
+
upstream ChatGPT WebSocket only accepts ``data:image/…`` payloads; an
|
|
932
|
+
external ``https://`` image URL causes it to silently reject or hang
|
|
933
|
+
the request.
|
|
934
|
+
|
|
935
|
+
This method applies the same transformation to the already-serialised
|
|
936
|
+
``text_data`` JSON that will be sent on the upstream WebSocket.
|
|
937
|
+
If any external image URLs survive inlining (because the fetch failed),
|
|
938
|
+
the request is rejected immediately with a 400 error rather than
|
|
939
|
+
allowing the upstream to hang.
|
|
940
|
+
"""
|
|
941
|
+
settings = _service_get_settings()
|
|
942
|
+
# Quick string-level pre-check: skip the parse/fetch cycle when the
|
|
943
|
+
# payload contains no ``input_image`` items with an ``http`` URL.
|
|
944
|
+
if "input_image" not in text_data:
|
|
945
|
+
return text_data
|
|
946
|
+
try:
|
|
947
|
+
payload_dict: dict[str, JsonValue] = json.loads(text_data)
|
|
948
|
+
except (json.JSONDecodeError, TypeError):
|
|
949
|
+
return text_data
|
|
950
|
+
connect_timeout = getattr(settings, "upstream_connect_timeout_seconds", 5.0)
|
|
951
|
+
async with _service_lease_http_session()() as http_session:
|
|
952
|
+
image_fetch_session = _service_as_image_fetch_session()(http_session)
|
|
953
|
+
inlined = await _service_inline_input_image_urls()(
|
|
954
|
+
payload_dict,
|
|
955
|
+
image_fetch_session,
|
|
956
|
+
connect_timeout,
|
|
957
|
+
)
|
|
958
|
+
inlined = await _inline_top_level_input_image_urls(inlined, image_fetch_session, connect_timeout)
|
|
959
|
+
# After inlining, check if any external URLs survived (i.e. fetch
|
|
960
|
+
# failed). The upstream WS only accepts data: URLs so sending an
|
|
961
|
+
# external URL would just cause a silent hang.
|
|
962
|
+
remaining_external = _count_external_image_urls(inlined)
|
|
963
|
+
if remaining_external > 0:
|
|
964
|
+
raise ProxyResponseError(
|
|
965
|
+
400,
|
|
966
|
+
openai_error(
|
|
967
|
+
"image_download_failed",
|
|
968
|
+
(
|
|
969
|
+
f"Failed to download {remaining_external} external image(s). "
|
|
970
|
+
"The upstream API only accepts inline data: URLs. "
|
|
971
|
+
"Send images as base64 data URLs (data:image/png;base64,...) "
|
|
972
|
+
"or ensure the image URLs are publicly accessible."
|
|
973
|
+
),
|
|
974
|
+
),
|
|
975
|
+
)
|
|
976
|
+
updated_text = json.dumps(inlined, ensure_ascii=True, separators=(",", ":"))
|
|
977
|
+
if updated_text == text_data:
|
|
978
|
+
return text_data
|
|
979
|
+
request_state.request_text = updated_text
|
|
980
|
+
_enforce_response_create_size_limit(request_state)
|
|
981
|
+
return updated_text
|
|
982
|
+
|
|
983
|
+
async def _submit_http_bridge_request(
|
|
984
|
+
self: Any,
|
|
985
|
+
session: "_HTTPBridgeSession",
|
|
986
|
+
*,
|
|
987
|
+
request_state: _WebSocketRequestState,
|
|
988
|
+
text_data: str,
|
|
989
|
+
queue_limit: int,
|
|
990
|
+
recovery_turn_state: str | None = None,
|
|
991
|
+
) -> None:
|
|
992
|
+
request_scope_id = ensure_request_scope_id()
|
|
993
|
+
owned_unanchored_handoff = session.unanchored_reservation_id == request_scope_id
|
|
994
|
+
try:
|
|
995
|
+
await self._submit_http_bridge_request_with_handoff(
|
|
996
|
+
session,
|
|
997
|
+
request_state=request_state,
|
|
998
|
+
text_data=text_data,
|
|
999
|
+
queue_limit=queue_limit,
|
|
1000
|
+
request_scope_id=request_scope_id,
|
|
1001
|
+
owned_unanchored_handoff=owned_unanchored_handoff,
|
|
1002
|
+
recovery_turn_state=recovery_turn_state,
|
|
1003
|
+
)
|
|
1004
|
+
finally:
|
|
1005
|
+
_release_http_bridge_unanchored_handoff(
|
|
1006
|
+
session,
|
|
1007
|
+
request_scope_id=request_scope_id,
|
|
1008
|
+
)
|
|
1009
|
+
# Like the half-open lease below, the admission registration the
|
|
1010
|
+
# submit took at entry is handed back here when no dispatch took
|
|
1011
|
+
# it over.
|
|
1012
|
+
released_admission_waiter = await self._release_http_bridge_admission_preregistration(
|
|
1013
|
+
session, request_state=request_state
|
|
1014
|
+
)
|
|
1015
|
+
if request_state.claimed_half_open_until > 0.0 and request_state.response_create_attempt_count == 0:
|
|
1016
|
+
# This admission claimed the half-open probe but the request
|
|
1017
|
+
# never ATTEMPTED the upstream send — a poisoned-anchor
|
|
1018
|
+
# rejection, a recovery-journal or operation-ledger refusal,
|
|
1019
|
+
# a reconnect failure, or a completed-operation spool return.
|
|
1020
|
+
# The attempt marker, not the sent timestamp, decides this:
|
|
1021
|
+
# an ambiguous send failure clears the timestamp while the
|
|
1022
|
+
# frame may already be running upstream, and releasing that
|
|
1023
|
+
# probe would let a second dispatch run beside it.
|
|
1024
|
+
# Hand exactly that probe back and restore the transition
|
|
1025
|
+
# marker (an expired but positive cooldown) so the next
|
|
1026
|
+
# request re-claims the lease instead of the whole window
|
|
1027
|
+
# suppressing traffic behind a probe that never flew.
|
|
1028
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1029
|
+
unused_probe_state = self._http_bridge_retry_circuits.get(session.key)
|
|
1030
|
+
if (
|
|
1031
|
+
unused_probe_state is not None
|
|
1032
|
+
and unused_probe_state.half_open_until == request_state.claimed_half_open_until
|
|
1033
|
+
):
|
|
1034
|
+
unused_probe_state.half_open_until = 0.0
|
|
1035
|
+
unused_probe_state.cooldown_until = clock_for(self).monotonic() - 1.0
|
|
1036
|
+
request_state.claimed_half_open_until = 0.0
|
|
1037
|
+
# Inner pre-submit cleanup may clear the reservation before control
|
|
1038
|
+
# returns here, so ownership must be captured before awaiting it.
|
|
1039
|
+
# Only that request, or one whose pre-dispatch exit just released
|
|
1040
|
+
# the admission registration a retirement was deferring on, can
|
|
1041
|
+
# make detached-session retirement newly ready; an ordinary
|
|
1042
|
+
# send/reader failure already owns terminal settlement, and
|
|
1043
|
+
# closing again would run that funnel twice.
|
|
1044
|
+
if (
|
|
1045
|
+
(owned_unanchored_handoff or released_admission_waiter)
|
|
1046
|
+
and session.upstream_control.retire_after_drain
|
|
1047
|
+
and not session.upstream_close_attempted
|
|
1048
|
+
):
|
|
1049
|
+
await self._retire_http_bridge_after_drain_if_ready(session)
|
|
1050
|
+
|
|
1051
|
+
async def _submit_http_bridge_request_with_handoff(
|
|
1052
|
+
self: Any,
|
|
1053
|
+
session: "_HTTPBridgeSession",
|
|
1054
|
+
*,
|
|
1055
|
+
request_state: _WebSocketRequestState,
|
|
1056
|
+
text_data: str,
|
|
1057
|
+
queue_limit: int,
|
|
1058
|
+
request_scope_id: str,
|
|
1059
|
+
owned_unanchored_handoff: bool,
|
|
1060
|
+
recovery_turn_state: str | None = None,
|
|
1061
|
+
) -> None:
|
|
1062
|
+
clock = clock_for(self)
|
|
1063
|
+
# Own admission from submit entry, not from the dispatch registration:
|
|
1064
|
+
# a concurrent cooldown-suppressed sibling retires a session no turn
|
|
1065
|
+
# owns, and between the retry-circuit gate and that registration this
|
|
1066
|
+
# turn would otherwise be invisible to it (issue #1943 follow-up).
|
|
1067
|
+
# The dispatch registration takes this over; the interruption cleanup
|
|
1068
|
+
# and the submit finalizer release it on every earlier exit.
|
|
1069
|
+
async with session.pending_lock:
|
|
1070
|
+
session.admission_waiter_count += 1
|
|
1071
|
+
request_state.admission_waiter_preregistered = True
|
|
1072
|
+
recovery_attempt_consumed = False
|
|
1073
|
+
# Eventless upstream timeouts retire the current socket. A client
|
|
1074
|
+
# reconnect can otherwise create a fresh socket for the same hard key
|
|
1075
|
+
# and submit the identical request repeatedly while the retry circuit
|
|
1076
|
+
# is cooling down. Gate new submissions before any reconnect/send so
|
|
1077
|
+
# the circuit turns this into a bounded 503 instead of another
|
|
1078
|
+
# response.create attempt. A proof-gated full resend remains allowed
|
|
1079
|
+
# because it is the client's own replay-safe request, not an opaque
|
|
1080
|
+
# continuation replay.
|
|
1081
|
+
allow_proof_gated_continuity_replay = bool(
|
|
1082
|
+
request_state.previous_response_id is not None
|
|
1083
|
+
and request_state.fresh_upstream_request_is_retry_safe
|
|
1084
|
+
and request_state.fresh_upstream_request_text
|
|
1085
|
+
and request_state.response_event_count == 0
|
|
1086
|
+
and request_state.replay_count == 0
|
|
1087
|
+
)
|
|
1088
|
+
admission_claimed_leases: list[float] = []
|
|
1089
|
+
retry_allowed = await self._http_bridge_precreated_retry_allowed(
|
|
1090
|
+
session,
|
|
1091
|
+
allow_proof_gated_continuity_replay=allow_proof_gated_continuity_replay,
|
|
1092
|
+
claimed_lease_out=admission_claimed_leases,
|
|
1093
|
+
)
|
|
1094
|
+
# The exact lease this admission claimed, handed out by the claim
|
|
1095
|
+
# itself under its own lock: empty when this request claimed
|
|
1096
|
+
# nothing, so a later fail-closed path can hand back only its own
|
|
1097
|
+
# probe and never a lease another submission installed under any
|
|
1098
|
+
# interleaving.
|
|
1099
|
+
claimed_half_open_until = admission_claimed_leases[-1] if admission_claimed_leases else 0.0
|
|
1100
|
+
request_state.claimed_half_open_until = claimed_half_open_until
|
|
1101
|
+
if not retry_allowed:
|
|
1102
|
+
block_seconds, block_reason = await self._http_bridge_precreated_retry_block(session)
|
|
1103
|
+
retry_after_seconds = max(1, math.ceil(block_seconds))
|
|
1104
|
+
# The session may have been created before this late admission
|
|
1105
|
+
# check (issue #1943). Do not leave an unsubmitted socket
|
|
1106
|
+
# reusable during the cooldown: hand back this request's own
|
|
1107
|
+
# admission registration, then retire the session when no other
|
|
1108
|
+
# turn owns it, letting the bounded drain path handle registry
|
|
1109
|
+
# detach, aliases, leases, and close ownership. An admitted
|
|
1110
|
+
# sibling (the half-open probe this cooldown permits) is a live
|
|
1111
|
+
# owner even before its dispatch registration; flagging the
|
|
1112
|
+
# session under it would fail that one allowed submission.
|
|
1113
|
+
await self._release_http_bridge_admission_preregistration(session, request_state=request_state)
|
|
1114
|
+
await self._retire_idle_http_bridge_session_on_cooldown_suppression(
|
|
1115
|
+
session,
|
|
1116
|
+
owned_unanchored_handoff=owned_unanchored_handoff,
|
|
1117
|
+
)
|
|
1118
|
+
_log_http_bridge_event(
|
|
1119
|
+
"submit_retry_circuit_suppressed",
|
|
1120
|
+
session.key,
|
|
1121
|
+
account_id=session.account.id,
|
|
1122
|
+
model=session.request_model,
|
|
1123
|
+
detail=block_reason,
|
|
1124
|
+
cache_key_family=session.key.affinity_kind,
|
|
1125
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1126
|
+
)
|
|
1127
|
+
cooldown_error = ProxyResponseError(
|
|
1128
|
+
503,
|
|
1129
|
+
openai_error(
|
|
1130
|
+
"upstream_request_timeout",
|
|
1131
|
+
_http_bridge_retry_circuit_suppression_message(block_reason, retry_after_seconds),
|
|
1132
|
+
),
|
|
1133
|
+
retry_after_seconds=retry_after_seconds,
|
|
1134
|
+
)
|
|
1135
|
+
if _http_bridge_cooldown_suppression_is_replay_safe(request_state):
|
|
1136
|
+
setattr(cooldown_error, _HTTP_BRIDGE_PRE_SUBMIT_FAILURE_ATTR, True)
|
|
1137
|
+
setattr(cooldown_error, _HTTP_BRIDGE_COOLDOWN_SUPPRESSION_ATTR, True)
|
|
1138
|
+
raise cooldown_error
|
|
1139
|
+
submit_time_anchor_quarantined = _http_bridge_session_key_poison_quarantined(self, session.key)
|
|
1140
|
+
submit_time_anchor_tombstoned = False
|
|
1141
|
+
if request_state.proxy_injected_previous_response_id and not submit_time_anchor_quarantined:
|
|
1142
|
+
# A remotely written abandonment tombstone arms no quarantine by
|
|
1143
|
+
# design; the submit-time load adopted it, and the injected
|
|
1144
|
+
# anchor it fences is just as dead as a quarantined one.
|
|
1145
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1146
|
+
submit_tombstone_state = self._http_bridge_retry_circuits.get(session.key)
|
|
1147
|
+
submit_time_anchor_tombstoned = bool(
|
|
1148
|
+
submit_tombstone_state is not None
|
|
1149
|
+
and submit_tombstone_state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
1150
|
+
)
|
|
1151
|
+
if request_state.proxy_injected_previous_response_id and (
|
|
1152
|
+
submit_time_anchor_quarantined or submit_time_anchor_tombstoned
|
|
1153
|
+
):
|
|
1154
|
+
# The submit-time load above can be the first to arm the poison
|
|
1155
|
+
# quarantine: planning may have served a cached view while a
|
|
1156
|
+
# replica's opening — stamped by an arbitrary wall clock —
|
|
1157
|
+
# already looked expired here. The payload was serialized with
|
|
1158
|
+
# the anchor that quarantine just proved dead, so dispatching it
|
|
1159
|
+
# would spend the admitted probe on a known-poisoned attach.
|
|
1160
|
+
# Fail fast with the same rejection the planning gate surfaces
|
|
1161
|
+
# and let the client resend its full history; the planning
|
|
1162
|
+
# caches stay a performance bound, never a correctness
|
|
1163
|
+
# assumption. Client-supplied anchors are untouched — only the
|
|
1164
|
+
# proxy's own injection is refused.
|
|
1165
|
+
_log_http_bridge_event(
|
|
1166
|
+
"previous_response_poisoned_anchor_fail_fast",
|
|
1167
|
+
session.key,
|
|
1168
|
+
account_id=session.account.id,
|
|
1169
|
+
model=session.request_model,
|
|
1170
|
+
detail=(
|
|
1171
|
+
"outcome=submit_time_quarantine_fail_closed"
|
|
1172
|
+
if submit_time_anchor_quarantined
|
|
1173
|
+
else "outcome=submit_time_tombstone_fail_closed"
|
|
1174
|
+
),
|
|
1175
|
+
cache_key_family=session.key.affinity_kind,
|
|
1176
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1177
|
+
)
|
|
1178
|
+
raise ProxyResponseError(
|
|
1179
|
+
404,
|
|
1180
|
+
openai_error(
|
|
1181
|
+
"bridge_previous_response_not_found",
|
|
1182
|
+
"The reattached anchor for this session was proven dead; resend the full conversation.",
|
|
1183
|
+
),
|
|
1184
|
+
)
|
|
1185
|
+
# Persist the recovery checkpoint only after the retry circuit has
|
|
1186
|
+
# admitted this request. A client reconnect suppressed by the
|
|
1187
|
+
# cooldown must not create or refresh a journal entry for a request
|
|
1188
|
+
# that was never dispatched upstream.
|
|
1189
|
+
if (
|
|
1190
|
+
request_state.fresh_upstream_request_is_retry_safe
|
|
1191
|
+
and request_state.fresh_upstream_request_text
|
|
1192
|
+
and request_state.replay_count == 0
|
|
1193
|
+
and request_state.recovery_attempt_fingerprint is None
|
|
1194
|
+
and session.durable_session_id is not None
|
|
1195
|
+
and session.durable_owner_epoch is not None
|
|
1196
|
+
):
|
|
1197
|
+
attempt_fingerprint = durable_bridge_hash(request_state.fresh_upstream_request_text)
|
|
1198
|
+
try:
|
|
1199
|
+
attempt = await self._durable_bridge.record_recovery_attempt(
|
|
1200
|
+
session_id=session.durable_session_id,
|
|
1201
|
+
api_key_id=session.key.api_key_id,
|
|
1202
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
1203
|
+
owner_epoch=session.durable_owner_epoch,
|
|
1204
|
+
request_fingerprint=attempt_fingerprint,
|
|
1205
|
+
request_id=request_state.request_id,
|
|
1206
|
+
account_id=session.account.id,
|
|
1207
|
+
model=request_state.model,
|
|
1208
|
+
replay_safe=True,
|
|
1209
|
+
)
|
|
1210
|
+
if attempt is None:
|
|
1211
|
+
# ``None`` is the durable owner fence rejecting this
|
|
1212
|
+
# worker, not an unavailable journal (which raises and
|
|
1213
|
+
# is handled below). Never dispatch from a stale owner.
|
|
1214
|
+
session.closed = True
|
|
1215
|
+
session.upstream_control.reconnect_requested = True
|
|
1216
|
+
session.upstream_control.retire_after_drain = True
|
|
1217
|
+
_record_continuity_fail_closed(
|
|
1218
|
+
surface="http_bridge",
|
|
1219
|
+
reason="recovery_attempt_owner_fence_rejected",
|
|
1220
|
+
previous_response_id=request_state.previous_response_id,
|
|
1221
|
+
session_id=request_state.session_id,
|
|
1222
|
+
upstream_error_code="bridge_continuity_persistence_failed",
|
|
1223
|
+
)
|
|
1224
|
+
raise ProxyResponseError(
|
|
1225
|
+
502,
|
|
1226
|
+
openai_error(
|
|
1227
|
+
"bridge_continuity_persistence_failed",
|
|
1228
|
+
"HTTP responses session ownership changed; retry the request.",
|
|
1229
|
+
),
|
|
1230
|
+
)
|
|
1231
|
+
if getattr(attempt.state, "value", attempt.state) != "unknown":
|
|
1232
|
+
if getattr(attempt.state, "value", attempt.state) != "replayed":
|
|
1233
|
+
raise ProxyResponseError(
|
|
1234
|
+
502,
|
|
1235
|
+
openai_error(
|
|
1236
|
+
"bridge_continuity_persistence_failed",
|
|
1237
|
+
"The recovery checkpoint was already consumed; retry the request.",
|
|
1238
|
+
),
|
|
1239
|
+
)
|
|
1240
|
+
# A REPLAYED checkpoint may belong to a completed
|
|
1241
|
+
# operation whose finalized transcript is safe to replay.
|
|
1242
|
+
# Defer rejection until the operation ledger lookup below
|
|
1243
|
+
# can decide that terminal-spool case; nonterminal rows
|
|
1244
|
+
# remain fail-closed after that lookup.
|
|
1245
|
+
recovery_attempt_consumed = True
|
|
1246
|
+
if (
|
|
1247
|
+
not recovery_attempt_consumed
|
|
1248
|
+
and getattr(attempt, "request_id", request_state.request_id) != request_state.request_id
|
|
1249
|
+
):
|
|
1250
|
+
raise ProxyResponseError(
|
|
1251
|
+
502,
|
|
1252
|
+
openai_error(
|
|
1253
|
+
"bridge_continuity_persistence_failed",
|
|
1254
|
+
"Another recovery request is already in flight; retry the request.",
|
|
1255
|
+
),
|
|
1256
|
+
)
|
|
1257
|
+
request_state.recovery_attempt_fingerprint = attempt_fingerprint
|
|
1258
|
+
request_state.recovery_attempt_session_id = session.durable_session_id
|
|
1259
|
+
request_state.recovery_attempt_owner_epoch = session.durable_owner_epoch
|
|
1260
|
+
except ProxyResponseError:
|
|
1261
|
+
raise
|
|
1262
|
+
except Exception as exc:
|
|
1263
|
+
# The journal is an additional recovery fence. Without a
|
|
1264
|
+
# durable UNKNOWN row, an ambiguous send cannot be claimed by
|
|
1265
|
+
# another owner, so fail closed instead of dispatching an
|
|
1266
|
+
# unjournaled recovery-safe request.
|
|
1267
|
+
session.closed = True
|
|
1268
|
+
session.upstream_control.reconnect_requested = True
|
|
1269
|
+
session.upstream_control.retire_after_drain = True
|
|
1270
|
+
_record_continuity_fail_closed(
|
|
1271
|
+
surface="http_bridge",
|
|
1272
|
+
reason="recovery_attempt_persistence_failed",
|
|
1273
|
+
previous_response_id=request_state.previous_response_id,
|
|
1274
|
+
session_id=request_state.session_id,
|
|
1275
|
+
upstream_error_code="bridge_continuity_persistence_failed",
|
|
1276
|
+
)
|
|
1277
|
+
raise ProxyResponseError(
|
|
1278
|
+
502,
|
|
1279
|
+
openai_error(
|
|
1280
|
+
"bridge_continuity_persistence_failed",
|
|
1281
|
+
"Recovered response continuity could not be persisted; retry the request.",
|
|
1282
|
+
),
|
|
1283
|
+
) from exc
|
|
1284
|
+
# Account installation metadata is part of the final upstream frame.
|
|
1285
|
+
# Apply and size-check it before recording the operation so a local
|
|
1286
|
+
# payload-too-large rejection cannot leave a submitted retry fence.
|
|
1287
|
+
text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data)
|
|
1288
|
+
record_operation = getattr(self._durable_bridge, "record_operation", None)
|
|
1289
|
+
if (
|
|
1290
|
+
callable(record_operation)
|
|
1291
|
+
and (
|
|
1292
|
+
request_state.previous_response_id is not None
|
|
1293
|
+
or request_state.operation_rebind_required
|
|
1294
|
+
or recovery_attempt_consumed
|
|
1295
|
+
)
|
|
1296
|
+
and (request_state.operation_id is None or request_state.operation_rebind_required)
|
|
1297
|
+
and session.durable_session_id is not None
|
|
1298
|
+
and session.durable_owner_epoch is not None
|
|
1299
|
+
):
|
|
1300
|
+
text_data = _text_without_operation_id(text_data)
|
|
1301
|
+
api_key_scope = durable_bridge_api_key_scope(session.key.api_key_id)
|
|
1302
|
+
operation_fingerprint = (
|
|
1303
|
+
request_state.operation_fingerprint
|
|
1304
|
+
if request_state.operation_rebind_required and request_state.operation_fingerprint is not None
|
|
1305
|
+
else _http_bridge_operation_fingerprint(
|
|
1306
|
+
session_id=session.durable_session_id,
|
|
1307
|
+
api_key_scope=api_key_scope,
|
|
1308
|
+
request_state=request_state,
|
|
1309
|
+
text_data=text_data,
|
|
1310
|
+
)
|
|
1311
|
+
)
|
|
1312
|
+
operation_id = (
|
|
1313
|
+
request_state.operation_id
|
|
1314
|
+
if request_state.operation_rebind_required and request_state.operation_id is not None
|
|
1315
|
+
else durable_bridge_operation_id(session.durable_session_id, operation_fingerprint)
|
|
1316
|
+
)
|
|
1317
|
+
operation_parent_response_id = (
|
|
1318
|
+
request_state.operation_parent_response_id
|
|
1319
|
+
if request_state.operation_rebind_required
|
|
1320
|
+
else request_state.previous_response_id
|
|
1321
|
+
)
|
|
1322
|
+
# The operation row must not be committed until the exact
|
|
1323
|
+
# operation-tagged frame is known to fit. Otherwise a local size
|
|
1324
|
+
# rejection before ``send_text`` leaves a submitted ledger row
|
|
1325
|
+
# that fences every identical retry as an unknown in-flight turn.
|
|
1326
|
+
operation_tagged_text = _text_with_operation_id(text_data, operation_id)
|
|
1327
|
+
_enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text)
|
|
1328
|
+
try:
|
|
1329
|
+
get_operation_by_fingerprint = getattr(self._durable_bridge, "get_operation_by_fingerprint", None)
|
|
1330
|
+
get_operation = getattr(self._durable_bridge, "get_operation", None)
|
|
1331
|
+
|
|
1332
|
+
async def lookup_operation() -> Any:
|
|
1333
|
+
operation = None
|
|
1334
|
+
if callable(get_operation_by_fingerprint):
|
|
1335
|
+
operation = await _call_with_supported_optional_kwargs(
|
|
1336
|
+
get_operation_by_fingerprint,
|
|
1337
|
+
optional_kwargs={"api_key_scope": api_key_scope},
|
|
1338
|
+
request_fingerprint=operation_fingerprint,
|
|
1339
|
+
)
|
|
1340
|
+
if operation is None and callable(get_operation):
|
|
1341
|
+
operation = await get_operation(operation_id=operation_id)
|
|
1342
|
+
return operation
|
|
1343
|
+
|
|
1344
|
+
existing_operation = await lookup_operation()
|
|
1345
|
+
if recovery_attempt_consumed and existing_operation is None:
|
|
1346
|
+
raise ProxyResponseError(
|
|
1347
|
+
502,
|
|
1348
|
+
openai_error(
|
|
1349
|
+
"bridge_continuity_persistence_failed",
|
|
1350
|
+
"The recovery checkpoint was already consumed; retry the request.",
|
|
1351
|
+
),
|
|
1352
|
+
)
|
|
1353
|
+
hard_turn_chain_advanced = False
|
|
1354
|
+
seen_hard_turn_response_ids: set[str] = set()
|
|
1355
|
+
while not recovery_attempt_consumed:
|
|
1356
|
+
terminal_hard_turn_response_id = _http_bridge_terminal_hard_turn_response_id(
|
|
1357
|
+
request_state,
|
|
1358
|
+
existing_operation,
|
|
1359
|
+
allow_anchored_continuation=hard_turn_chain_advanced,
|
|
1360
|
+
)
|
|
1361
|
+
if (
|
|
1362
|
+
terminal_hard_turn_response_id is not None
|
|
1363
|
+
and terminal_hard_turn_response_id not in seen_hard_turn_response_ids
|
|
1364
|
+
):
|
|
1365
|
+
# A completed operation with the same body is the
|
|
1366
|
+
# prior hard turn, not a replay request: advance from
|
|
1367
|
+
# its response instead of replaying that transcript.
|
|
1368
|
+
# Keep walking the chain because repeated identical
|
|
1369
|
+
# turns can have several terminal operations with
|
|
1370
|
+
# successive response anchors.
|
|
1371
|
+
seen_hard_turn_response_ids.add(terminal_hard_turn_response_id)
|
|
1372
|
+
hard_turn_chain_advanced = True
|
|
1373
|
+
text_data = _text_with_previous_response_id(text_data, terminal_hard_turn_response_id)
|
|
1374
|
+
request_state.request_text = text_data
|
|
1375
|
+
_bind_http_bridge_proxy_injected_anchor(
|
|
1376
|
+
self,
|
|
1377
|
+
request_state,
|
|
1378
|
+
response_id=terminal_hard_turn_response_id,
|
|
1379
|
+
)
|
|
1380
|
+
request_state.hard_continuity_anchor = True
|
|
1381
|
+
operation_parent_response_id = terminal_hard_turn_response_id
|
|
1382
|
+
operation_fingerprint = durable_bridge_operation_fingerprint(
|
|
1383
|
+
api_key_scope=api_key_scope,
|
|
1384
|
+
request_text=_text_without_account_installation_id(text_data),
|
|
1385
|
+
)
|
|
1386
|
+
operation_id = durable_bridge_operation_id(
|
|
1387
|
+
session.durable_session_id,
|
|
1388
|
+
operation_fingerprint,
|
|
1389
|
+
)
|
|
1390
|
+
operation_tagged_text = _text_with_operation_id(text_data, operation_id)
|
|
1391
|
+
_enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text)
|
|
1392
|
+
existing_operation = await lookup_operation()
|
|
1393
|
+
continue
|
|
1394
|
+
|
|
1395
|
+
# If another worker durably observed the previous turn's
|
|
1396
|
+
# completion, advance a new continuation to that response
|
|
1397
|
+
# anchor instead of replaying the timed-out turn. Re-run
|
|
1398
|
+
# the operation lookup after this race-path advancement so
|
|
1399
|
+
# two completions observed back-to-back are both walked.
|
|
1400
|
+
if existing_operation is None:
|
|
1401
|
+
get_latest_completed = getattr(self._durable_bridge, "get_latest_completed_operation", None)
|
|
1402
|
+
if callable(get_latest_completed):
|
|
1403
|
+
completed_operation = await _call_with_supported_optional_kwargs(
|
|
1404
|
+
get_latest_completed,
|
|
1405
|
+
optional_kwargs={"request_fingerprint": operation_fingerprint},
|
|
1406
|
+
session_id=session.durable_session_id,
|
|
1407
|
+
parent_response_id=operation_parent_response_id,
|
|
1408
|
+
)
|
|
1409
|
+
if completed_operation is None:
|
|
1410
|
+
get_latest_completed_any_session = getattr(
|
|
1411
|
+
self._durable_bridge,
|
|
1412
|
+
"get_latest_completed_operation_any_session",
|
|
1413
|
+
None,
|
|
1414
|
+
)
|
|
1415
|
+
if callable(get_latest_completed_any_session):
|
|
1416
|
+
completed_operation = await _call_with_supported_optional_kwargs(
|
|
1417
|
+
get_latest_completed_any_session,
|
|
1418
|
+
optional_kwargs={
|
|
1419
|
+
"api_key_scope": api_key_scope,
|
|
1420
|
+
"request_fingerprint": operation_fingerprint,
|
|
1421
|
+
},
|
|
1422
|
+
parent_response_id=request_state.previous_response_id,
|
|
1423
|
+
)
|
|
1424
|
+
completed_response_id = getattr(completed_operation, "response_id", None)
|
|
1425
|
+
if completed_response_id and completed_response_id != request_state.previous_response_id:
|
|
1426
|
+
text_data = _text_with_previous_response_id(text_data, completed_response_id)
|
|
1427
|
+
request_state.request_text = text_data
|
|
1428
|
+
_bind_http_bridge_proxy_injected_anchor(
|
|
1429
|
+
self,
|
|
1430
|
+
request_state,
|
|
1431
|
+
response_id=completed_response_id,
|
|
1432
|
+
)
|
|
1433
|
+
operation_parent_response_id = completed_response_id
|
|
1434
|
+
hard_turn_chain_advanced = True
|
|
1435
|
+
seen_hard_turn_response_ids.add(completed_response_id)
|
|
1436
|
+
request_state.hard_continuity_anchor = True
|
|
1437
|
+
operation_fingerprint = durable_bridge_operation_fingerprint(
|
|
1438
|
+
api_key_scope=api_key_scope,
|
|
1439
|
+
request_text=_text_without_account_installation_id(text_data),
|
|
1440
|
+
)
|
|
1441
|
+
operation_id = durable_bridge_operation_id(
|
|
1442
|
+
session.durable_session_id,
|
|
1443
|
+
operation_fingerprint,
|
|
1444
|
+
)
|
|
1445
|
+
operation_tagged_text = _text_with_operation_id(text_data, operation_id)
|
|
1446
|
+
_enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text)
|
|
1447
|
+
existing_operation = await lookup_operation()
|
|
1448
|
+
continue
|
|
1449
|
+
break
|
|
1450
|
+
operation = await _call_with_supported_optional_kwargs(
|
|
1451
|
+
record_operation,
|
|
1452
|
+
optional_kwargs={
|
|
1453
|
+
"recovery_attempt_session_id": request_state.recovery_attempt_session_id
|
|
1454
|
+
if request_state.recovery_attempt_claimed
|
|
1455
|
+
else None,
|
|
1456
|
+
"recovery_attempt_owner_epoch": request_state.recovery_attempt_owner_epoch
|
|
1457
|
+
if request_state.recovery_attempt_claimed
|
|
1458
|
+
else None,
|
|
1459
|
+
"recovery_attempt_fingerprint": request_state.recovery_attempt_fingerprint
|
|
1460
|
+
if request_state.recovery_attempt_claimed
|
|
1461
|
+
else None,
|
|
1462
|
+
"recovery_attempt_consumed": recovery_attempt_consumed,
|
|
1463
|
+
},
|
|
1464
|
+
operation_id=operation_id,
|
|
1465
|
+
session_id=session.durable_session_id,
|
|
1466
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
1467
|
+
owner_epoch=session.durable_owner_epoch,
|
|
1468
|
+
request_fingerprint=operation_fingerprint,
|
|
1469
|
+
api_key_scope=api_key_scope,
|
|
1470
|
+
account_id=session.account.id,
|
|
1471
|
+
model=request_state.model,
|
|
1472
|
+
parent_response_id=operation_parent_response_id,
|
|
1473
|
+
request_text=text_data,
|
|
1474
|
+
)
|
|
1475
|
+
except Exception as exc:
|
|
1476
|
+
session.closed = True
|
|
1477
|
+
session.upstream_control.reconnect_requested = True
|
|
1478
|
+
session.upstream_control.retire_after_drain = True
|
|
1479
|
+
_record_continuity_fail_closed(
|
|
1480
|
+
surface="http_bridge",
|
|
1481
|
+
reason="operation_persistence_failed",
|
|
1482
|
+
previous_response_id=request_state.previous_response_id,
|
|
1483
|
+
session_id=request_state.session_id,
|
|
1484
|
+
upstream_error_code="bridge_continuity_persistence_failed",
|
|
1485
|
+
)
|
|
1486
|
+
raise ProxyResponseError(
|
|
1487
|
+
502,
|
|
1488
|
+
openai_error(
|
|
1489
|
+
"bridge_continuity_persistence_failed",
|
|
1490
|
+
"Response operation continuity could not be persisted; retry the request.",
|
|
1491
|
+
),
|
|
1492
|
+
) from exc
|
|
1493
|
+
if operation is None:
|
|
1494
|
+
session.closed = True
|
|
1495
|
+
session.upstream_control.reconnect_requested = True
|
|
1496
|
+
session.upstream_control.retire_after_drain = True
|
|
1497
|
+
raise ProxyResponseError(
|
|
1498
|
+
502,
|
|
1499
|
+
openai_error(
|
|
1500
|
+
"bridge_continuity_persistence_failed",
|
|
1501
|
+
"HTTP responses session ownership changed; retry the request.",
|
|
1502
|
+
),
|
|
1503
|
+
)
|
|
1504
|
+
if recovery_attempt_consumed and operation.created:
|
|
1505
|
+
raise ProxyResponseError(
|
|
1506
|
+
502,
|
|
1507
|
+
openai_error(
|
|
1508
|
+
"bridge_continuity_persistence_failed",
|
|
1509
|
+
"The recovery checkpoint was already consumed; retry the request.",
|
|
1510
|
+
),
|
|
1511
|
+
)
|
|
1512
|
+
if not operation.created and operation.state == "abandoned":
|
|
1513
|
+
_record_continuity_fail_closed(
|
|
1514
|
+
surface="http_bridge",
|
|
1515
|
+
reason="abandoned_operation_full_history_recovery",
|
|
1516
|
+
previous_response_id=request_state.previous_response_id,
|
|
1517
|
+
session_id=request_state.session_id,
|
|
1518
|
+
upstream_error_code="previous_response_not_found",
|
|
1519
|
+
)
|
|
1520
|
+
# The recovery journal may already hold an UNKNOWN row for
|
|
1521
|
+
# this request. Release it before rejecting so an identical
|
|
1522
|
+
# anchored resend sees the same 400 instead of a 502
|
|
1523
|
+
# "another recovery request is already in flight".
|
|
1524
|
+
await self._cleanup_http_bridge_submit_interruption(
|
|
1525
|
+
session,
|
|
1526
|
+
request_state=request_state,
|
|
1527
|
+
gate_acquired=False,
|
|
1528
|
+
request_enqueued=False,
|
|
1529
|
+
counted_in_queue=False,
|
|
1530
|
+
)
|
|
1531
|
+
raise ProxyResponseError(
|
|
1532
|
+
400,
|
|
1533
|
+
_http_bridge_client_full_history_recovery_error(),
|
|
1534
|
+
)
|
|
1535
|
+
if not operation.created and not getattr(operation, "rebound", False):
|
|
1536
|
+
if operation.state in {"completed", "incomplete"}:
|
|
1537
|
+
if getattr(operation, "event_spool_complete", False):
|
|
1538
|
+
get_operation_events = getattr(self._durable_bridge, "get_operation_events", None)
|
|
1539
|
+
replay_events = (
|
|
1540
|
+
await get_operation_events(operation_id=operation.operation_id)
|
|
1541
|
+
if callable(get_operation_events)
|
|
1542
|
+
else []
|
|
1543
|
+
)
|
|
1544
|
+
if replay_events and request_state.event_queue is not None:
|
|
1545
|
+
request_state.operation_replay = True
|
|
1546
|
+
request_state.operation_id = operation.operation_id
|
|
1547
|
+
request_state.operation_fingerprint = operation_fingerprint
|
|
1548
|
+
request_state.operation_registered = True
|
|
1549
|
+
for replay_event in replay_events:
|
|
1550
|
+
await request_state.event_queue.put(replay_event)
|
|
1551
|
+
await request_state.event_queue.put(None)
|
|
1552
|
+
return
|
|
1553
|
+
if recovery_attempt_consumed:
|
|
1554
|
+
raise ProxyResponseError(
|
|
1555
|
+
502,
|
|
1556
|
+
openai_error(
|
|
1557
|
+
"bridge_continuity_persistence_failed",
|
|
1558
|
+
"The recovery checkpoint was already consumed; retry the request.",
|
|
1559
|
+
),
|
|
1560
|
+
)
|
|
1561
|
+
# A prior dispatch with the same parent and body may have been
|
|
1562
|
+
# accepted by upstream. Without upstream idempotency/status
|
|
1563
|
+
# proof, never submit it a second time.
|
|
1564
|
+
_record_continuity_fail_closed(
|
|
1565
|
+
surface="http_bridge",
|
|
1566
|
+
reason="operation_already_recorded_no_status_proof",
|
|
1567
|
+
previous_response_id=request_state.previous_response_id,
|
|
1568
|
+
session_id=request_state.session_id,
|
|
1569
|
+
upstream_error_code="upstream_operation_status_unknown",
|
|
1570
|
+
)
|
|
1571
|
+
retry_after_seconds = max(
|
|
1572
|
+
1,
|
|
1573
|
+
math.ceil(await self._http_bridge_precreated_retry_cooldown_seconds(session)),
|
|
1574
|
+
)
|
|
1575
|
+
raise ProxyResponseError(
|
|
1576
|
+
503,
|
|
1577
|
+
openai_error(
|
|
1578
|
+
"upstream_operation_status_unknown",
|
|
1579
|
+
"The previous response operation may still be running; retry after the cooldown.",
|
|
1580
|
+
),
|
|
1581
|
+
retry_after_seconds=retry_after_seconds,
|
|
1582
|
+
)
|
|
1583
|
+
request_state.operation_id = operation.operation_id
|
|
1584
|
+
request_state.operation_fingerprint = operation_fingerprint
|
|
1585
|
+
request_state.operation_parent_response_id = operation_parent_response_id
|
|
1586
|
+
request_state.operation_registered = True
|
|
1587
|
+
request_state.operation_rebind_required = False
|
|
1588
|
+
request_state.operation_created = operation.created
|
|
1589
|
+
request_state.operation_rebound = getattr(operation, "rebound", False)
|
|
1590
|
+
request_state.operation_rebound_from_session_id = getattr(operation, "rebound_from_session_id", None)
|
|
1591
|
+
request_state.operation_rebound_from_account_id = getattr(operation, "rebound_from_account_id", None)
|
|
1592
|
+
request_state.operation_rebound_from_model = getattr(operation, "rebound_from_model", None)
|
|
1593
|
+
request_state.operation_rebound_from_parent_response_id = getattr(
|
|
1594
|
+
operation, "rebound_from_parent_response_id", None
|
|
1595
|
+
)
|
|
1596
|
+
request_state.operation_persisted_response_id = getattr(operation, "response_id", None)
|
|
1597
|
+
|
|
1598
|
+
async def _cleanup_unsubmitted_recovery_claim() -> None:
|
|
1599
|
+
if (
|
|
1600
|
+
not request_state.operation_created and not request_state.operation_rebound
|
|
1601
|
+
) or request_state.operation_dispatched:
|
|
1602
|
+
return
|
|
1603
|
+
await self._cleanup_http_bridge_submit_interruption(
|
|
1604
|
+
session,
|
|
1605
|
+
request_state=request_state,
|
|
1606
|
+
gate_acquired=False,
|
|
1607
|
+
request_enqueued=False,
|
|
1608
|
+
counted_in_queue=False,
|
|
1609
|
+
)
|
|
1610
|
+
|
|
1611
|
+
text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data)
|
|
1612
|
+
if request_state.response_id is not None or request_state.response_event_count > 0:
|
|
1613
|
+
await _cleanup_unsubmitted_recovery_claim()
|
|
1614
|
+
_log_http_bridge_event(
|
|
1615
|
+
"submit_after_response_event",
|
|
1616
|
+
session.key,
|
|
1617
|
+
account_id=session.account.id,
|
|
1618
|
+
model=session.request_model,
|
|
1619
|
+
detail=(
|
|
1620
|
+
f"response_id={request_state.response_id}, "
|
|
1621
|
+
f"response_events_seen={request_state.response_event_count}"
|
|
1622
|
+
),
|
|
1623
|
+
cache_key_family=session.key.affinity_kind,
|
|
1624
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1625
|
+
)
|
|
1626
|
+
raise ProxyResponseError(
|
|
1627
|
+
502,
|
|
1628
|
+
openai_error(
|
|
1629
|
+
"upstream_unavailable",
|
|
1630
|
+
"HTTP responses session bridge request already has upstream response events",
|
|
1631
|
+
error_type="server_error",
|
|
1632
|
+
),
|
|
1633
|
+
)
|
|
1634
|
+
if session.upstream_control.retire_after_drain and not owned_unanchored_handoff:
|
|
1635
|
+
await _cleanup_unsubmitted_recovery_claim()
|
|
1636
|
+
await self._release_http_bridge_admission_preregistration(session, request_state=request_state)
|
|
1637
|
+
if not session.upstream_close_attempted:
|
|
1638
|
+
await self._retire_http_bridge_after_drain_if_ready(session)
|
|
1639
|
+
raise ProxyResponseError(
|
|
1640
|
+
502,
|
|
1641
|
+
openai_error("upstream_unavailable", "HTTP responses session bridge is retiring"),
|
|
1642
|
+
)
|
|
1643
|
+
if session.closed:
|
|
1644
|
+
async with session.lifecycle_lock:
|
|
1645
|
+
if session.closed:
|
|
1646
|
+
current_session = session
|
|
1647
|
+
http_bridge_sessions = getattr(self, "_http_bridge_sessions", None)
|
|
1648
|
+
bridge_lock = getattr(self, "_http_bridge_lock", None)
|
|
1649
|
+
if bridge_lock is not None:
|
|
1650
|
+
async with bridge_lock:
|
|
1651
|
+
if http_bridge_sessions is not None:
|
|
1652
|
+
current_session = http_bridge_sessions.get(session.key)
|
|
1653
|
+
elif http_bridge_sessions is not None:
|
|
1654
|
+
current_session = http_bridge_sessions.get(session.key)
|
|
1655
|
+
if current_session is None and _http_bridge_key_strength(session.key) == "hard":
|
|
1656
|
+
await _cleanup_unsubmitted_recovery_claim()
|
|
1657
|
+
_log_http_bridge_event(
|
|
1658
|
+
"submit_on_closed",
|
|
1659
|
+
session.key,
|
|
1660
|
+
account_id=session.account.id,
|
|
1661
|
+
model=session.request_model,
|
|
1662
|
+
detail="session_unregistered_before_reconnect",
|
|
1663
|
+
cache_key_family=session.key.affinity_kind,
|
|
1664
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1665
|
+
)
|
|
1666
|
+
raise ProxyResponseError(
|
|
1667
|
+
502,
|
|
1668
|
+
openai_error("upstream_unavailable", "HTTP responses session bridge is closed"),
|
|
1669
|
+
)
|
|
1670
|
+
if current_session is not session:
|
|
1671
|
+
_log_http_bridge_event(
|
|
1672
|
+
"submit_on_closed",
|
|
1673
|
+
session.key,
|
|
1674
|
+
account_id=session.account.id,
|
|
1675
|
+
model=session.request_model,
|
|
1676
|
+
detail="session_replaced_before_reconnect",
|
|
1677
|
+
cache_key_family=session.key.affinity_kind,
|
|
1678
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1679
|
+
)
|
|
1680
|
+
# Try reconnecting the upstream websocket first. For requests
|
|
1681
|
+
# carrying previous_response_id we only reconnect (send_request=
|
|
1682
|
+
# False) because the fresh upstream won't recognise the old
|
|
1683
|
+
# response id. If reconnection itself fails, raise 502 so the
|
|
1684
|
+
# client retries with previous_response_id intact rather than
|
|
1685
|
+
# receiving 400 previous_response_not_found (which causes the
|
|
1686
|
+
# CLI to drop previous_response_id and resend the full
|
|
1687
|
+
# conversation history, inflating per-turn context by ~20x).
|
|
1688
|
+
try:
|
|
1689
|
+
recovered = await self._retry_http_bridge_request_on_fresh_upstream(
|
|
1690
|
+
session,
|
|
1691
|
+
request_state=request_state,
|
|
1692
|
+
text_data=text_data,
|
|
1693
|
+
send_request=False,
|
|
1694
|
+
require_same_account=_http_bridge_key_strength(session.key) == "hard",
|
|
1695
|
+
)
|
|
1696
|
+
except BaseException:
|
|
1697
|
+
await _cleanup_unsubmitted_recovery_claim()
|
|
1698
|
+
raise
|
|
1699
|
+
if recovered:
|
|
1700
|
+
session.closed = False
|
|
1701
|
+
else:
|
|
1702
|
+
await _cleanup_unsubmitted_recovery_claim()
|
|
1703
|
+
_log_http_bridge_event(
|
|
1704
|
+
"submit_on_closed",
|
|
1705
|
+
session.key,
|
|
1706
|
+
account_id=session.account.id,
|
|
1707
|
+
model=session.request_model,
|
|
1708
|
+
cache_key_family=session.key.affinity_kind,
|
|
1709
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1710
|
+
)
|
|
1711
|
+
raise ProxyResponseError(
|
|
1712
|
+
502,
|
|
1713
|
+
openai_error("upstream_unavailable", "HTTP responses session bridge is closed"),
|
|
1714
|
+
)
|
|
1715
|
+
text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data)
|
|
1716
|
+
request_state.session_previous_gap_ms = int(max(0.0, request_state.started_at - session.last_used_at) * 1000)
|
|
1717
|
+
gate_acquired = False
|
|
1718
|
+
request_enqueued = False
|
|
1719
|
+
admission_waiter_registered = False
|
|
1720
|
+
try:
|
|
1721
|
+
# Register the submit as an admission waiter BEFORE any suspension
|
|
1722
|
+
# outside the lock: the waiter count keeps the idle sweeper and
|
|
1723
|
+
# close-time retire from evicting the session while the settings
|
|
1724
|
+
# snapshot below resolves, and a previous turn's finalizer
|
|
1725
|
+
# unwinding concurrently cannot see an apparently idle session
|
|
1726
|
+
# and release the lease the reacquire installs.
|
|
1727
|
+
async with session.pending_lock:
|
|
1728
|
+
if request_state.admission_waiter_preregistered:
|
|
1729
|
+
# The submit entry already counted this turn; take that
|
|
1730
|
+
# registration over instead of counting it twice.
|
|
1731
|
+
request_state.admission_waiter_preregistered = False
|
|
1732
|
+
else:
|
|
1733
|
+
session.admission_waiter_count += 1
|
|
1734
|
+
admission_waiter_registered = True
|
|
1735
|
+
# Snapshot under the same lock: with the waiter registered, a
|
|
1736
|
+
# held lease cannot be idle-released, so a session that does
|
|
1737
|
+
# not need reacquisition here will not need it at the
|
|
1738
|
+
# reacquire either.
|
|
1739
|
+
needs_stream_lease = session.account_lease is None and not session.closed
|
|
1740
|
+
# Resolved before the reacquire's pending_lock — the settings-cache
|
|
1741
|
+
# refresh behind this can run a DB query and must not suspend the
|
|
1742
|
+
# critical section (issue #1971) — and only when the reacquire can
|
|
1743
|
+
# actually run: a session already holding its lease never depended
|
|
1744
|
+
# on a settings read to admit a turn.
|
|
1745
|
+
fair_share_threshold_pct, routing_tunables = (
|
|
1746
|
+
await self._http_bridge_reacquire_snapshot(session) if needs_stream_lease else (0, None)
|
|
1747
|
+
)
|
|
1748
|
+
async with session.pending_lock:
|
|
1749
|
+
await self._ensure_http_bridge_session_stream_lease_locked(
|
|
1750
|
+
session,
|
|
1751
|
+
request_state=request_state,
|
|
1752
|
+
fair_share_threshold_pct=fair_share_threshold_pct,
|
|
1753
|
+
routing_tunables=routing_tunables,
|
|
1754
|
+
)
|
|
1755
|
+
except BaseException:
|
|
1756
|
+
# Recovery claims are made before admission. If reacquiring an
|
|
1757
|
+
# idle session's stream lease fails, no upstream frame can have
|
|
1758
|
+
# been sent; restore that claim before propagating the admission
|
|
1759
|
+
# error so a later reconnect is not fenced as already dispatched.
|
|
1760
|
+
if getattr(session, "unanchored_reservation_id", None) == request_scope_id:
|
|
1761
|
+
session.unanchored_reservation_id = None
|
|
1762
|
+
cleanup_task = scheduler_for(self).create_task(
|
|
1763
|
+
self._cleanup_http_bridge_submit_interruption(
|
|
1764
|
+
session,
|
|
1765
|
+
request_state=request_state,
|
|
1766
|
+
gate_acquired=False,
|
|
1767
|
+
request_enqueued=False,
|
|
1768
|
+
counted_in_queue=False,
|
|
1769
|
+
admission_waiter_registered=admission_waiter_registered,
|
|
1770
|
+
)
|
|
1771
|
+
)
|
|
1772
|
+
await _await_task_deferring_cancellation(cleanup_task)
|
|
1773
|
+
raise
|
|
1774
|
+
try:
|
|
1775
|
+
await self._maybe_prewarm_http_bridge_session(
|
|
1776
|
+
session,
|
|
1777
|
+
request_state=request_state,
|
|
1778
|
+
text_data=text_data,
|
|
1779
|
+
)
|
|
1780
|
+
except BaseException:
|
|
1781
|
+
if getattr(session, "unanchored_reservation_id", None) == request_scope_id:
|
|
1782
|
+
session.unanchored_reservation_id = None
|
|
1783
|
+
cleanup_task = scheduler_for(self).create_task(
|
|
1784
|
+
self._cleanup_http_bridge_submit_interruption(
|
|
1785
|
+
session,
|
|
1786
|
+
request_state=request_state,
|
|
1787
|
+
gate_acquired=False,
|
|
1788
|
+
request_enqueued=False,
|
|
1789
|
+
counted_in_queue=False,
|
|
1790
|
+
admission_waiter_registered=admission_waiter_registered,
|
|
1791
|
+
)
|
|
1792
|
+
)
|
|
1793
|
+
await _await_task_deferring_cancellation(cleanup_task)
|
|
1794
|
+
raise
|
|
1795
|
+
try:
|
|
1796
|
+
# Reuses the pre-prewarm snapshot: the registered admission waiter
|
|
1797
|
+
# kept the lease from idle release, so this reacquire is a no-op
|
|
1798
|
+
# unless the session closed mid-prewarm — a fresh refresh here
|
|
1799
|
+
# could only stall an otherwise admissible request on a
|
|
1800
|
+
# TTL-expired settings read (issue #1971).
|
|
1801
|
+
async with session.pending_lock:
|
|
1802
|
+
if session.queued_request_count >= queue_limit:
|
|
1803
|
+
_log_http_bridge_event(
|
|
1804
|
+
"bridge_queue_full",
|
|
1805
|
+
session.key,
|
|
1806
|
+
account_id=session.account.id,
|
|
1807
|
+
model=session.request_model,
|
|
1808
|
+
pending_count=session.queued_request_count,
|
|
1809
|
+
cache_key_family=session.key.affinity_kind,
|
|
1810
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1811
|
+
)
|
|
1812
|
+
raise ProxyResponseError(
|
|
1813
|
+
429,
|
|
1814
|
+
openai_error(
|
|
1815
|
+
"bridge_queue_full",
|
|
1816
|
+
"HTTP responses session bridge queue is full",
|
|
1817
|
+
error_type="rate_limit_error",
|
|
1818
|
+
),
|
|
1819
|
+
)
|
|
1820
|
+
await self._ensure_http_bridge_session_stream_lease_locked(
|
|
1821
|
+
session,
|
|
1822
|
+
request_state=request_state,
|
|
1823
|
+
fair_share_threshold_pct=fair_share_threshold_pct,
|
|
1824
|
+
routing_tunables=routing_tunables,
|
|
1825
|
+
)
|
|
1826
|
+
session.queued_request_count += 1
|
|
1827
|
+
if getattr(session, "unanchored_reservation_id", None) == request_scope_id:
|
|
1828
|
+
session.unanchored_reservation_id = None
|
|
1829
|
+
except BaseException:
|
|
1830
|
+
if getattr(session, "unanchored_reservation_id", None) == request_scope_id:
|
|
1831
|
+
session.unanchored_reservation_id = None
|
|
1832
|
+
cleanup_task = scheduler_for(self).create_task(
|
|
1833
|
+
self._cleanup_http_bridge_submit_interruption(
|
|
1834
|
+
session,
|
|
1835
|
+
request_state=request_state,
|
|
1836
|
+
gate_acquired=False,
|
|
1837
|
+
request_enqueued=False,
|
|
1838
|
+
counted_in_queue=False,
|
|
1839
|
+
admission_waiter_registered=admission_waiter_registered,
|
|
1840
|
+
)
|
|
1841
|
+
)
|
|
1842
|
+
await _await_task_deferring_cancellation(cleanup_task)
|
|
1843
|
+
raise
|
|
1844
|
+
try:
|
|
1845
|
+
text_data = await self._inline_http_bridge_image_urls(text_data, request_state)
|
|
1846
|
+
text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data)
|
|
1847
|
+
self._start_request_state_api_key_reservation_heartbeat(
|
|
1848
|
+
request_state,
|
|
1849
|
+
api_key=request_state.api_key,
|
|
1850
|
+
surface="http_bridge",
|
|
1851
|
+
)
|
|
1852
|
+
_copy_websocket_route_metadata_from_session(request_state, session)
|
|
1853
|
+
request_state.bridge_queue_wait_started_at = clock.monotonic()
|
|
1854
|
+
request_state.response_create_gate_wait_started_at = clock.monotonic()
|
|
1855
|
+
# Bridge ownership is established before this late admission. A
|
|
1856
|
+
# cap race stays a bounded error/wait on that owner; it must not
|
|
1857
|
+
# publish a replacement bridge as a spillover side effect.
|
|
1858
|
+
await self._acquire_request_state_response_create_admission(
|
|
1859
|
+
request_state,
|
|
1860
|
+
response_create_gate=session.response_create_gate,
|
|
1861
|
+
account_id=session.account.id,
|
|
1862
|
+
routing_tunables=routing_tunables,
|
|
1863
|
+
surface="http_bridge",
|
|
1864
|
+
bridge_session=session,
|
|
1865
|
+
)
|
|
1866
|
+
gate_acquired = True
|
|
1867
|
+
if request_state.bridge_queue_wait_started_at is not None:
|
|
1868
|
+
request_state.latency_bridge_queue_wait_ms = int(
|
|
1869
|
+
max(0.0, clock.monotonic() - request_state.bridge_queue_wait_started_at) * 1000
|
|
1870
|
+
)
|
|
1871
|
+
async with session.lifecycle_lock:
|
|
1872
|
+
current_session = session
|
|
1873
|
+
http_bridge_sessions = getattr(self, "_http_bridge_sessions", None)
|
|
1874
|
+
bridge_lock = getattr(self, "_http_bridge_lock", None)
|
|
1875
|
+
if bridge_lock is not None:
|
|
1876
|
+
async with bridge_lock:
|
|
1877
|
+
if http_bridge_sessions is not None:
|
|
1878
|
+
current_session = http_bridge_sessions.get(session.key)
|
|
1879
|
+
elif http_bridge_sessions is not None:
|
|
1880
|
+
current_session = http_bridge_sessions.get(session.key)
|
|
1881
|
+
session_unregistered = current_session is None and _http_bridge_key_strength(session.key) == "hard"
|
|
1882
|
+
session_replaced = current_session is not None and current_session is not session
|
|
1883
|
+
# Queue publication clears the mutable reservation marker. The
|
|
1884
|
+
# proof captured before the first await still authorizes exactly
|
|
1885
|
+
# that request to submit on its detached, draining generation.
|
|
1886
|
+
detached_handoff_can_submit = (
|
|
1887
|
+
owned_unanchored_handoff and session.upstream_control.retire_after_drain and not session.closed
|
|
1888
|
+
)
|
|
1889
|
+
if session.closed and current_session is session and not session.upstream_control.retire_after_drain:
|
|
1890
|
+
recovered = await self._retry_http_bridge_request_on_fresh_upstream(
|
|
1891
|
+
session,
|
|
1892
|
+
request_state=request_state,
|
|
1893
|
+
text_data=text_data,
|
|
1894
|
+
send_request=False,
|
|
1895
|
+
require_same_account=_http_bridge_key_strength(session.key) == "hard",
|
|
1896
|
+
)
|
|
1897
|
+
if recovered:
|
|
1898
|
+
session.closed = False
|
|
1899
|
+
if session.closed or ((session_unregistered or session_replaced) and not detached_handoff_can_submit):
|
|
1900
|
+
_log_http_bridge_event(
|
|
1901
|
+
"submit_on_closed",
|
|
1902
|
+
session.key,
|
|
1903
|
+
account_id=session.account.id,
|
|
1904
|
+
model=session.request_model,
|
|
1905
|
+
detail=(
|
|
1906
|
+
"session_retired_after_admission"
|
|
1907
|
+
if session.closed
|
|
1908
|
+
else (
|
|
1909
|
+
"session_unregistered_after_admission"
|
|
1910
|
+
if session_unregistered
|
|
1911
|
+
else "session_replaced_after_admission"
|
|
1912
|
+
)
|
|
1913
|
+
),
|
|
1914
|
+
cache_key_family=session.key.affinity_kind,
|
|
1915
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1916
|
+
)
|
|
1917
|
+
raise ProxyResponseError(
|
|
1918
|
+
502,
|
|
1919
|
+
openai_error("upstream_unavailable", "HTTP responses session bridge is closed"),
|
|
1920
|
+
)
|
|
1921
|
+
recovery_receipt: DurableBridgeAliasRegistrationReceipt | None = None
|
|
1922
|
+
upstream_send_started = False
|
|
1923
|
+
try:
|
|
1924
|
+
if recovery_turn_state is not None:
|
|
1925
|
+
registration_cancellation: asyncio.CancelledError | None = None
|
|
1926
|
+
try:
|
|
1927
|
+
async with session.recovery_alias_lock:
|
|
1928
|
+
registration_task = scheduler_for(self).create_task(
|
|
1929
|
+
self._register_http_bridge_recovery_turn_state_locked(
|
|
1930
|
+
session,
|
|
1931
|
+
recovery_turn_state,
|
|
1932
|
+
)
|
|
1933
|
+
)
|
|
1934
|
+
(
|
|
1935
|
+
(
|
|
1936
|
+
recovery_alias_registered,
|
|
1937
|
+
recovery_receipt,
|
|
1938
|
+
),
|
|
1939
|
+
registration_cancellation,
|
|
1940
|
+
) = await _await_task_deferring_cancellation(registration_task)
|
|
1941
|
+
if registration_cancellation is not None or not recovery_alias_registered:
|
|
1942
|
+
session.closed = True
|
|
1943
|
+
session.upstream_control.reconnect_requested = True
|
|
1944
|
+
session.upstream_control.retire_after_drain = True
|
|
1945
|
+
if (
|
|
1946
|
+
recovery_receipt is not None
|
|
1947
|
+
and recovery_receipt.status == DurableBridgeAliasRegistration.REGISTERED
|
|
1948
|
+
):
|
|
1949
|
+
try:
|
|
1950
|
+
(
|
|
1951
|
+
rolled_back,
|
|
1952
|
+
rollback_cancellation,
|
|
1953
|
+
) = await _rollback_http_bridge_recovery_turn_state_registration(
|
|
1954
|
+
self,
|
|
1955
|
+
recovery_receipt,
|
|
1956
|
+
)
|
|
1957
|
+
except Exception:
|
|
1958
|
+
rolled_back = False
|
|
1959
|
+
rollback_cancellation = None
|
|
1960
|
+
logger.warning(
|
|
1961
|
+
"Failed to roll back recovered HTTP bridge turn-state alias",
|
|
1962
|
+
exc_info=True,
|
|
1963
|
+
)
|
|
1964
|
+
registration_cancellation = registration_cancellation or rollback_cancellation
|
|
1965
|
+
recovery_receipt = None
|
|
1966
|
+
if not rolled_back:
|
|
1967
|
+
_record_continuity_fail_closed(
|
|
1968
|
+
surface="http_bridge",
|
|
1969
|
+
reason="recovery_alias_rollback_failed",
|
|
1970
|
+
previous_response_id=request_state.previous_response_id,
|
|
1971
|
+
session_id=request_state.session_id,
|
|
1972
|
+
upstream_error_code="bridge_continuity_persistence_failed",
|
|
1973
|
+
)
|
|
1974
|
+
if registration_cancellation is not None:
|
|
1975
|
+
raise registration_cancellation
|
|
1976
|
+
except asyncio.CancelledError:
|
|
1977
|
+
raise
|
|
1978
|
+
except Exception:
|
|
1979
|
+
logger.warning(
|
|
1980
|
+
"Failed to persist recovered HTTP bridge turn-state before upstream dispatch",
|
|
1981
|
+
exc_info=True,
|
|
1982
|
+
)
|
|
1983
|
+
recovery_alias_registered = False
|
|
1984
|
+
if not recovery_alias_registered:
|
|
1985
|
+
session.closed = True
|
|
1986
|
+
session.upstream_control.reconnect_requested = True
|
|
1987
|
+
session.upstream_control.retire_after_drain = True
|
|
1988
|
+
_record_continuity_fail_closed(
|
|
1989
|
+
surface="http_bridge",
|
|
1990
|
+
reason="recovery_alias_registration_failed",
|
|
1991
|
+
previous_response_id=request_state.previous_response_id,
|
|
1992
|
+
session_id=request_state.session_id,
|
|
1993
|
+
upstream_error_code="bridge_continuity_persistence_failed",
|
|
1994
|
+
)
|
|
1995
|
+
raise ProxyResponseError(
|
|
1996
|
+
502,
|
|
1997
|
+
openai_error(
|
|
1998
|
+
"bridge_continuity_persistence_failed",
|
|
1999
|
+
"Recovered response continuity could not be persisted; retry the request.",
|
|
2000
|
+
),
|
|
2001
|
+
)
|
|
2002
|
+
if request_state.recovery_attempt_fingerprint is not None:
|
|
2003
|
+
try:
|
|
2004
|
+
owner_lookup = await self._durable_bridge.renew_live_session(
|
|
2005
|
+
session_id=session.durable_session_id,
|
|
2006
|
+
api_key_id=session.key.api_key_id,
|
|
2007
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
2008
|
+
owner_epoch=session.durable_owner_epoch,
|
|
2009
|
+
lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(),
|
|
2010
|
+
latest_turn_state=session.downstream_turn_state,
|
|
2011
|
+
latest_response_id=None,
|
|
2012
|
+
)
|
|
2013
|
+
except Exception as exc:
|
|
2014
|
+
session.closed = True
|
|
2015
|
+
session.upstream_control.reconnect_requested = True
|
|
2016
|
+
session.upstream_control.retire_after_drain = True
|
|
2017
|
+
raise ProxyResponseError(
|
|
2018
|
+
502,
|
|
2019
|
+
openai_error(
|
|
2020
|
+
"bridge_continuity_persistence_failed",
|
|
2021
|
+
"HTTP responses session ownership could not be renewed; retry the request.",
|
|
2022
|
+
),
|
|
2023
|
+
) from exc
|
|
2024
|
+
if (
|
|
2025
|
+
owner_lookup is None
|
|
2026
|
+
or owner_lookup.owner_instance_id
|
|
2027
|
+
!= _service_get_settings().http_responses_session_bridge_instance_id
|
|
2028
|
+
or owner_lookup.owner_epoch != session.durable_owner_epoch
|
|
2029
|
+
):
|
|
2030
|
+
session.closed = True
|
|
2031
|
+
session.upstream_control.reconnect_requested = True
|
|
2032
|
+
session.upstream_control.retire_after_drain = True
|
|
2033
|
+
raise ProxyResponseError(
|
|
2034
|
+
502,
|
|
2035
|
+
openai_error(
|
|
2036
|
+
"bridge_continuity_persistence_failed",
|
|
2037
|
+
"HTTP responses session ownership changed before dispatch; retry the request.",
|
|
2038
|
+
),
|
|
2039
|
+
)
|
|
2040
|
+
# The journal entry is created before queue admission so
|
|
2041
|
+
# concurrent requests cannot both enter the gate without
|
|
2042
|
+
# a recovery generation. Revalidate it after the gate and
|
|
2043
|
+
# lifecycle locks, immediately before dispatch: a waiter
|
|
2044
|
+
# may have observed UNKNOWN while the first request
|
|
2045
|
+
# settled the row REPLAYED.
|
|
2046
|
+
if (
|
|
2047
|
+
request_state.recovery_attempt_fingerprint is not None
|
|
2048
|
+
and not request_state.recovery_attempt_claimed
|
|
2049
|
+
):
|
|
2050
|
+
if (
|
|
2051
|
+
request_state.recovery_attempt_session_id != session.durable_session_id
|
|
2052
|
+
or request_state.recovery_attempt_owner_epoch != session.durable_owner_epoch
|
|
2053
|
+
):
|
|
2054
|
+
session.closed = True
|
|
2055
|
+
session.upstream_control.reconnect_requested = True
|
|
2056
|
+
session.upstream_control.retire_after_drain = True
|
|
2057
|
+
raise ProxyResponseError(
|
|
2058
|
+
502,
|
|
2059
|
+
openai_error(
|
|
2060
|
+
"bridge_continuity_persistence_failed",
|
|
2061
|
+
"HTTP responses session ownership changed before dispatch; retry the request.",
|
|
2062
|
+
),
|
|
2063
|
+
)
|
|
2064
|
+
try:
|
|
2065
|
+
dispatch_attempt = await self._durable_bridge.record_recovery_attempt(
|
|
2066
|
+
session_id=session.durable_session_id,
|
|
2067
|
+
api_key_id=session.key.api_key_id,
|
|
2068
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
2069
|
+
owner_epoch=session.durable_owner_epoch,
|
|
2070
|
+
request_fingerprint=request_state.recovery_attempt_fingerprint,
|
|
2071
|
+
request_id=request_state.request_id,
|
|
2072
|
+
account_id=session.account.id,
|
|
2073
|
+
model=request_state.model,
|
|
2074
|
+
replay_safe=True,
|
|
2075
|
+
)
|
|
2076
|
+
except Exception as exc:
|
|
2077
|
+
session.closed = True
|
|
2078
|
+
session.upstream_control.reconnect_requested = True
|
|
2079
|
+
session.upstream_control.retire_after_drain = True
|
|
2080
|
+
raise ProxyResponseError(
|
|
2081
|
+
502,
|
|
2082
|
+
openai_error(
|
|
2083
|
+
"bridge_continuity_persistence_failed",
|
|
2084
|
+
"Recovered response continuity could not be revalidated; retry the request.",
|
|
2085
|
+
),
|
|
2086
|
+
) from exc
|
|
2087
|
+
if (
|
|
2088
|
+
dispatch_attempt is None
|
|
2089
|
+
or getattr(dispatch_attempt.state, "value", dispatch_attempt.state) != "unknown"
|
|
2090
|
+
or getattr(dispatch_attempt, "request_id", request_state.request_id)
|
|
2091
|
+
!= request_state.request_id
|
|
2092
|
+
):
|
|
2093
|
+
session.closed = True
|
|
2094
|
+
session.upstream_control.reconnect_requested = True
|
|
2095
|
+
session.upstream_control.retire_after_drain = True
|
|
2096
|
+
raise ProxyResponseError(
|
|
2097
|
+
502,
|
|
2098
|
+
openai_error(
|
|
2099
|
+
"bridge_continuity_persistence_failed",
|
|
2100
|
+
"The recovery checkpoint was consumed before dispatch; retry the request.",
|
|
2101
|
+
),
|
|
2102
|
+
)
|
|
2103
|
+
if (
|
|
2104
|
+
request_state.proxy_injected_previous_response_id
|
|
2105
|
+
and request_state.previous_response_id is not None
|
|
2106
|
+
and (
|
|
2107
|
+
request_state.previous_response_id in session.denied_proxy_injected_anchor_ids
|
|
2108
|
+
or request_state.denied_proxy_injected_anchor_fence_was_already_denied
|
|
2109
|
+
or _http_bridge_denied_anchor_fence_advanced(self, request_state)
|
|
2110
|
+
)
|
|
2111
|
+
):
|
|
2112
|
+
_record_continuity_fail_closed(
|
|
2113
|
+
surface="http_bridge",
|
|
2114
|
+
reason="denied_proxy_anchor_before_dispatch",
|
|
2115
|
+
previous_response_id=request_state.previous_response_id,
|
|
2116
|
+
session_id=request_state.session_id,
|
|
2117
|
+
upstream_error_code="previous_response_not_found",
|
|
2118
|
+
)
|
|
2119
|
+
# The fence sits above the pending-request enqueue and
|
|
2120
|
+
# the upstream send, so this refusal is proven to be
|
|
2121
|
+
# ours rather than an upstream transport failure. Say
|
|
2122
|
+
# so on the exception: without the flag the native
|
|
2123
|
+
# Codex lifecycle aborts the committed body with no
|
|
2124
|
+
# terminal event and the client sees an empty 200
|
|
2125
|
+
# (issue #2364).
|
|
2126
|
+
raise ProxyResponseError(
|
|
2127
|
+
502,
|
|
2128
|
+
openai_error(
|
|
2129
|
+
"stream_incomplete",
|
|
2130
|
+
"The previous response anchor was rejected upstream; retry the request.",
|
|
2131
|
+
),
|
|
2132
|
+
local_pre_dispatch_refusal=True,
|
|
2133
|
+
)
|
|
2134
|
+
if (
|
|
2135
|
+
request_state.verified_stale_anchor_replay
|
|
2136
|
+
and request_state.verified_stale_anchor_retry_circuit_generation_captured
|
|
2137
|
+
):
|
|
2138
|
+
circuit_key = request_state.verified_stale_anchor_retry_circuit_key
|
|
2139
|
+
claim_outcome: bool | None = False
|
|
2140
|
+
if circuit_key is not None:
|
|
2141
|
+
claim_outcome = await self._claim_http_bridge_retry_circuit_generation(
|
|
2142
|
+
key=circuit_key,
|
|
2143
|
+
captured=request_state.verified_stale_anchor_retry_circuit_generation_captured,
|
|
2144
|
+
generation=request_state.verified_stale_anchor_retry_circuit_generation,
|
|
2145
|
+
)
|
|
2146
|
+
generation_claimed = claim_outcome is True
|
|
2147
|
+
if not generation_claimed:
|
|
2148
|
+
remote_probe_holds_lease = False
|
|
2149
|
+
if claim_outcome is False and circuit_key is not None:
|
|
2150
|
+
remote_probe_holds_lease = await self._http_bridge_claim_miss_shows_remote_probe(
|
|
2151
|
+
circuit_key,
|
|
2152
|
+
request_state.verified_stale_anchor_retry_circuit_generation,
|
|
2153
|
+
)
|
|
2154
|
+
# The block lives on the source hard key: this
|
|
2155
|
+
# replacement session's own unique key never has a
|
|
2156
|
+
# circuit, and the source key's cooldown alone is
|
|
2157
|
+
# already zero during its half-open lease.
|
|
2158
|
+
(
|
|
2159
|
+
suppressed_block_seconds,
|
|
2160
|
+
suppressed_block_reason,
|
|
2161
|
+
) = await self._http_bridge_precreated_retry_block_for_key(
|
|
2162
|
+
circuit_key or session.key,
|
|
2163
|
+
assume_remote_half_open_lease=remote_probe_holds_lease,
|
|
2164
|
+
)
|
|
2165
|
+
suppressed_retry_after_seconds = max(1, math.ceil(suppressed_block_seconds))
|
|
2166
|
+
_log_http_bridge_event(
|
|
2167
|
+
"stale_anchor_replay_generation_suppressed",
|
|
2168
|
+
circuit_key or session.key,
|
|
2169
|
+
account_id=session.account.id,
|
|
2170
|
+
model=session.request_model,
|
|
2171
|
+
detail="circuit_generation_advanced_before_dispatch_claim",
|
|
2172
|
+
cache_key_family=(circuit_key or session.key).affinity_kind,
|
|
2173
|
+
model_class=(
|
|
2174
|
+
_extract_model_class(session.request_model) if session.request_model else None
|
|
2175
|
+
),
|
|
2176
|
+
)
|
|
2177
|
+
raise ProxyResponseError(
|
|
2178
|
+
503,
|
|
2179
|
+
openai_error(
|
|
2180
|
+
"upstream_request_timeout",
|
|
2181
|
+
_http_bridge_retry_circuit_suppression_message(
|
|
2182
|
+
suppressed_block_reason, suppressed_retry_after_seconds
|
|
2183
|
+
),
|
|
2184
|
+
),
|
|
2185
|
+
retry_after_seconds=suppressed_retry_after_seconds,
|
|
2186
|
+
)
|
|
2187
|
+
async with session.pending_lock:
|
|
2188
|
+
session.pending_requests.append(request_state)
|
|
2189
|
+
session.admission_waiter_count = max(0, session.admission_waiter_count - 1)
|
|
2190
|
+
admission_waiter_registered = False
|
|
2191
|
+
request_enqueued = True
|
|
2192
|
+
|
|
2193
|
+
def mark_upstream_send_started() -> None:
|
|
2194
|
+
nonlocal upstream_send_started
|
|
2195
|
+
# The helper invokes this only after the final frame
|
|
2196
|
+
# size preflight. A payload_too_large rejection must
|
|
2197
|
+
# therefore remain proven pre-dispatch so cleanup can
|
|
2198
|
+
# roll back a newly-created operation.
|
|
2199
|
+
upstream_send_started = True
|
|
2200
|
+
|
|
2201
|
+
try:
|
|
2202
|
+
await _send_http_bridge_request_text_with_archive_id(
|
|
2203
|
+
session,
|
|
2204
|
+
request_state,
|
|
2205
|
+
text_data,
|
|
2206
|
+
on_send_started=mark_upstream_send_started,
|
|
2207
|
+
clock=clock,
|
|
2208
|
+
)
|
|
2209
|
+
except BaseException as exc:
|
|
2210
|
+
request_state.recovery_attempt_dispatched = upstream_send_started
|
|
2211
|
+
request_state.operation_dispatched = (
|
|
2212
|
+
request_state.operation_id is not None and upstream_send_started
|
|
2213
|
+
)
|
|
2214
|
+
# Publish retirement while lifecycle ownership is still
|
|
2215
|
+
# held; a gate waiter must never reuse an ambiguously sent
|
|
2216
|
+
# response.create socket between unlock and cleanup.
|
|
2217
|
+
session.closed = True
|
|
2218
|
+
session.upstream_control.reconnect_requested = True
|
|
2219
|
+
session.upstream_control.retire_after_drain = True
|
|
2220
|
+
if (
|
|
2221
|
+
isinstance(exc, UpstreamWebSocketTransportError)
|
|
2222
|
+
and exc.error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE
|
|
2223
|
+
):
|
|
2224
|
+
# Only this narrow claim, not ``closed``, tells the
|
|
2225
|
+
# reader that the submitter will settle siblings.
|
|
2226
|
+
# Keep it inside lifecycle_lock with the failing
|
|
2227
|
+
# send so the reader cannot observe an ownership gap.
|
|
2228
|
+
session.claim_liveness_settlement()
|
|
2229
|
+
raise
|
|
2230
|
+
request_state.recovery_attempt_dispatched = True
|
|
2231
|
+
request_state.operation_dispatched = request_state.operation_id is not None
|
|
2232
|
+
session.last_used_at = clock.monotonic()
|
|
2233
|
+
except asyncio.CancelledError:
|
|
2234
|
+
if recovery_receipt is not None and not upstream_send_started:
|
|
2235
|
+
session.closed = True
|
|
2236
|
+
session.upstream_control.reconnect_requested = True
|
|
2237
|
+
session.upstream_control.retire_after_drain = True
|
|
2238
|
+
async with session.recovery_alias_lock:
|
|
2239
|
+
try:
|
|
2240
|
+
(
|
|
2241
|
+
rolled_back,
|
|
2242
|
+
_rollback_cancellation,
|
|
2243
|
+
) = await _rollback_http_bridge_recovery_turn_state_registration(
|
|
2244
|
+
self,
|
|
2245
|
+
recovery_receipt,
|
|
2246
|
+
)
|
|
2247
|
+
except Exception:
|
|
2248
|
+
rolled_back = False
|
|
2249
|
+
logger.warning(
|
|
2250
|
+
"Failed to roll back cancelled HTTP bridge recovery alias",
|
|
2251
|
+
exc_info=True,
|
|
2252
|
+
)
|
|
2253
|
+
if not rolled_back:
|
|
2254
|
+
_record_continuity_fail_closed(
|
|
2255
|
+
surface="http_bridge",
|
|
2256
|
+
reason="recovery_alias_rollback_failed",
|
|
2257
|
+
previous_response_id=request_state.previous_response_id,
|
|
2258
|
+
session_id=request_state.session_id,
|
|
2259
|
+
upstream_error_code="bridge_continuity_persistence_failed",
|
|
2260
|
+
)
|
|
2261
|
+
raise
|
|
2262
|
+
except ProxyResponseError:
|
|
2263
|
+
await self._cleanup_http_bridge_submit_interruption(
|
|
2264
|
+
session,
|
|
2265
|
+
request_state=request_state,
|
|
2266
|
+
gate_acquired=gate_acquired,
|
|
2267
|
+
request_enqueued=request_enqueued,
|
|
2268
|
+
counted_in_queue=True,
|
|
2269
|
+
admission_waiter_registered=admission_waiter_registered,
|
|
2270
|
+
)
|
|
2271
|
+
if not session.upstream_close_attempted:
|
|
2272
|
+
await self._retire_http_bridge_after_drain_if_ready(session)
|
|
2273
|
+
raise
|
|
2274
|
+
except asyncio.CancelledError as cancellation:
|
|
2275
|
+
cleanup_task = scheduler_for(self).create_task(
|
|
2276
|
+
self._cleanup_http_bridge_submit_interruption(
|
|
2277
|
+
session,
|
|
2278
|
+
request_state=request_state,
|
|
2279
|
+
gate_acquired=gate_acquired,
|
|
2280
|
+
request_enqueued=request_enqueued,
|
|
2281
|
+
counted_in_queue=True,
|
|
2282
|
+
admission_waiter_registered=admission_waiter_registered,
|
|
2283
|
+
)
|
|
2284
|
+
)
|
|
2285
|
+
try:
|
|
2286
|
+
await _await_task_deferring_cancellation(cleanup_task)
|
|
2287
|
+
except Exception:
|
|
2288
|
+
logger.warning("Failed to clean up cancelled HTTP bridge submit", exc_info=True)
|
|
2289
|
+
if session.upstream_control.retire_after_drain and not session.upstream_close_attempted:
|
|
2290
|
+
retire_task = scheduler_for(self).create_task(self._retire_http_bridge_after_drain_if_ready(session))
|
|
2291
|
+
try:
|
|
2292
|
+
await _await_task_deferring_cancellation(retire_task)
|
|
2293
|
+
except Exception:
|
|
2294
|
+
logger.warning("Failed to retire cancelled HTTP bridge recovery", exc_info=True)
|
|
2295
|
+
raise cancellation
|
|
2296
|
+
except Exception as exc:
|
|
2297
|
+
_log_http_bridge_event(
|
|
2298
|
+
"send_failure",
|
|
2299
|
+
session.key,
|
|
2300
|
+
account_id=session.account.id,
|
|
2301
|
+
model=session.request_model,
|
|
2302
|
+
detail=str(exc) or None,
|
|
2303
|
+
cache_key_family=session.key.affinity_kind,
|
|
2304
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
2305
|
+
)
|
|
2306
|
+
# send_text may fail after the complete response.create frame was
|
|
2307
|
+
# handed to the kernel. Never reconnect-and-resend from this path;
|
|
2308
|
+
# only failures proven to precede dispatch may be replayed.
|
|
2309
|
+
error_code = exc.error_code if isinstance(exc, UpstreamWebSocketTransportError) else "stream_incomplete"
|
|
2310
|
+
failure_error_message = str(exc) or "Upstream websocket closed before response.completed"
|
|
2311
|
+
# Liveness expiry and local network loss are transport failures,
|
|
2312
|
+
# not evidence against the selected account. Keep this in sync
|
|
2313
|
+
# with the reader path's shared provenance classification.
|
|
2314
|
+
account_neutral = is_account_neutral_websocket_error_code(error_code)
|
|
2315
|
+
if error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE:
|
|
2316
|
+
# The sender claimed ownership beside the failing send while
|
|
2317
|
+
# holding lifecycle_lock. It therefore owns the entire session
|
|
2318
|
+
# deque, including older in-flight requests; settling only this
|
|
2319
|
+
# request would strand its siblings after the reader yields.
|
|
2320
|
+
# Publish the cleanup task before the first await after the
|
|
2321
|
+
# claim. Shielding it makes cancellation wait for settlement,
|
|
2322
|
+
# so the claim can never outlive its exactly-once owner.
|
|
2323
|
+
settlement_task = scheduler_for(self).create_task(
|
|
2324
|
+
_settle_claimed_http_bridge_liveness_failure(
|
|
2325
|
+
self,
|
|
2326
|
+
session,
|
|
2327
|
+
error_message=str(exc) or "Upstream websocket liveness failed",
|
|
2328
|
+
),
|
|
2329
|
+
name="http-bridge-liveness-send-settlement",
|
|
2330
|
+
)
|
|
2331
|
+
_, settlement_cancellation = await _await_task_deferring_cancellation(settlement_task)
|
|
2332
|
+
if settlement_cancellation is not None:
|
|
2333
|
+
raise settlement_cancellation
|
|
2334
|
+
else:
|
|
2335
|
+
# Once the operation-tagged frame has been handed to the
|
|
2336
|
+
# socket, the transport exception is ambiguous: upstream may
|
|
2337
|
+
# have accepted it even though this worker saw no
|
|
2338
|
+
# acknowledgement. Persist UNKNOWN under the owner fence
|
|
2339
|
+
# before cleanup can retire the closed session and release
|
|
2340
|
+
# that fence.
|
|
2341
|
+
if (
|
|
2342
|
+
request_state.operation_dispatched
|
|
2343
|
+
and request_state.operation_registered
|
|
2344
|
+
and request_state.operation_id is not None
|
|
2345
|
+
and session.durable_session_id is not None
|
|
2346
|
+
and session.durable_owner_epoch is not None
|
|
2347
|
+
):
|
|
2348
|
+
mark_operation_unknown = getattr(self._durable_bridge, "mark_operation_unknown", None)
|
|
2349
|
+
marked_unknown = False
|
|
2350
|
+
if callable(mark_operation_unknown):
|
|
2351
|
+
try:
|
|
2352
|
+
marked_unknown = await mark_operation_unknown(
|
|
2353
|
+
operation_id=request_state.operation_id,
|
|
2354
|
+
session_id=session.durable_session_id,
|
|
2355
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
2356
|
+
owner_epoch=session.durable_owner_epoch,
|
|
2357
|
+
)
|
|
2358
|
+
except Exception:
|
|
2359
|
+
logger.warning(
|
|
2360
|
+
"Failed to mark ambiguous HTTP bridge operation UNKNOWN operation_id=%s",
|
|
2361
|
+
request_state.operation_id,
|
|
2362
|
+
exc_info=True,
|
|
2363
|
+
)
|
|
2364
|
+
if not marked_unknown:
|
|
2365
|
+
request_state.operation_registered = False
|
|
2366
|
+
error_code = "bridge_continuity_persistence_failed"
|
|
2367
|
+
failure_error_message = (
|
|
2368
|
+
"Ambiguous response operation could not be persisted; retry the request."
|
|
2369
|
+
)
|
|
2370
|
+
_record_continuity_fail_closed(
|
|
2371
|
+
surface="http_bridge",
|
|
2372
|
+
reason="ambiguous_operation_unknown_persistence_failed",
|
|
2373
|
+
previous_response_id=request_state.previous_response_id,
|
|
2374
|
+
session_id=request_state.session_id,
|
|
2375
|
+
upstream_error_code=error_code,
|
|
2376
|
+
)
|
|
2377
|
+
await self._cleanup_http_bridge_submit_interruption(
|
|
2378
|
+
session,
|
|
2379
|
+
request_state=request_state,
|
|
2380
|
+
gate_acquired=gate_acquired,
|
|
2381
|
+
request_enqueued=request_enqueued,
|
|
2382
|
+
counted_in_queue=True,
|
|
2383
|
+
admission_waiter_registered=admission_waiter_registered,
|
|
2384
|
+
)
|
|
2385
|
+
await self._fail_pending_websocket_requests(
|
|
2386
|
+
account=session.account,
|
|
2387
|
+
account_id_value=session.account.id,
|
|
2388
|
+
pending_requests=deque([request_state]),
|
|
2389
|
+
pending_lock=fast_lock(),
|
|
2390
|
+
error_code=error_code,
|
|
2391
|
+
error_message=failure_error_message,
|
|
2392
|
+
api_key=None,
|
|
2393
|
+
response_create_gate=session.response_create_gate,
|
|
2394
|
+
penalize_account=not account_neutral,
|
|
2395
|
+
)
|
|
2396
|
+
session.closed = True
|
|
2397
|
+
try:
|
|
2398
|
+
await session.upstream.close()
|
|
2399
|
+
except Exception:
|
|
2400
|
+
logger.debug("Failed to close HTTP bridge upstream websocket after send failure", exc_info=True)
|
|
2401
|
+
# Always raise 502 so the client can retry with
|
|
2402
|
+
# previous_response_id intact. Returning 400
|
|
2403
|
+
# previous_response_not_found causes the client to drop
|
|
2404
|
+
# previous_response_id and resend the full conversation
|
|
2405
|
+
# history, inflating per-turn context by ~20x.
|
|
2406
|
+
raise ProxyResponseError(
|
|
2407
|
+
502,
|
|
2408
|
+
openai_error(error_code, failure_error_message),
|
|
2409
|
+
) from exc
|
|
2410
|
+
|
|
2411
|
+
async def _maybe_prewarm_http_bridge_session(
|
|
2412
|
+
self: Any,
|
|
2413
|
+
session: "_HTTPBridgeSession",
|
|
2414
|
+
*,
|
|
2415
|
+
request_state: _WebSocketRequestState,
|
|
2416
|
+
text_data: str,
|
|
2417
|
+
) -> None:
|
|
2418
|
+
clock = clock_for(self)
|
|
2419
|
+
settings = _service_get_settings()
|
|
2420
|
+
if not session.codex_session or session.prewarmed or request_state.previous_response_id is not None:
|
|
2421
|
+
request_state.prewarm_status = request_state.prewarm_status or "not_applicable"
|
|
2422
|
+
return
|
|
2423
|
+
# M3 codex prewarm: the switch is dashboard-managed, and it reaches this
|
|
2424
|
+
# path through the request-bound overlay already applied by
|
|
2425
|
+
# ``_service_get_settings()`` above (the field is in
|
|
2426
|
+
# ``DASHBOARD_OVERRIDE_SETTINGS``, and the entry-point middleware bound
|
|
2427
|
+
# the snapshot once for this request). Resolving it is therefore a plain
|
|
2428
|
+
# memory read that adds neither a settings read nor an ``await`` -- here
|
|
2429
|
+
# or, more importantly, under ``prewarm_lock`` below, where a cache
|
|
2430
|
+
# refresh could run a DB query and suspend (issues #1971 and #1972
|
|
2431
|
+
# wedged every keyed submit on exactly that pattern).
|
|
2432
|
+
if not _http_bridge_prewarm_enabled(settings):
|
|
2433
|
+
request_state.prewarm_status = "not_applicable"
|
|
2434
|
+
return
|
|
2435
|
+
prewarm_lock = session.prewarm_lock
|
|
2436
|
+
if prewarm_lock is None:
|
|
2437
|
+
request_state.prewarm_status = "skipped"
|
|
2438
|
+
_record_http_bridge_prewarm_outcome(outcome="skipped")
|
|
2439
|
+
return
|
|
2440
|
+
# Build the warm-up outside the lock. It is a pure function of
|
|
2441
|
+
# ``text_data``, so the locked body gets the same answer, and this keeps
|
|
2442
|
+
# a parse of a possibly large payload off the critical section.
|
|
2443
|
+
warmup_text = _build_http_bridge_prewarm_text(text_data)
|
|
2444
|
+
# The two settings readers the locked body reaches -- the admission
|
|
2445
|
+
# gate's account caps/tunables and the reconnect the timeout path takes
|
|
2446
|
+
# -- get this one row instead of reading it themselves, so neither
|
|
2447
|
+
# awaits the settings cache while ``prewarm_lock`` is held. A refresh
|
|
2448
|
+
# behind that read runs a DB query under a process-global lock; one
|
|
2449
|
+
# stalled query would then hold this session's prewarm lock and stall
|
|
2450
|
+
# every later turn on it (issues #1971 and #1972). A payload with no
|
|
2451
|
+
# warm-up to send reaches neither reader, so it resolves nothing and
|
|
2452
|
+
# reads nothing, exactly as before the snapshot existed.
|
|
2453
|
+
dashboard_settings: DashboardSettings | None = None
|
|
2454
|
+
if warmup_text is not None:
|
|
2455
|
+
settings_cache = _service_get_settings_cache()
|
|
2456
|
+
try:
|
|
2457
|
+
dashboard_settings = await settings_cache.get()
|
|
2458
|
+
except Exception: # noqa: BLE001 - a prewarm must never fail a servable request
|
|
2459
|
+
# The same fallback the request entry point's dashboard-overrides
|
|
2460
|
+
# middleware applies to this row: prefer the last one this replica
|
|
2461
|
+
# loaded. With no row at all the prewarm is skipped rather than run
|
|
2462
|
+
# without a snapshot, which would put the read back under the lock.
|
|
2463
|
+
dashboard_settings = settings_cache.cached_row()
|
|
2464
|
+
logger.warning(
|
|
2465
|
+
"HTTP bridge prewarm settings snapshot unavailable; %s",
|
|
2466
|
+
"using the last loaded dashboard values"
|
|
2467
|
+
if dashboard_settings is not None
|
|
2468
|
+
else "skipping the prewarm",
|
|
2469
|
+
exc_info=True,
|
|
2470
|
+
)
|
|
2471
|
+
if dashboard_settings is None:
|
|
2472
|
+
request_state.prewarm_status = "skipped"
|
|
2473
|
+
_record_http_bridge_prewarm_outcome(outcome="skipped")
|
|
2474
|
+
return
|
|
2475
|
+
async with prewarm_lock:
|
|
2476
|
+
if session.prewarmed:
|
|
2477
|
+
request_state.prewarm_status = "skipped"
|
|
2478
|
+
_record_http_bridge_prewarm_outcome(outcome="skipped")
|
|
2479
|
+
return
|
|
2480
|
+
session.prewarmed = True
|
|
2481
|
+
if warmup_text is None:
|
|
2482
|
+
request_state.prewarm_status = "skipped"
|
|
2483
|
+
_record_http_bridge_prewarm_outcome(outcome="skipped")
|
|
2484
|
+
return
|
|
2485
|
+
|
|
2486
|
+
prewarm_started_at = clock.monotonic()
|
|
2487
|
+
warmup_state = _WebSocketRequestState(
|
|
2488
|
+
request_id=f"http_prewarm_{uuid4().hex}",
|
|
2489
|
+
model=request_state.model,
|
|
2490
|
+
service_tier=request_state.service_tier,
|
|
2491
|
+
reasoning_effort=request_state.reasoning_effort,
|
|
2492
|
+
api_key_reservation=None,
|
|
2493
|
+
started_at=clock.monotonic(),
|
|
2494
|
+
thread_affinity_last_touch_at=clock.monotonic(),
|
|
2495
|
+
api_key_reservation_last_touch_at=clock.monotonic(),
|
|
2496
|
+
requested_service_tier=request_state.requested_service_tier,
|
|
2497
|
+
actual_service_tier=request_state.actual_service_tier,
|
|
2498
|
+
awaiting_response_created=True,
|
|
2499
|
+
event_queue=asyncio.Queue(),
|
|
2500
|
+
transport=_REQUEST_TRANSPORT_HTTP,
|
|
2501
|
+
request_text=warmup_text,
|
|
2502
|
+
skip_request_log=True,
|
|
2503
|
+
)
|
|
2504
|
+
gate_acquired = False
|
|
2505
|
+
request_enqueued = False
|
|
2506
|
+
warmup_send_started = False
|
|
2507
|
+
try:
|
|
2508
|
+
event_queue = warmup_state.event_queue
|
|
2509
|
+
assert event_queue is not None
|
|
2510
|
+
await self._acquire_request_state_response_create_admission(
|
|
2511
|
+
warmup_state,
|
|
2512
|
+
response_create_gate=session.response_create_gate,
|
|
2513
|
+
account_id=session.account.id,
|
|
2514
|
+
surface="http_bridge_prewarm",
|
|
2515
|
+
bridge_session=session,
|
|
2516
|
+
dashboard_settings=dashboard_settings,
|
|
2517
|
+
)
|
|
2518
|
+
gate_acquired = True
|
|
2519
|
+
async with session.lifecycle_lock:
|
|
2520
|
+
current_session = session
|
|
2521
|
+
http_bridge_sessions = getattr(self, "_http_bridge_sessions", None)
|
|
2522
|
+
bridge_lock = getattr(self, "_http_bridge_lock", None)
|
|
2523
|
+
if bridge_lock is not None:
|
|
2524
|
+
async with bridge_lock:
|
|
2525
|
+
if http_bridge_sessions is not None:
|
|
2526
|
+
current_session = http_bridge_sessions.get(session.key)
|
|
2527
|
+
elif http_bridge_sessions is not None:
|
|
2528
|
+
current_session = http_bridge_sessions.get(session.key)
|
|
2529
|
+
session_replaced = current_session is not session
|
|
2530
|
+
if session.closed or session_replaced:
|
|
2531
|
+
_log_http_bridge_event(
|
|
2532
|
+
"submit_on_closed",
|
|
2533
|
+
session.key,
|
|
2534
|
+
account_id=session.account.id,
|
|
2535
|
+
model=session.request_model,
|
|
2536
|
+
detail=(
|
|
2537
|
+
"prewarm_session_retired_after_admission"
|
|
2538
|
+
if session.closed
|
|
2539
|
+
else "prewarm_session_replaced_after_admission"
|
|
2540
|
+
),
|
|
2541
|
+
cache_key_family=session.key.affinity_kind,
|
|
2542
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
2543
|
+
)
|
|
2544
|
+
session.prewarmed = False
|
|
2545
|
+
await self._cleanup_http_bridge_submit_interruption(
|
|
2546
|
+
session,
|
|
2547
|
+
request_state=warmup_state,
|
|
2548
|
+
gate_acquired=gate_acquired,
|
|
2549
|
+
request_enqueued=request_enqueued,
|
|
2550
|
+
counted_in_queue=False,
|
|
2551
|
+
)
|
|
2552
|
+
gate_acquired = False
|
|
2553
|
+
return
|
|
2554
|
+
async with session.pending_lock:
|
|
2555
|
+
session.pending_requests.append(warmup_state)
|
|
2556
|
+
request_enqueued = True
|
|
2557
|
+
warmup_send_started = True
|
|
2558
|
+
await _send_http_bridge_request_text_with_archive_id(
|
|
2559
|
+
session, warmup_state, warmup_text, clock=clock
|
|
2560
|
+
)
|
|
2561
|
+
while True:
|
|
2562
|
+
try:
|
|
2563
|
+
event_block = await scheduler_for(self).wait_for(
|
|
2564
|
+
event_queue.get(),
|
|
2565
|
+
timeout=_prewarm_response_timeout_seconds(),
|
|
2566
|
+
)
|
|
2567
|
+
except asyncio.TimeoutError:
|
|
2568
|
+
request_state.prewarm_latency_ms = int(max(0.0, clock.monotonic() - prewarm_started_at) * 1000)
|
|
2569
|
+
request_state.prewarm_status = "timeout"
|
|
2570
|
+
_record_http_bridge_prewarm_outcome(outcome="timeout")
|
|
2571
|
+
logger.warning(
|
|
2572
|
+
"HTTP bridge prewarm timed out request_id=%s model=%s",
|
|
2573
|
+
request_state.request_id,
|
|
2574
|
+
request_state.model,
|
|
2575
|
+
)
|
|
2576
|
+
session.prewarmed = False
|
|
2577
|
+
try:
|
|
2578
|
+
# The warmup request has already been sent upstream. Close/reconnect the
|
|
2579
|
+
# socket while the warmup state is still attached so any late warmup
|
|
2580
|
+
# response cannot be assigned to the next visible request on this session.
|
|
2581
|
+
await self._reconnect_http_bridge_session(
|
|
2582
|
+
session,
|
|
2583
|
+
request_state=request_state,
|
|
2584
|
+
restart_reader=True,
|
|
2585
|
+
require_same_account=is_http_bridge_account_neutral_replay(
|
|
2586
|
+
kind=session.key.affinity_kind,
|
|
2587
|
+
key=session.key.affinity_key,
|
|
2588
|
+
),
|
|
2589
|
+
dashboard_settings=dashboard_settings,
|
|
2590
|
+
)
|
|
2591
|
+
except Exception:
|
|
2592
|
+
session.closed = True
|
|
2593
|
+
raise
|
|
2594
|
+
finally:
|
|
2595
|
+
async with session.pending_lock:
|
|
2596
|
+
if warmup_state in session.pending_requests:
|
|
2597
|
+
session.pending_requests.remove(warmup_state)
|
|
2598
|
+
self._cancel_request_state_api_key_reservation_heartbeat(warmup_state)
|
|
2599
|
+
if gate_acquired:
|
|
2600
|
+
await _release_websocket_response_create_gate(
|
|
2601
|
+
warmup_state,
|
|
2602
|
+
session.response_create_gate,
|
|
2603
|
+
scheduler=scheduler_for(self),
|
|
2604
|
+
)
|
|
2605
|
+
return
|
|
2606
|
+
if event_block is None:
|
|
2607
|
+
break
|
|
2608
|
+
payload = parse_sse_data_json(event_block)
|
|
2609
|
+
event = parse_sse_event(event_block)
|
|
2610
|
+
event_type = _event_type_from_payload(event, payload)
|
|
2611
|
+
if event_type in {"response.failed", "response.incomplete", "error"}:
|
|
2612
|
+
raise ProxyResponseError(
|
|
2613
|
+
502,
|
|
2614
|
+
openai_error(
|
|
2615
|
+
"upstream_unavailable",
|
|
2616
|
+
"HTTP responses session bridge prewarm failed",
|
|
2617
|
+
),
|
|
2618
|
+
)
|
|
2619
|
+
session.last_used_at = clock.monotonic()
|
|
2620
|
+
request_state.prewarm_latency_ms = int(max(0.0, session.last_used_at - prewarm_started_at) * 1000)
|
|
2621
|
+
request_state.prewarm_status = "success"
|
|
2622
|
+
_record_http_bridge_prewarm_outcome(outcome="success")
|
|
2623
|
+
except ProxyResponseError as exc:
|
|
2624
|
+
error = _parse_openai_error(exc.payload)
|
|
2625
|
+
code = _normalize_error_code(error.code if error else None, error.type if error else None)
|
|
2626
|
+
await self._cleanup_http_bridge_submit_interruption(
|
|
2627
|
+
session,
|
|
2628
|
+
request_state=warmup_state,
|
|
2629
|
+
gate_acquired=gate_acquired,
|
|
2630
|
+
request_enqueued=request_enqueued,
|
|
2631
|
+
counted_in_queue=False,
|
|
2632
|
+
)
|
|
2633
|
+
if is_local_overload_error_code(code):
|
|
2634
|
+
session.prewarmed = False
|
|
2635
|
+
request_state.prewarm_latency_ms = int(max(0.0, clock.monotonic() - prewarm_started_at) * 1000)
|
|
2636
|
+
request_state.prewarm_status = "skipped"
|
|
2637
|
+
_record_http_bridge_prewarm_outcome(outcome="skipped")
|
|
2638
|
+
return
|
|
2639
|
+
session.prewarmed = False
|
|
2640
|
+
request_state.prewarm_latency_ms = int(max(0.0, clock.monotonic() - prewarm_started_at) * 1000)
|
|
2641
|
+
request_state.prewarm_status = "error"
|
|
2642
|
+
_record_http_bridge_prewarm_outcome(outcome="error")
|
|
2643
|
+
raise
|
|
2644
|
+
except BaseException:
|
|
2645
|
+
if warmup_send_started:
|
|
2646
|
+
session.closed = True
|
|
2647
|
+
session.upstream_control.reconnect_requested = True
|
|
2648
|
+
session.upstream_control.retire_after_drain = True
|
|
2649
|
+
session.prewarmed = False
|
|
2650
|
+
request_state.prewarm_latency_ms = int(max(0.0, clock.monotonic() - prewarm_started_at) * 1000)
|
|
2651
|
+
request_state.prewarm_status = "error"
|
|
2652
|
+
_record_http_bridge_prewarm_outcome(outcome="error")
|
|
2653
|
+
cleanup_task = scheduler_for(self).create_task(
|
|
2654
|
+
self._cleanup_http_bridge_submit_interruption(
|
|
2655
|
+
session,
|
|
2656
|
+
request_state=warmup_state,
|
|
2657
|
+
gate_acquired=gate_acquired,
|
|
2658
|
+
request_enqueued=request_enqueued,
|
|
2659
|
+
counted_in_queue=False,
|
|
2660
|
+
)
|
|
2661
|
+
)
|
|
2662
|
+
await _await_task_deferring_cancellation(cleanup_task)
|
|
2663
|
+
if (
|
|
2664
|
+
warmup_send_started
|
|
2665
|
+
and session.upstream_control.retire_after_drain
|
|
2666
|
+
and not session.upstream_close_attempted
|
|
2667
|
+
):
|
|
2668
|
+
retire_task = scheduler_for(self).create_task(
|
|
2669
|
+
self._retire_http_bridge_after_drain_if_ready(session)
|
|
2670
|
+
)
|
|
2671
|
+
await _await_task_deferring_cancellation(retire_task)
|
|
2672
|
+
raise
|
|
2673
|
+
|
|
2674
|
+
async def _cleanup_http_bridge_submit_interruption(
|
|
2675
|
+
self: Any,
|
|
2676
|
+
session: "_HTTPBridgeSession",
|
|
2677
|
+
*,
|
|
2678
|
+
request_state: _WebSocketRequestState,
|
|
2679
|
+
gate_acquired: bool,
|
|
2680
|
+
request_enqueued: bool,
|
|
2681
|
+
counted_in_queue: bool,
|
|
2682
|
+
admission_waiter_registered: bool = False,
|
|
2683
|
+
) -> None:
|
|
2684
|
+
retire_closed_session = False
|
|
2685
|
+
async with session.pending_lock:
|
|
2686
|
+
if request_enqueued and request_state in session.pending_requests:
|
|
2687
|
+
session.pending_requests.remove(request_state)
|
|
2688
|
+
if counted_in_queue:
|
|
2689
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
2690
|
+
if admission_waiter_registered or request_state.admission_waiter_preregistered:
|
|
2691
|
+
session.admission_waiter_count = max(0, session.admission_waiter_count - 1)
|
|
2692
|
+
request_state.admission_waiter_preregistered = False
|
|
2693
|
+
retire_closed_session = session.closed and session.admission_waiter_count == 0
|
|
2694
|
+
if (
|
|
2695
|
+
request_state.recovery_attempt_fingerprint is not None
|
|
2696
|
+
and not request_state.recovery_attempt_claimed
|
|
2697
|
+
and not request_state.recovery_attempt_dispatched
|
|
2698
|
+
and session.durable_session_id is not None
|
|
2699
|
+
and session.durable_owner_epoch is not None
|
|
2700
|
+
):
|
|
2701
|
+
rollback_recovery_attempt = getattr(self._durable_bridge, "rollback_recovery_attempt_before_dispatch", None)
|
|
2702
|
+
if callable(rollback_recovery_attempt):
|
|
2703
|
+
try:
|
|
2704
|
+
await _call_with_supported_optional_kwargs(
|
|
2705
|
+
rollback_recovery_attempt,
|
|
2706
|
+
optional_kwargs={},
|
|
2707
|
+
session_id=session.durable_session_id,
|
|
2708
|
+
api_key_id=session.key.api_key_id,
|
|
2709
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
2710
|
+
owner_epoch=session.durable_owner_epoch,
|
|
2711
|
+
request_fingerprint=request_state.recovery_attempt_fingerprint,
|
|
2712
|
+
)
|
|
2713
|
+
except Exception:
|
|
2714
|
+
logger.warning(
|
|
2715
|
+
"Failed to roll back pre-dispatch HTTP bridge recovery checkpoint request_id=%s",
|
|
2716
|
+
request_state.request_id,
|
|
2717
|
+
exc_info=True,
|
|
2718
|
+
)
|
|
2719
|
+
if (
|
|
2720
|
+
(request_state.operation_created or request_state.operation_rebound)
|
|
2721
|
+
and request_state.operation_registered
|
|
2722
|
+
and request_state.operation_id is not None
|
|
2723
|
+
and not request_state.operation_dispatched
|
|
2724
|
+
and session.durable_session_id is not None
|
|
2725
|
+
and session.durable_owner_epoch is not None
|
|
2726
|
+
):
|
|
2727
|
+
rollback_operation = getattr(self._durable_bridge, "rollback_operation_before_dispatch", None)
|
|
2728
|
+
if callable(rollback_operation):
|
|
2729
|
+
try:
|
|
2730
|
+
rolled_back = await _call_with_supported_optional_kwargs(
|
|
2731
|
+
rollback_operation,
|
|
2732
|
+
optional_kwargs={
|
|
2733
|
+
"restore_rebound": request_state.operation_rebound,
|
|
2734
|
+
"rebound_from_session_id": request_state.operation_rebound_from_session_id,
|
|
2735
|
+
"rebound_from_account_id": request_state.operation_rebound_from_account_id,
|
|
2736
|
+
"rebound_from_model": request_state.operation_rebound_from_model,
|
|
2737
|
+
"rebound_from_parent_response_id": request_state.operation_rebound_from_parent_response_id,
|
|
2738
|
+
},
|
|
2739
|
+
operation_id=request_state.operation_id,
|
|
2740
|
+
session_id=session.durable_session_id,
|
|
2741
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
2742
|
+
owner_epoch=session.durable_owner_epoch,
|
|
2743
|
+
)
|
|
2744
|
+
except Exception:
|
|
2745
|
+
rolled_back = False
|
|
2746
|
+
logger.warning(
|
|
2747
|
+
"Failed to roll back pre-dispatch HTTP bridge operation operation_id=%s",
|
|
2748
|
+
request_state.operation_id,
|
|
2749
|
+
exc_info=True,
|
|
2750
|
+
)
|
|
2751
|
+
if rolled_back:
|
|
2752
|
+
request_state.operation_registered = False
|
|
2753
|
+
request_state.operation_created = False
|
|
2754
|
+
if request_state.operation_rebound:
|
|
2755
|
+
# The failed durable row was restored rather than
|
|
2756
|
+
# deleted. Preserve its identity so a capacity/gate
|
|
2757
|
+
# retry must rebind that exact fence before send.
|
|
2758
|
+
request_state.operation_rebind_required = True
|
|
2759
|
+
request_state.operation_rebound = False
|
|
2760
|
+
else:
|
|
2761
|
+
request_state.operation_id = None
|
|
2762
|
+
request_state.operation_fingerprint = None
|
|
2763
|
+
request_state.operation_parent_response_id = None
|
|
2764
|
+
self._cancel_request_state_api_key_reservation_heartbeat(request_state)
|
|
2765
|
+
if request_state.response_create_gate is not None:
|
|
2766
|
+
if gate_acquired or request_state.response_create_gate_acquired:
|
|
2767
|
+
await _release_websocket_response_create_gate(
|
|
2768
|
+
request_state, session.response_create_gate, scheduler=scheduler_for(self)
|
|
2769
|
+
)
|
|
2770
|
+
else:
|
|
2771
|
+
account_response_create_lease = request_state.account_response_create_lease
|
|
2772
|
+
account_response_create_release = request_state.account_response_create_release
|
|
2773
|
+
request_state.account_response_create_lease = None
|
|
2774
|
+
request_state.account_response_create_release = None
|
|
2775
|
+
if account_response_create_lease is not None and account_response_create_release is not None:
|
|
2776
|
+
await account_response_create_release(account_response_create_lease)
|
|
2777
|
+
if request_state.response_create_admission is not None:
|
|
2778
|
+
request_state.response_create_admission.release()
|
|
2779
|
+
request_state.response_create_admission = None
|
|
2780
|
+
request_state.awaiting_response_created = False
|
|
2781
|
+
request_state.response_create_gate = None
|
|
2782
|
+
request_state.response_create_gate_acquired = False
|
|
2783
|
+
elif gate_acquired:
|
|
2784
|
+
await _release_websocket_response_create_gate(
|
|
2785
|
+
request_state, session.response_create_gate, scheduler=scheduler_for(self)
|
|
2786
|
+
)
|
|
2787
|
+
if retire_closed_session:
|
|
2788
|
+
await self._retire_stale_pending_http_bridge_session(
|
|
2789
|
+
session,
|
|
2790
|
+
detail="last_admission_waiter_cancelled",
|
|
2791
|
+
response_events_seen=max(
|
|
2792
|
+
request_state.response_event_count,
|
|
2793
|
+
int(request_state.response_id is not None or request_state.latency_response_created_ms is not None),
|
|
2794
|
+
),
|
|
2795
|
+
# The session was already closed when the last admission
|
|
2796
|
+
# waiter cancelled; its reader is gone, so post-suspension
|
|
2797
|
+
# liveness signals cannot make it serviceable again.
|
|
2798
|
+
allow_liveness_revive=False,
|
|
2799
|
+
)
|
|
2800
|
+
await self._maybe_release_idle_http_bridge_session_lease(session)
|
|
2801
|
+
|
|
2802
|
+
async def _http_bridge_reacquire_snapshot(
|
|
2803
|
+
self: Any,
|
|
2804
|
+
session: "_HTTPBridgeSession",
|
|
2805
|
+
) -> tuple[int, RoutingTunables | None]:
|
|
2806
|
+
"""Resolve the reacquire's settings inputs WITHOUT holding pending_lock.
|
|
2807
|
+
|
|
2808
|
+
For a keyed session one cached dashboard row yields both the fair-share
|
|
2809
|
+
threshold and the routing tunables the lease is judged against (C2-2
|
|
2810
|
+
routing/overload: the lease TTL). An unkeyed session reads no settings
|
|
2811
|
+
here, as before: its threshold is ``0`` and the balancer falls back to
|
|
2812
|
+
the snapshot of its most recent request. The settings-cache refresh
|
|
2813
|
+
runs a DB query behind a process-global lock; awaiting it while holding
|
|
2814
|
+
a session's ``pending_lock`` let one stalled query wedge every keyed
|
|
2815
|
+
submit process-wide (issue #1971). Callers resolve the snapshot first
|
|
2816
|
+
and pass both values into ``_ensure_http_bridge_session_stream_lease_locked``.
|
|
2817
|
+
"""
|
|
2818
|
+
if session.key.api_key_id is None:
|
|
2819
|
+
return 0, None
|
|
2820
|
+
dashboard_settings = await _service_get_settings_cache().get()
|
|
2821
|
+
return (
|
|
2822
|
+
_api_key_fair_share_threshold_pct_from_settings(dashboard_settings),
|
|
2823
|
+
effective_routing_tunables(dashboard_settings),
|
|
2824
|
+
)
|
|
2825
|
+
|
|
2826
|
+
async def _ensure_http_bridge_session_stream_lease_locked(
|
|
2827
|
+
self: Any,
|
|
2828
|
+
session: "_HTTPBridgeSession",
|
|
2829
|
+
*,
|
|
2830
|
+
request_state: _WebSocketRequestState | None = None,
|
|
2831
|
+
fair_share_threshold_pct: int | None = None,
|
|
2832
|
+
routing_tunables: RoutingTunables | None = None,
|
|
2833
|
+
) -> None:
|
|
2834
|
+
"""Reacquire the account stream lease for a session idled between turns.
|
|
2835
|
+
|
|
2836
|
+
Callers hold ``session.pending_lock`` and MUST pass both
|
|
2837
|
+
``fair_share_threshold_pct`` and ``routing_tunables`` (see
|
|
2838
|
+
``_http_bridge_reacquire_snapshot``) computed before acquiring it:
|
|
2839
|
+
resolving either reads the settings cache, whose refresh runs a DB
|
|
2840
|
+
query behind a process-global lock — one stalled refresh under
|
|
2841
|
+
``pending_lock`` wedged every keyed submit for days (issue #1971).
|
|
2842
|
+
The ``None`` fallback resolves them inline from one cached row and
|
|
2843
|
+
exists for lock-free callers only.
|
|
2844
|
+
|
|
2845
|
+
The lease is released when the
|
|
2846
|
+
session's last in-flight turn detaches, so an idle session does not
|
|
2847
|
+
occupy a per-account stream slot; the next turn must pass normal cap
|
|
2848
|
+
admission again. Denial raises the standard local-cap envelope so the
|
|
2849
|
+
existing recoverable capacity wait applies.
|
|
2850
|
+
|
|
2851
|
+
The lease stays per-session (one upstream stream): a session that
|
|
2852
|
+
already holds a lease admits further queued turns without acquiring
|
|
2853
|
+
another, because those turns multiplex over the session's single
|
|
2854
|
+
upstream WebSocket — unchanged from the pre-existing per-session
|
|
2855
|
+
lease lifecycle.
|
|
2856
|
+
|
|
2857
|
+
Keyed sessions thread their API key through the reacquire so the
|
|
2858
|
+
turn joins the per-key stream accounting and passes the same
|
|
2859
|
+
congestion-gated fair-share admission as initial selection; a
|
|
2860
|
+
fair-share denial raises the standard local-cap envelope with the
|
|
2861
|
+
fair-share code so the recoverable capacity wait applies.
|
|
2862
|
+
"""
|
|
2863
|
+
if session.account_lease is not None or session.closed:
|
|
2864
|
+
return
|
|
2865
|
+
load_balancer = getattr(self, "_load_balancer", None)
|
|
2866
|
+
if load_balancer is None:
|
|
2867
|
+
return
|
|
2868
|
+
api_key_id = session.key.api_key_id
|
|
2869
|
+
if api_key_id is None:
|
|
2870
|
+
fair_share_threshold_pct = 0
|
|
2871
|
+
elif fair_share_threshold_pct is None:
|
|
2872
|
+
# Lock-free callers only (see the docstring): one cached row serves
|
|
2873
|
+
# both inputs.
|
|
2874
|
+
dashboard_settings = await _service_get_settings_cache().get()
|
|
2875
|
+
fair_share_threshold_pct = _api_key_fair_share_threshold_pct_from_settings(dashboard_settings)
|
|
2876
|
+
if routing_tunables is None:
|
|
2877
|
+
routing_tunables = effective_routing_tunables(dashboard_settings)
|
|
2878
|
+
try:
|
|
2879
|
+
lease = await load_balancer.acquire_account_lease(
|
|
2880
|
+
session.account.id,
|
|
2881
|
+
kind="stream",
|
|
2882
|
+
routing_tunables=routing_tunables,
|
|
2883
|
+
# Carry the turn's usage-budget estimate like initial selection
|
|
2884
|
+
# and reconnect do, so capacity-weighted routing pressure still
|
|
2885
|
+
# sees large turns on reused warm sessions.
|
|
2886
|
+
estimated_tokens=_estimated_lease_tokens_from_request_usage_budget(
|
|
2887
|
+
request_state.request_usage_budget if request_state is not None else None
|
|
2888
|
+
),
|
|
2889
|
+
api_key_id=api_key_id,
|
|
2890
|
+
api_key_stream_fair_share_threshold_pct=fair_share_threshold_pct,
|
|
2891
|
+
)
|
|
2892
|
+
except ApiKeyFairShareDenialError as denial:
|
|
2893
|
+
raise ProxyResponseError(
|
|
2894
|
+
429,
|
|
2895
|
+
openai_error(
|
|
2896
|
+
API_KEY_STREAM_FAIR_SHARE_ERROR_CODE,
|
|
2897
|
+
str(denial),
|
|
2898
|
+
error_type="rate_limit_error",
|
|
2899
|
+
),
|
|
2900
|
+
) from None
|
|
2901
|
+
if lease is None:
|
|
2902
|
+
raise ProxyResponseError(
|
|
2903
|
+
429,
|
|
2904
|
+
openai_error(
|
|
2905
|
+
"account_stream_cap",
|
|
2906
|
+
"Account stream capacity is exhausted; wait for active streams to finish.",
|
|
2907
|
+
error_type="rate_limit_error",
|
|
2908
|
+
),
|
|
2909
|
+
)
|
|
2910
|
+
if session.closed:
|
|
2911
|
+
# A close or eviction ran while the acquire await was suspended
|
|
2912
|
+
# (the close path does not take pending_lock before settling the
|
|
2913
|
+
# session's lease). Installing this lease on the closed session
|
|
2914
|
+
# would leak the slot: close already settled, and the idle
|
|
2915
|
+
# release helper skips closed sessions. Return the slot and fail
|
|
2916
|
+
# the turn like any other submit on a closed bridge.
|
|
2917
|
+
async def release_detached_lease() -> None:
|
|
2918
|
+
try:
|
|
2919
|
+
await load_balancer.release_account_lease(lease)
|
|
2920
|
+
except Exception:
|
|
2921
|
+
logger.warning(
|
|
2922
|
+
"Failed to release stream lease acquired for closed HTTP bridge session",
|
|
2923
|
+
exc_info=True,
|
|
2924
|
+
)
|
|
2925
|
+
|
|
2926
|
+
release_task = scheduler_for(self).create_task(release_detached_lease())
|
|
2927
|
+
_, cancellation = await _await_task_deferring_cancellation(release_task)
|
|
2928
|
+
if cancellation is not None:
|
|
2929
|
+
raise cancellation
|
|
2930
|
+
raise ProxyResponseError(
|
|
2931
|
+
502,
|
|
2932
|
+
openai_error("upstream_unavailable", "HTTP responses session bridge is closed"),
|
|
2933
|
+
)
|
|
2934
|
+
session.account_lease = lease
|
|
2935
|
+
|
|
2936
|
+
async def _maybe_release_idle_http_bridge_session_lease(
|
|
2937
|
+
self: Any,
|
|
2938
|
+
session: "_HTTPBridgeSession",
|
|
2939
|
+
) -> None:
|
|
2940
|
+
"""Release the account stream lease once a session has no in-flight work.
|
|
2941
|
+
|
|
2942
|
+
The per-account stream cap exists to bound concurrent upstream
|
|
2943
|
+
streams (one bridge session holds one slot for its single upstream
|
|
2944
|
+
WebSocket); an idle bridge session keeping its upstream WebSocket warm
|
|
2945
|
+
must not occupy a slot for its whole idle TTL. Session close releases
|
|
2946
|
+
via its own path, so a session that closed keeps that settlement.
|
|
2947
|
+
"""
|
|
2948
|
+
load_balancer = getattr(self, "_load_balancer", None)
|
|
2949
|
+
if load_balancer is None:
|
|
2950
|
+
return
|
|
2951
|
+
lease = None
|
|
2952
|
+
async with session.pending_lock:
|
|
2953
|
+
if (
|
|
2954
|
+
not session.closed
|
|
2955
|
+
and session.account_lease is not None
|
|
2956
|
+
and session.queued_request_count == 0
|
|
2957
|
+
and session.admission_waiter_count == 0
|
|
2958
|
+
and not session.pending_requests
|
|
2959
|
+
):
|
|
2960
|
+
lease = session.account_lease
|
|
2961
|
+
session.account_lease = None
|
|
2962
|
+
if lease is None:
|
|
2963
|
+
return
|
|
2964
|
+
|
|
2965
|
+
async def release_idle_lease() -> None:
|
|
2966
|
+
try:
|
|
2967
|
+
await load_balancer.release_account_lease(lease)
|
|
2968
|
+
except Exception:
|
|
2969
|
+
logger.warning("Failed to release idle HTTP bridge account lease", exc_info=True)
|
|
2970
|
+
|
|
2971
|
+
release_task = scheduler_for(self).create_task(release_idle_lease())
|
|
2972
|
+
_, cancellation = await _await_task_deferring_cancellation(release_task)
|
|
2973
|
+
if cancellation is not None:
|
|
2974
|
+
raise cancellation
|
|
2975
|
+
|
|
2976
|
+
async def _detach_http_bridge_request(
|
|
2977
|
+
self: Any,
|
|
2978
|
+
session: "_HTTPBridgeSession",
|
|
2979
|
+
*,
|
|
2980
|
+
request_state: _WebSocketRequestState,
|
|
2981
|
+
) -> bool:
|
|
2982
|
+
detached = False
|
|
2983
|
+
async with session.pending_lock:
|
|
2984
|
+
if request_state in session.pending_requests and not request_state.draining_until_terminal:
|
|
2985
|
+
request_state.draining_until_terminal = True
|
|
2986
|
+
request_state.downstream_visible = False
|
|
2987
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
2988
|
+
session.upstream_control.reconnect_requested = True
|
|
2989
|
+
session.upstream_control.retire_after_drain = True
|
|
2990
|
+
detached = True
|
|
2991
|
+
# Queue revocation and pending ownership use the same lock. A
|
|
2992
|
+
# completed handler that wins first keeps its local queue reference;
|
|
2993
|
+
# a detach that wins first leaves no queue for that handler to claim.
|
|
2994
|
+
request_state.event_queue = None
|
|
2995
|
+
await _release_websocket_response_create_gate(
|
|
2996
|
+
request_state, session.response_create_gate, scheduler=scheduler_for(self)
|
|
2997
|
+
)
|
|
2998
|
+
if not detached:
|
|
2999
|
+
if request_state.operation_replay:
|
|
3000
|
+
# Replay requests are delivered from the durable transcript
|
|
3001
|
+
# without entering pending ownership, so the normal detach
|
|
3002
|
+
# branch cannot settle their API-key reservation.
|
|
3003
|
+
self._cancel_request_state_api_key_reservation_heartbeat(request_state)
|
|
3004
|
+
await self._release_websocket_request_state_reservation(request_state)
|
|
3005
|
+
request_state.api_key_reservation = None
|
|
3006
|
+
request_state.operation_replay = False
|
|
3007
|
+
return False
|
|
3008
|
+
if request_state.terminal_settlement_phase == "abandoned":
|
|
3009
|
+
# Belt-and-braces for issue #1594: terminal bookkeeping
|
|
3010
|
+
# claimed this request out of pending ownership, aborted, and
|
|
3011
|
+
# its shielded abort settlement also failed. Nothing else owns
|
|
3012
|
+
# the reservation any more, so reclaim settlement here instead
|
|
3013
|
+
# of keying solely on pending-deque membership.
|
|
3014
|
+
self._cancel_request_state_api_key_reservation_heartbeat(request_state)
|
|
3015
|
+
await self._release_websocket_request_state_reservation(request_state)
|
|
3016
|
+
request_state.api_key_reservation = None
|
|
3017
|
+
request_state.terminal_settlement_phase = None
|
|
3018
|
+
return False
|
|
3019
|
+
self._cancel_request_state_api_key_reservation_heartbeat(request_state)
|
|
3020
|
+
await self._release_websocket_request_state_reservation(request_state)
|
|
3021
|
+
request_state.api_key_reservation = None
|
|
3022
|
+
await self._retire_http_bridge_after_drain_if_ready(session)
|
|
3023
|
+
return True
|
|
3024
|
+
|
|
3025
|
+
async def _fail_stale_http_bridge_pending_requests(
|
|
3026
|
+
self: Any,
|
|
3027
|
+
session: "_HTTPBridgeSession",
|
|
3028
|
+
request_states: list[_WebSocketRequestState],
|
|
3029
|
+
*,
|
|
3030
|
+
detail: str,
|
|
3031
|
+
retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None,
|
|
3032
|
+
) -> None:
|
|
3033
|
+
if retry_circuit_attempt_selection is None:
|
|
3034
|
+
# Capture the physical sends before waiting for pending ownership.
|
|
3035
|
+
# A concurrent recovery may replace request_state.response_create_attempt
|
|
3036
|
+
# while this task is suspended on pending_lock.
|
|
3037
|
+
retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
|
|
3038
|
+
request_states
|
|
3039
|
+
)
|
|
3040
|
+
stale_requests: deque[_WebSocketRequestState] = deque()
|
|
3041
|
+
response_events_seen = 0
|
|
3042
|
+
async with session.pending_lock:
|
|
3043
|
+
for request_state in request_states:
|
|
3044
|
+
if request_state not in session.pending_requests:
|
|
3045
|
+
continue
|
|
3046
|
+
response_events_seen = max(
|
|
3047
|
+
response_events_seen,
|
|
3048
|
+
request_state.response_event_count,
|
|
3049
|
+
int(
|
|
3050
|
+
request_state.response_id is not None
|
|
3051
|
+
or request_state.latency_response_created_ms is not None
|
|
3052
|
+
or request_state.downstream_visible
|
|
3053
|
+
),
|
|
3054
|
+
)
|
|
3055
|
+
session.pending_requests.remove(request_state)
|
|
3056
|
+
if _http_bridge_request_counts_against_queue(request_state):
|
|
3057
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
3058
|
+
stale_requests.append(request_state)
|
|
3059
|
+
if not stale_requests:
|
|
3060
|
+
return
|
|
3061
|
+
# A stale gate holder that streamed response events without ever
|
|
3062
|
+
# receiving ``response.created`` proves the reattach wedge (#1534)
|
|
3063
|
+
# even when the session itself survives with other active requests.
|
|
3064
|
+
_record_http_bridge_quarantine_wedged_pending(self, session, stale_requests)
|
|
3065
|
+
partial_strike_detail = detail
|
|
3066
|
+
# ``_fail_pending_websocket_requests`` empties the deque it is handed
|
|
3067
|
+
# when it claims the states, so the settle predicate below must read
|
|
3068
|
+
# a frozen copy or a removed safe-replay holder silently vanishes
|
|
3069
|
+
# from the settlement decision.
|
|
3070
|
+
removed_request_states = list(stale_requests)
|
|
3071
|
+
|
|
3072
|
+
async def _finalize_and_settle_partial_cleanup() -> None:
|
|
3073
|
+
# The strike lands before the failure frames are published, so a
|
|
3074
|
+
# client that resends the moment it sees the end-of-stream
|
|
3075
|
+
# sentinel already meets the cooldown and quarantine this strike
|
|
3076
|
+
# armed. The durable abandonment stays after publication per the
|
|
3077
|
+
# ordering contract.
|
|
3078
|
+
consecutive_failures: int | None = None
|
|
3079
|
+
if response_events_seen == 0:
|
|
3080
|
+
consecutive_failures = await self._record_http_bridge_retry_circuit_failure_for_attempt_selection(
|
|
3081
|
+
session,
|
|
3082
|
+
detail=partial_strike_detail,
|
|
3083
|
+
selection=retry_circuit_attempt_selection,
|
|
3084
|
+
)
|
|
3085
|
+
await self._fail_pending_websocket_requests(
|
|
3086
|
+
account=session.account,
|
|
3087
|
+
account_id_value=session.account.id,
|
|
3088
|
+
pending_requests=stale_requests,
|
|
3089
|
+
pending_lock=session.pending_lock,
|
|
3090
|
+
error_code="upstream_request_timeout",
|
|
3091
|
+
error_message="HTTP bridge response-create gate holder timed out",
|
|
3092
|
+
api_key=None,
|
|
3093
|
+
response_create_gate=session.response_create_gate,
|
|
3094
|
+
penalize_account=False,
|
|
3095
|
+
)
|
|
3096
|
+
if response_events_seen == 0:
|
|
3097
|
+
poison_detail = await self._http_bridge_effective_anchor_poison_detail(session, partial_strike_detail)
|
|
3098
|
+
if poison_detail is None:
|
|
3099
|
+
return
|
|
3100
|
+
# The consult returns the exact episode it validated; the
|
|
3101
|
+
# marker below scopes to it.
|
|
3102
|
+
poison_episode, poison_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
|
|
3103
|
+
session,
|
|
3104
|
+
consecutive_failures=consecutive_failures,
|
|
3105
|
+
)
|
|
3106
|
+
if poison_episode is None:
|
|
3107
|
+
return
|
|
3108
|
+
# A surviving safe-replay holder is about to re-dispatch
|
|
3109
|
+
# and claim the circuit generation; settling under it would
|
|
3110
|
+
# strip that fence, so survivors count alongside the removed
|
|
3111
|
+
# holders. Snapshotted here, not before finalization: a
|
|
3112
|
+
# request that joined the still-live session while the
|
|
3113
|
+
# removed holders were being failed must count too.
|
|
3114
|
+
async with session.pending_lock:
|
|
3115
|
+
surviving_request_states = list(session.pending_requests)
|
|
3116
|
+
durable_cleared = await _abandon_durable_http_bridge_continuity(
|
|
3117
|
+
self,
|
|
3118
|
+
session,
|
|
3119
|
+
detail=poison_detail,
|
|
3120
|
+
settle_circuit=_http_bridge_abandonment_may_settle_circuit(
|
|
3121
|
+
[*removed_request_states, *surviving_request_states]
|
|
3122
|
+
),
|
|
3123
|
+
expected_continuity=poison_expected_anchor,
|
|
3124
|
+
authorized_episode=poison_episode,
|
|
3125
|
+
)
|
|
3126
|
+
if durable_cleared:
|
|
3127
|
+
await self._http_bridge_mark_poison_anchor_cleared(session, episode=poison_episode)
|
|
3128
|
+
return
|
|
3129
|
+
if session.durable_session_id is not None:
|
|
3130
|
+
_log_http_bridge_event(
|
|
3131
|
+
"durable_anchor_poison_clear_failed",
|
|
3132
|
+
session.key,
|
|
3133
|
+
account_id=session.account.id,
|
|
3134
|
+
model=session.request_model,
|
|
3135
|
+
pending_count=len(removed_request_states),
|
|
3136
|
+
detail=poison_detail,
|
|
3137
|
+
cache_key_family=session.key.affinity_kind,
|
|
3138
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
3139
|
+
)
|
|
3140
|
+
|
|
3141
|
+
# One owned task covers finalization and settlement together: a
|
|
3142
|
+
# cancellation landing while the holders are being failed would
|
|
3143
|
+
# otherwise re-raise out of finalization before the settlement task
|
|
3144
|
+
# even exists, with the holders already removed from pending and no
|
|
3145
|
+
# lifecycle left to retry the strike, the consult, or the
|
|
3146
|
+
# poisoned-anchor abandonment. Defer cancellation across the whole
|
|
3147
|
+
# sequence like the other funnels, then re-raise it.
|
|
3148
|
+
partial_cleanup_task = scheduler_for(self).create_task(
|
|
3149
|
+
_finalize_and_settle_partial_cleanup(),
|
|
3150
|
+
name=f"http-bridge-partial-cleanup-poison-settlement-{session.durable_session_id}",
|
|
3151
|
+
)
|
|
3152
|
+
_partial_result, partial_cancellation = await _await_task_deferring_cancellation(partial_cleanup_task)
|
|
3153
|
+
if partial_cancellation is not None:
|
|
3154
|
+
raise partial_cancellation
|
|
3155
|
+
|
|
3156
|
+
def _classify_http_bridge_stale_gate_holders(
|
|
3157
|
+
self: Any,
|
|
3158
|
+
pending_states: list[_WebSocketRequestState],
|
|
3159
|
+
*,
|
|
3160
|
+
now: float,
|
|
3161
|
+
threshold_seconds: float,
|
|
3162
|
+
session_closed: bool,
|
|
3163
|
+
) -> tuple[list[_WebSocketRequestState], bool]:
|
|
3164
|
+
stale_states = [
|
|
3165
|
+
state
|
|
3166
|
+
for state in pending_states
|
|
3167
|
+
if not state.draining_until_terminal
|
|
3168
|
+
and self._http_bridge_pending_state_is_stale(
|
|
3169
|
+
state,
|
|
3170
|
+
now=now,
|
|
3171
|
+
threshold_seconds=threshold_seconds,
|
|
3172
|
+
session_closed=session_closed,
|
|
3173
|
+
)
|
|
3174
|
+
]
|
|
3175
|
+
active_states = [
|
|
3176
|
+
# A draining request still owns terminal response continuity. It
|
|
3177
|
+
# must keep the session alive while stale holders are cleaned up.
|
|
3178
|
+
state
|
|
3179
|
+
for state in pending_states
|
|
3180
|
+
if state not in stale_states
|
|
3181
|
+
]
|
|
3182
|
+
if stale_states and active_states:
|
|
3183
|
+
return stale_states, False
|
|
3184
|
+
return [], bool(stale_states)
|
|
3185
|
+
|
|
3186
|
+
async def _snapshot_http_bridge_stale_gate_state(
|
|
3187
|
+
self: Any,
|
|
3188
|
+
session: "_HTTPBridgeSession",
|
|
3189
|
+
*,
|
|
3190
|
+
now: float,
|
|
3191
|
+
) -> _HTTPBridgeStaleGateSnapshot:
|
|
3192
|
+
threshold_seconds = float(_http_bridge_helpers.HTTP_BRIDGE_STUCK_GATE_RETIRE_AFTER_SECONDS)
|
|
3193
|
+
async with session.pending_lock:
|
|
3194
|
+
pending_states = list(session.pending_requests)
|
|
3195
|
+
stale_request_states, should_retire = self._classify_http_bridge_stale_gate_holders(
|
|
3196
|
+
pending_states,
|
|
3197
|
+
now=now,
|
|
3198
|
+
threshold_seconds=threshold_seconds,
|
|
3199
|
+
session_closed=session.closed,
|
|
3200
|
+
)
|
|
3201
|
+
retry_circuit_request_states = (
|
|
3202
|
+
stale_request_states if stale_request_states else (pending_states if should_retire else ())
|
|
3203
|
+
)
|
|
3204
|
+
return _HTTPBridgeStaleGateSnapshot(
|
|
3205
|
+
pending_states=pending_states,
|
|
3206
|
+
queued_count=session.queued_request_count,
|
|
3207
|
+
threshold_seconds=threshold_seconds,
|
|
3208
|
+
stale_request_states=stale_request_states,
|
|
3209
|
+
should_retire=should_retire,
|
|
3210
|
+
retry_circuit_attempt_selection=(
|
|
3211
|
+
_http_bridge_retry_circuit_attempt_selection_for_pending_requests(retry_circuit_request_states)
|
|
3212
|
+
),
|
|
3213
|
+
)
|
|
3214
|
+
|
|
3215
|
+
async def _release_http_bridge_admission_preregistration(
|
|
3216
|
+
self: Any,
|
|
3217
|
+
session: "_HTTPBridgeSession",
|
|
3218
|
+
*,
|
|
3219
|
+
request_state: _WebSocketRequestState,
|
|
3220
|
+
) -> bool:
|
|
3221
|
+
"""Hand back a submit-entry admission registration the dispatch never took over.
|
|
3222
|
+
|
|
3223
|
+
Mirrors the interruption cleanup's waiter release for a turn that
|
|
3224
|
+
never reached dispatch: the last waiter to leave a session that was
|
|
3225
|
+
closed under it retires the session when nothing else owns it (a
|
|
3226
|
+
fence-closed session with pending work and a live reader is settled
|
|
3227
|
+
by that reader, not here), and a session left with no in-flight work
|
|
3228
|
+
gives up an idle lease.
|
|
3229
|
+
"""
|
|
3230
|
+
if not request_state.admission_waiter_preregistered:
|
|
3231
|
+
return False
|
|
3232
|
+
async with session.pending_lock:
|
|
3233
|
+
session.admission_waiter_count = max(0, session.admission_waiter_count - 1)
|
|
3234
|
+
request_state.admission_waiter_preregistered = False
|
|
3235
|
+
retire_closed_session = session.closed and _http_bridge_session_unowned_locked(session)
|
|
3236
|
+
if retire_closed_session:
|
|
3237
|
+
await self._retire_stale_pending_http_bridge_session(
|
|
3238
|
+
session,
|
|
3239
|
+
detail="last_admission_waiter_cancelled",
|
|
3240
|
+
response_events_seen=max(
|
|
3241
|
+
request_state.response_event_count,
|
|
3242
|
+
int(request_state.response_id is not None or request_state.latency_response_created_ms is not None),
|
|
3243
|
+
),
|
|
3244
|
+
allow_liveness_revive=False,
|
|
3245
|
+
)
|
|
3246
|
+
await self._maybe_release_idle_http_bridge_session_lease(session)
|
|
3247
|
+
return True
|
|
3248
|
+
|
|
3249
|
+
async def _retire_http_bridge_after_drain_if_ready(
|
|
3250
|
+
self: Any,
|
|
3251
|
+
session: "_HTTPBridgeSession",
|
|
3252
|
+
*,
|
|
3253
|
+
lock_wait_timeout_seconds: float | None = None,
|
|
3254
|
+
) -> bool:
|
|
3255
|
+
"""Retire a drained session flagged ``retire_after_drain``.
|
|
3256
|
+
|
|
3257
|
+
``lock_wait_timeout_seconds`` bounds the ``pending_lock`` wait for
|
|
3258
|
+
callers on a shared hot path (the per-request fail-safe sweep over
|
|
3259
|
+
detached sessions). When the bound elapses the check is skipped for
|
|
3260
|
+
this pass and ``False`` is returned; the session stays tracked and is
|
|
3261
|
+
reconsidered by the next sweep or by its own drain/close paths.
|
|
3262
|
+
``None`` keeps the unbounded wait for owners of the session lifecycle.
|
|
3263
|
+
"""
|
|
3264
|
+
if not (session.upstream_control.reconnect_requested and session.upstream_control.retire_after_drain):
|
|
3265
|
+
return False
|
|
3266
|
+
|
|
3267
|
+
def decide_locked() -> bool:
|
|
3268
|
+
unowned = _http_bridge_session_unowned_locked(session)
|
|
3269
|
+
if unowned:
|
|
3270
|
+
session.pending_requests.clear()
|
|
3271
|
+
session.upstream_close_attempted = True
|
|
3272
|
+
return unowned
|
|
3273
|
+
|
|
3274
|
+
if lock_wait_timeout_seconds is None:
|
|
3275
|
+
async with session.pending_lock:
|
|
3276
|
+
should_reconnect = decide_locked()
|
|
3277
|
+
else:
|
|
3278
|
+
try:
|
|
3279
|
+
await scheduler_for(self).wait_for(session.pending_lock.acquire(), timeout=lock_wait_timeout_seconds)
|
|
3280
|
+
except TimeoutError:
|
|
3281
|
+
logger.warning(
|
|
3282
|
+
"Skipping detached HTTP bridge retire check: pending_lock busy for %.1fs session_key=%s",
|
|
3283
|
+
lock_wait_timeout_seconds,
|
|
3284
|
+
_hash_identifier(session.key.affinity_key),
|
|
3285
|
+
)
|
|
3286
|
+
return False
|
|
3287
|
+
try:
|
|
3288
|
+
should_reconnect = decide_locked()
|
|
3289
|
+
finally:
|
|
3290
|
+
session.pending_lock.release()
|
|
3291
|
+
if not should_reconnect:
|
|
3292
|
+
return False
|
|
3293
|
+
|
|
3294
|
+
await self._close_http_bridge_session_bounded(session, reason="retire_after_drain")
|
|
3295
|
+
return True
|
|
3296
|
+
|
|
3297
|
+
async def _retire_idle_http_bridge_session_on_cooldown_suppression(
|
|
3298
|
+
self: Any,
|
|
3299
|
+
session: "_HTTPBridgeSession",
|
|
3300
|
+
*,
|
|
3301
|
+
owned_unanchored_handoff: bool,
|
|
3302
|
+
) -> bool:
|
|
3303
|
+
"""Retire a cooldown-suppressed session unless another turn owns it.
|
|
3304
|
+
|
|
3305
|
+
The bridge opens or selects the session before the hard-key retry
|
|
3306
|
+
circuit decides admission, so a suppressed request can leave a socket
|
|
3307
|
+
that never carried ``response.create`` in the reuse pool, where the
|
|
3308
|
+
next half-open probe picks it up (issue #1943). Flagging the session
|
|
3309
|
+
for retirement and running the bounded drain path closes that socket
|
|
3310
|
+
— but only when no other turn owns the session. A pending or queued
|
|
3311
|
+
turn, a registered admission waiter (the admitted probe between its
|
|
3312
|
+
gate and dispatch), or a foreign unanchored handoff is a live owner
|
|
3313
|
+
whose own lifecycle governs retirement; flagging the session under it
|
|
3314
|
+
would trip the pre-dispatch retiring fence for the one submission the
|
|
3315
|
+
cooldown permits. The caller's own unanchored handoff does not count:
|
|
3316
|
+
its submit finalizer releases it and retires the flagged session.
|
|
3317
|
+
"""
|
|
3318
|
+
async with session.pending_lock:
|
|
3319
|
+
idle = _http_bridge_session_unowned_locked(
|
|
3320
|
+
session,
|
|
3321
|
+
ignore_unanchored_reservation=owned_unanchored_handoff,
|
|
3322
|
+
)
|
|
3323
|
+
if idle:
|
|
3324
|
+
session.upstream_control.reconnect_requested = True
|
|
3325
|
+
session.upstream_control.retire_after_drain = True
|
|
3326
|
+
if not idle:
|
|
3327
|
+
return False
|
|
3328
|
+
await self._retire_http_bridge_after_drain_if_ready(session)
|
|
3329
|
+
return True
|
|
3330
|
+
|
|
3331
|
+
async def _http_bridge_claim_miss_shows_remote_probe(
|
|
3332
|
+
self: Any,
|
|
3333
|
+
circuit_key: Any,
|
|
3334
|
+
captured_generation: Any,
|
|
3335
|
+
) -> bool:
|
|
3336
|
+
"""Whether a claim CAS miss actually means a probe holds the lease.
|
|
3337
|
+
|
|
3338
|
+
A confirmed CAS loss only says the row moved: a sibling completion
|
|
3339
|
+
resets the row and changes its version without touching the
|
|
3340
|
+
admission generation. Only an advanced admission generation proves a
|
|
3341
|
+
probe won; a reset, purge, strike, or lookup outage must report the
|
|
3342
|
+
timer the fresh row actually carries instead of a 600-second wait no
|
|
3343
|
+
probe owns.
|
|
3344
|
+
"""
|
|
3345
|
+
captured_admission_generation = captured_generation[0] if captured_generation else 0
|
|
3346
|
+
captured_lineage_epoch = captured_generation[1] if captured_generation else 0.0
|
|
3347
|
+
try:
|
|
3348
|
+
moved_row = await self._durable_bridge.lookup_retry_circuit(
|
|
3349
|
+
session_key_kind=circuit_key.affinity_kind,
|
|
3350
|
+
session_key_value=circuit_key.affinity_key,
|
|
3351
|
+
api_key_id=circuit_key.api_key_id,
|
|
3352
|
+
)
|
|
3353
|
+
except Exception:
|
|
3354
|
+
return False
|
|
3355
|
+
return bool(
|
|
3356
|
+
moved_row is not None
|
|
3357
|
+
# A lease exists only for an active at-threshold lineage: a
|
|
3358
|
+
# probe-then-reset sequence keeps the advanced generation on a
|
|
3359
|
+
# zero-count row with no timer left, and a purge-and-recreate
|
|
3360
|
+
# restarts generations below the captured one.
|
|
3361
|
+
and moved_row.consecutive_failures >= _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD
|
|
3362
|
+
and getattr(moved_row, "admission_generation", 0) > captured_admission_generation
|
|
3363
|
+
# A reset preserves the admission generation while starting a
|
|
3364
|
+
# new lineage under a new version; only an advance within the
|
|
3365
|
+
# captured lineage is a probe in THIS episode. A same-lineage
|
|
3366
|
+
# strike moves the version too and under-reports here, which
|
|
3367
|
+
# falls back to the timer the fresh row actually carries.
|
|
3368
|
+
and getattr(moved_row, "updated_at_epoch", None) == captured_lineage_epoch
|
|
3369
|
+
)
|
|
3370
|
+
|
|
3371
|
+
async def _retire_stale_pending_http_bridge_session(
|
|
3372
|
+
self: Any,
|
|
3373
|
+
session: "_HTTPBridgeSession",
|
|
3374
|
+
*,
|
|
3375
|
+
detail: str,
|
|
3376
|
+
retry_circuit_detail: str | None = None,
|
|
3377
|
+
response_events_seen: int | None = None,
|
|
3378
|
+
retired_request_count: int | None = None,
|
|
3379
|
+
retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None,
|
|
3380
|
+
retired_request_states: list[_WebSocketRequestState] | None = None,
|
|
3381
|
+
allow_liveness_revive: bool = True,
|
|
3382
|
+
) -> None:
|
|
3383
|
+
async with session.pending_lock:
|
|
3384
|
+
if retired_request_states is None:
|
|
3385
|
+
retired_request_states = list(session.pending_requests)
|
|
3386
|
+
baseline_completed_response_id = session.last_completed_response_id
|
|
3387
|
+
# Baseline the upstream event generation at entry, alongside the
|
|
3388
|
+
# completed-response baseline, so a prelude-only upstream event
|
|
3389
|
+
# arriving during the retry-circuit strike await (which bumps the
|
|
3390
|
+
# generation without touching per-request response counters) is
|
|
3391
|
+
# still detected as post-suspension liveness.
|
|
3392
|
+
baseline_event_generation = session.last_upstream_event_generation
|
|
3393
|
+
# Snapshot the retirement fences at entry. Fence owners (durable
|
|
3394
|
+
# stale-owner rejection, never-graft replacement rejection,
|
|
3395
|
+
# previous-response-owner unavailability, alias-registration
|
|
3396
|
+
# failure) set ``closed``/``reconnect_requested``/
|
|
3397
|
+
# ``retire_after_drain`` without detaching the session or claiming
|
|
3398
|
+
# the upstream close, so identity and close-claim guards alone
|
|
3399
|
+
# cannot see them. A fence raised after this snapshot must survive
|
|
3400
|
+
# the liveness revive below; erasing it would turn the fence
|
|
3401
|
+
# owner's drain-retirement into a permanent no-op and let a
|
|
3402
|
+
# condemned socket stay registered and reusable.
|
|
3403
|
+
entry_closed = session.closed
|
|
3404
|
+
entry_reconnect_requested = session.upstream_control.reconnect_requested
|
|
3405
|
+
entry_retire_after_drain = session.upstream_control.retire_after_drain
|
|
3406
|
+
if retired_request_count is None:
|
|
3407
|
+
retired_request_count = sum(
|
|
3408
|
+
1
|
|
3409
|
+
for request_state in retired_request_states
|
|
3410
|
+
if _http_bridge_request_counts_against_queue(request_state)
|
|
3411
|
+
)
|
|
3412
|
+
if response_events_seen is None:
|
|
3413
|
+
# Direct retirement must derive event evidence from the same
|
|
3414
|
+
# locked ownership snapshot as the pending count. Otherwise an
|
|
3415
|
+
# eventful stale-gate owner looks eventless merely because its
|
|
3416
|
+
# caller omitted this optional handoff, creating a false
|
|
3417
|
+
# circuit strike. Explicit values remain authoritative for
|
|
3418
|
+
# reader-failure callers whose pending deque was already
|
|
3419
|
+
# drained before entering this shared boundary.
|
|
3420
|
+
response_events_seen = max(
|
|
3421
|
+
(
|
|
3422
|
+
max(
|
|
3423
|
+
request_state.response_event_count,
|
|
3424
|
+
int(
|
|
3425
|
+
request_state.response_id is not None
|
|
3426
|
+
or request_state.latency_response_created_ms is not None
|
|
3427
|
+
or request_state.downstream_visible
|
|
3428
|
+
),
|
|
3429
|
+
)
|
|
3430
|
+
for request_state in retired_request_states
|
|
3431
|
+
),
|
|
3432
|
+
default=0,
|
|
3433
|
+
)
|
|
3434
|
+
if retry_circuit_attempt_selection is None:
|
|
3435
|
+
retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
|
|
3436
|
+
retired_request_states
|
|
3437
|
+
)
|
|
3438
|
+
# Direct retirement (for example the all-stale stuck-gate path, where
|
|
3439
|
+
# the wedged reattach is the only pending request) cancels the reader
|
|
3440
|
+
# and fails the pendings without passing the partial-cleanup hook or
|
|
3441
|
+
# the reader-failure funnel, so evaluate the wedge shape (#1534) here
|
|
3442
|
+
# too; recording is idempotent for callers that already quarantined.
|
|
3443
|
+
_record_http_bridge_quarantine_wedged_pending(self, session, retired_request_states)
|
|
3444
|
+
# This circuit measures failed request lifecycles, not upstream socket
|
|
3445
|
+
# churn. ``response_events_seen == 0`` is also true when an idle reader
|
|
3446
|
+
# closes with an empty pending deque. Charging that idle close creates a
|
|
3447
|
+
# phantom first strike, so one later response-create timeout opens the
|
|
3448
|
+
# nominally "repeated" 60-second cooldown and interrupts the client.
|
|
3449
|
+
# Keep the ownership proof at this shared retirement boundary unless a
|
|
3450
|
+
# caller already claimed and drained the deque. The reader-failure
|
|
3451
|
+
# funnel must pass its pre-drain count because terminal notification
|
|
3452
|
+
# deliberately empties ``pending_requests`` before retirement. Without
|
|
3453
|
+
# that handoff, genuine pre-response failures disappear from circuit
|
|
3454
|
+
# accounting while idle closes and request failures look identical.
|
|
3455
|
+
# Mirror the terminal and grouped settlement paths: a retired request
|
|
3456
|
+
# that still holds a verified safe replay is about to be re-dispatched
|
|
3457
|
+
# and will claim the circuit generation, so it must not strike the
|
|
3458
|
+
# circuit or trigger the poison clear. Only the safe-replay half of
|
|
3459
|
+
# the settlement predicate applies here — an unanchored eventless
|
|
3460
|
+
# owner has no replay to protect and keeps striking. A pre-drain
|
|
3461
|
+
# handoff supplies the drained states through
|
|
3462
|
+
# ``retired_request_states``, so a safe-replay holder that was
|
|
3463
|
+
# already claimed by finalization still counts; only a handoff with
|
|
3464
|
+
# genuinely no request states keeps striking.
|
|
3465
|
+
retired_states_present = [state for state in retired_request_states if state is not None]
|
|
3466
|
+
retirement_strike_eligible = not retired_states_present or any(
|
|
3467
|
+
not _http_bridge_request_state_holds_safe_replay(state) for state in retired_states_present
|
|
3468
|
+
)
|
|
3469
|
+
if retired_request_count > 0 and response_events_seen == 0 and retirement_strike_eligible:
|
|
3470
|
+
consecutive_failures = await self._record_http_bridge_retry_circuit_failure_for_attempt_selection(
|
|
3471
|
+
session,
|
|
3472
|
+
detail=retry_circuit_detail or detail,
|
|
3473
|
+
selection=retry_circuit_attempt_selection,
|
|
3474
|
+
)
|
|
3475
|
+
poison_detail = await self._http_bridge_effective_anchor_poison_detail(
|
|
3476
|
+
session, retry_circuit_detail or detail
|
|
3477
|
+
)
|
|
3478
|
+
poison_episode = None
|
|
3479
|
+
poison_expected_anchor: object = _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
3480
|
+
if poison_detail is not None:
|
|
3481
|
+
# The consult returns the exact episode it validated; the
|
|
3482
|
+
# marker below scopes to it, so a settle landing between the
|
|
3483
|
+
# consult and the rebind cannot be marked onto a replacement.
|
|
3484
|
+
poison_episode, poison_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
|
|
3485
|
+
session,
|
|
3486
|
+
consecutive_failures=consecutive_failures,
|
|
3487
|
+
)
|
|
3488
|
+
if poison_detail is not None and poison_episode is not None:
|
|
3489
|
+
# Consecutive eventless failures on one bridge key are
|
|
3490
|
+
# same-anchor failures (the anchor only advances on a
|
|
3491
|
+
# completed response, which resets the circuit). Clear the
|
|
3492
|
+
# poisoned durable anchor while this session still owns the
|
|
3493
|
+
# lease so the next attempt is not re-anchored into the same
|
|
3494
|
+
# failure. Without this, only the admission-waiter reader
|
|
3495
|
+
# path could ever poison an anchor, and an anchored session
|
|
3496
|
+
# failing without waiters cooled down forever (issue #1830).
|
|
3497
|
+
durable_cleared = await _abandon_durable_http_bridge_continuity(
|
|
3498
|
+
self,
|
|
3499
|
+
session,
|
|
3500
|
+
detail=poison_detail,
|
|
3501
|
+
settle_circuit=_http_bridge_abandonment_may_settle_circuit(retired_request_states),
|
|
3502
|
+
expected_continuity=poison_expected_anchor,
|
|
3503
|
+
authorized_episode=poison_episode,
|
|
3504
|
+
)
|
|
3505
|
+
if durable_cleared:
|
|
3506
|
+
await self._http_bridge_mark_poison_anchor_cleared(session, episode=poison_episode)
|
|
3507
|
+
if not durable_cleared and session.durable_session_id is not None:
|
|
3508
|
+
# Keep failed waiterless clears visible in the same
|
|
3509
|
+
# poison-clear telemetry the admission-waiter path emits;
|
|
3510
|
+
# the next threshold failure re-attempts the clear.
|
|
3511
|
+
_log_http_bridge_event(
|
|
3512
|
+
"durable_anchor_poison_clear_failed",
|
|
3513
|
+
session.key,
|
|
3514
|
+
account_id=session.account.id,
|
|
3515
|
+
model=session.request_model,
|
|
3516
|
+
pending_count=retired_request_count,
|
|
3517
|
+
detail=poison_detail,
|
|
3518
|
+
cache_key_family=session.key.affinity_kind,
|
|
3519
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
3520
|
+
)
|
|
3521
|
+
|
|
3522
|
+
async with session.pending_lock:
|
|
3523
|
+
current_response_events_seen = max(
|
|
3524
|
+
(getattr(request_state, "response_event_count", 0) for request_state in session.pending_requests),
|
|
3525
|
+
default=0,
|
|
3526
|
+
)
|
|
3527
|
+
current_response_created = any(
|
|
3528
|
+
request_state.response_id is not None or request_state.latency_response_created_ms is not None
|
|
3529
|
+
for request_state in session.pending_requests
|
|
3530
|
+
)
|
|
3531
|
+
completed_response_id = session.last_completed_response_id
|
|
3532
|
+
# The snapshot above avoids awaiting pending_lock while the global
|
|
3533
|
+
# registry lock is held, preserving bounded cleanup for other sessions.
|
|
3534
|
+
caller_response_events_seen = response_events_seen or 0
|
|
3535
|
+
observed_new_completed_response = (
|
|
3536
|
+
completed_response_id is not None and completed_response_id != baseline_completed_response_id
|
|
3537
|
+
)
|
|
3538
|
+
became_healthy_during_suspend = current_response_events_seen > caller_response_events_seen or (
|
|
3539
|
+
caller_response_events_seen == 0 and (current_response_created or observed_new_completed_response)
|
|
3540
|
+
)
|
|
3541
|
+
should_close = False
|
|
3542
|
+
async with self._http_bridge_lock:
|
|
3543
|
+
if session.last_upstream_event_generation != baseline_event_generation:
|
|
3544
|
+
became_healthy_during_suspend = True
|
|
3545
|
+
# A fence raised while this coroutine was suspended condemns the
|
|
3546
|
+
# session even if it also showed liveness: the fence owner leaves
|
|
3547
|
+
# the session registered and never claims the upstream close, so
|
|
3548
|
+
# neither guard below can see it. Only fences newly raised since
|
|
3549
|
+
# the entry snapshot block the revive; the entry-time retirement
|
|
3550
|
+
# state belongs to this retirement and is cleared on revive.
|
|
3551
|
+
fence_raised_during_suspend = (
|
|
3552
|
+
(session.closed and not entry_closed)
|
|
3553
|
+
or (session.upstream_control.reconnect_requested and not entry_reconnect_requested)
|
|
3554
|
+
or (session.upstream_control.retire_after_drain and not entry_retire_after_drain)
|
|
3555
|
+
)
|
|
3556
|
+
# Bounded close may return while resource finalization is still
|
|
3557
|
+
# running. Detachment transfers ownership instead of freeing the
|
|
3558
|
+
# capacity slot at canonical removal, and leaves a failed close
|
|
3559
|
+
# discoverable by shutdown/account invalidation for a later retry.
|
|
3560
|
+
if session.upstream_close_attempted:
|
|
3561
|
+
# A close owner already claimed retirement; converge with its
|
|
3562
|
+
# bookkeeping and fall through so this path still emits the
|
|
3563
|
+
# shared retirement telemetry (``should_close`` resolves to
|
|
3564
|
+
# False below, so the close is not re-attempted).
|
|
3565
|
+
session.closed = True
|
|
3566
|
+
self._detach_http_bridge_session_locked(session.key, expected_session=session)
|
|
3567
|
+
elif (
|
|
3568
|
+
allow_liveness_revive
|
|
3569
|
+
and became_healthy_during_suspend
|
|
3570
|
+
and not fence_raised_during_suspend
|
|
3571
|
+
and self._http_bridge_sessions.get(session.key) is session
|
|
3572
|
+
):
|
|
3573
|
+
# The pending snapshot is only advisory; a terminal response
|
|
3574
|
+
# may already have left the deque. A close owner cannot have
|
|
3575
|
+
# claimed retirement in between: the branch above falls
|
|
3576
|
+
# through under this same lock hold without awaiting.
|
|
3577
|
+
#
|
|
3578
|
+
# Revive only while this session is still the registered owner
|
|
3579
|
+
# of its key. The acquisition loop can detach a
|
|
3580
|
+
# ``retiring_with_visible_requests`` generation with
|
|
3581
|
+
# ``mark_closed=False`` while this coroutine is suspended;
|
|
3582
|
+
# clearing the retirement flags on that detached generation
|
|
3583
|
+
# would make its drain-retirement a permanent no-op and leak
|
|
3584
|
+
# the socket, durable/account leases, and capacity slot. A
|
|
3585
|
+
# detached generation falls through to the bounded close
|
|
3586
|
+
# below instead (the detach call is a no-op for it), as does
|
|
3587
|
+
# a session fenced during the suspension: the bounded close
|
|
3588
|
+
# conservatively satisfies the fence owner's intent.
|
|
3589
|
+
#
|
|
3590
|
+
# ``allow_liveness_revive=False`` callers (the reader-failure
|
|
3591
|
+
# funnel and deferred retirement of an already-closed session)
|
|
3592
|
+
# never reach this branch. Their pending turns were already
|
|
3593
|
+
# terminally failed and their reader is condemned, so there is
|
|
3594
|
+
# no turn left for the revive to save — and the
|
|
3595
|
+
# completed-response signal above can be advanced by durable
|
|
3596
|
+
# anchor rehydration, which copies registry state without any
|
|
3597
|
+
# upstream evidence. Reviving there would leave a condemned,
|
|
3598
|
+
# readerless session registered and reusable.
|
|
3599
|
+
session.closed = False
|
|
3600
|
+
session.upstream_control.reconnect_requested = False
|
|
3601
|
+
session.upstream_control.retire_after_drain = False
|
|
3602
|
+
return
|
|
3603
|
+
else:
|
|
3604
|
+
self._detach_http_bridge_session_locked(session.key, expected_session=session)
|
|
3605
|
+
async with session.pending_lock:
|
|
3606
|
+
should_close = not session.upstream_close_attempted
|
|
3607
|
+
if should_close:
|
|
3608
|
+
session.upstream_close_attempted = True
|
|
3609
|
+
if should_close:
|
|
3610
|
+
await self._close_http_bridge_session_bounded(session, reason="retire_stale_pending")
|
|
3611
|
+
_log_http_bridge_event(
|
|
3612
|
+
"retire_stale_pending",
|
|
3613
|
+
session.key,
|
|
3614
|
+
account_id=session.account.id,
|
|
3615
|
+
model=session.request_model,
|
|
3616
|
+
pending_count=await self._http_bridge_pending_count(session),
|
|
3617
|
+
detail=detail,
|
|
3618
|
+
cache_key_family=session.key.affinity_kind,
|
|
3619
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
3620
|
+
)
|
|
3621
|
+
|
|
3622
|
+
async def _retry_http_bridge_request_on_fresh_upstream(
|
|
3623
|
+
self: Any,
|
|
3624
|
+
session: "_HTTPBridgeSession",
|
|
3625
|
+
*,
|
|
3626
|
+
request_state: _WebSocketRequestState,
|
|
3627
|
+
text_data: str,
|
|
3628
|
+
send_request: bool = True,
|
|
3629
|
+
require_same_account: bool = False,
|
|
3630
|
+
) -> bool:
|
|
3631
|
+
clock = clock_for(self)
|
|
3632
|
+
require_same_account = require_same_account or is_http_bridge_account_neutral_replay(
|
|
3633
|
+
kind=session.key.affinity_kind,
|
|
3634
|
+
key=session.key.affinity_key,
|
|
3635
|
+
)
|
|
3636
|
+
retry_text_data = text_data
|
|
3637
|
+
using_fresh_replay = False
|
|
3638
|
+
if request_state.previous_response_id is not None and send_request:
|
|
3639
|
+
# After an ambiguous websocket send failure we cannot prove whether
|
|
3640
|
+
# upstream already accepted the continuation. Re-sending the same
|
|
3641
|
+
# previous_response_id request can fork continuity with duplicate
|
|
3642
|
+
# child responses, so only reconnect-without-resend is allowed.
|
|
3643
|
+
# The single exception is a proof-gated, trim-safe full-resend
|
|
3644
|
+
# payload: dropping the anchor and replaying the original
|
|
3645
|
+
# unanchored request is equivalent to the client's own retry.
|
|
3646
|
+
# The proof is independent of where the anchor came from; a
|
|
3647
|
+
# client-provided full resend is as safe as a durable injection.
|
|
3648
|
+
# Session-level follow-ups do not opt in because their context may
|
|
3649
|
+
# depend on the anchor.
|
|
3650
|
+
if not request_state.fresh_upstream_request_text or not request_state.fresh_upstream_request_is_retry_safe:
|
|
3651
|
+
return False
|
|
3652
|
+
retry_text_data = request_state.fresh_upstream_request_text
|
|
3653
|
+
using_fresh_replay = True
|
|
3654
|
+
if request_state.replay_count >= 1:
|
|
3655
|
+
return False
|
|
3656
|
+
if request_state.response_event_count > 0:
|
|
3657
|
+
return False
|
|
3658
|
+
request_state.replay_count += 1
|
|
3659
|
+
_log_http_bridge_event(
|
|
3660
|
+
"retry_fresh_upstream",
|
|
3661
|
+
session.key,
|
|
3662
|
+
account_id=session.account.id,
|
|
3663
|
+
model=session.request_model,
|
|
3664
|
+
pending_count=1,
|
|
3665
|
+
cache_key_family=session.key.affinity_kind,
|
|
3666
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
3667
|
+
)
|
|
3668
|
+
try:
|
|
3669
|
+
await self._reconnect_http_bridge_session(
|
|
3670
|
+
session,
|
|
3671
|
+
request_state=request_state,
|
|
3672
|
+
restart_reader=True,
|
|
3673
|
+
require_same_account=require_same_account,
|
|
3674
|
+
require_preferred_account=request_state.file_required_preferred_account,
|
|
3675
|
+
)
|
|
3676
|
+
if send_request:
|
|
3677
|
+
retry_text_data = self._http_bridge_text_with_account_installation_id(
|
|
3678
|
+
session,
|
|
3679
|
+
request_state,
|
|
3680
|
+
retry_text_data,
|
|
3681
|
+
)
|
|
3682
|
+
if using_fresh_replay:
|
|
3683
|
+
_bind_http_bridge_proxy_injected_anchor(
|
|
3684
|
+
self,
|
|
3685
|
+
request_state,
|
|
3686
|
+
response_id=None,
|
|
3687
|
+
proxy_injected=False,
|
|
3688
|
+
)
|
|
3689
|
+
request_state.request_text = retry_text_data
|
|
3690
|
+
await _send_http_bridge_request_text_with_archive_id(
|
|
3691
|
+
session, request_state, retry_text_data, clock=clock
|
|
3692
|
+
)
|
|
3693
|
+
_clear_websocket_request_error_overrides(request_state)
|
|
3694
|
+
session.last_used_at = clock.monotonic()
|
|
3695
|
+
return True
|
|
3696
|
+
except UpstreamWebSocketTransportError:
|
|
3697
|
+
# The new socket may have accepted response.create. Let the reader
|
|
3698
|
+
# owner retire the whole session with the typed, non-replayable
|
|
3699
|
+
# failure instead of falling back to the earlier close reason.
|
|
3700
|
+
raise
|
|
3701
|
+
except ProxyResponseError as exc:
|
|
3702
|
+
if _http_bridge_is_previous_response_owner_unavailable(exc):
|
|
3703
|
+
raise
|
|
3704
|
+
logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True)
|
|
3705
|
+
return False
|
|
3706
|
+
except Exception:
|
|
3707
|
+
logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True)
|
|
3708
|
+
return False
|
|
3709
|
+
|
|
3710
|
+
async def _retry_http_bridge_precreated_request(
|
|
3711
|
+
self: Any,
|
|
3712
|
+
session: "_HTTPBridgeSession",
|
|
3713
|
+
*,
|
|
3714
|
+
request_state: _WebSocketRequestState | None = None,
|
|
3715
|
+
restart_reader: bool = False,
|
|
3716
|
+
) -> bool:
|
|
3717
|
+
# The admitted body's admission gate can claim the half-open probe;
|
|
3718
|
+
# any exit that never advances a send attempt past its baseline must
|
|
3719
|
+
# hand that probe back, or a failed reconnect leaves the lease
|
|
3720
|
+
# suppressing traffic for up to the full window with no probe in
|
|
3721
|
+
# flight — the same handback contract the submit finalizer applies.
|
|
3722
|
+
admission_claimed_leases: list[float] = []
|
|
3723
|
+
retry_send_baselines: list[tuple[_WebSocketRequestState, int]] = []
|
|
3724
|
+
try:
|
|
3725
|
+
return await self._retry_http_bridge_precreated_request_admitted(
|
|
3726
|
+
session,
|
|
3727
|
+
request_state=request_state,
|
|
3728
|
+
restart_reader=restart_reader,
|
|
3729
|
+
admission_claimed_leases=admission_claimed_leases,
|
|
3730
|
+
retry_send_baselines=retry_send_baselines,
|
|
3731
|
+
)
|
|
3732
|
+
finally:
|
|
3733
|
+
retry_claimed_half_open_until = admission_claimed_leases[-1] if admission_claimed_leases else 0.0
|
|
3734
|
+
if retry_claimed_half_open_until > 0.0:
|
|
3735
|
+
send_attempt_advanced = bool(retry_send_baselines) and (
|
|
3736
|
+
retry_send_baselines[-1][0].response_create_attempt_count > retry_send_baselines[-1][1]
|
|
3737
|
+
)
|
|
3738
|
+
if not send_attempt_advanced:
|
|
3739
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
3740
|
+
unused_probe_state = self._http_bridge_retry_circuits.get(session.key)
|
|
3741
|
+
if (
|
|
3742
|
+
unused_probe_state is not None
|
|
3743
|
+
and unused_probe_state.half_open_until == retry_claimed_half_open_until
|
|
3744
|
+
):
|
|
3745
|
+
unused_probe_state.half_open_until = 0.0
|
|
3746
|
+
unused_probe_state.cooldown_until = clock_for(self).monotonic() - 1.0
|
|
3747
|
+
|
|
3748
|
+
async def _retry_http_bridge_precreated_request_admitted(
|
|
3749
|
+
self: Any,
|
|
3750
|
+
session: "_HTTPBridgeSession",
|
|
3751
|
+
*,
|
|
3752
|
+
request_state: _WebSocketRequestState | None = None,
|
|
3753
|
+
restart_reader: bool = False,
|
|
3754
|
+
admission_claimed_leases: list[float] | None = None,
|
|
3755
|
+
retry_send_baselines: list[tuple[_WebSocketRequestState, int]] | None = None,
|
|
3756
|
+
) -> bool:
|
|
3757
|
+
clock = clock_for(self)
|
|
3758
|
+
clean_close_retry_max_count = self._http_bridge_clean_close_retry_max_count()
|
|
3759
|
+
account_neutral_recovery = is_http_bridge_account_neutral_replay(
|
|
3760
|
+
kind=session.key.affinity_kind,
|
|
3761
|
+
key=session.key.affinity_key,
|
|
3762
|
+
)
|
|
3763
|
+
|
|
3764
|
+
def request_is_retryable(request_state: _WebSocketRequestState) -> bool:
|
|
3765
|
+
transport_only_unanchored_replay = bool(
|
|
3766
|
+
request_state.precreated_replay_reason is None
|
|
3767
|
+
and (
|
|
3768
|
+
request_state.verified_stale_anchor_replay
|
|
3769
|
+
or (
|
|
3770
|
+
request_state.previous_response_id is not None
|
|
3771
|
+
and not request_state.proxy_injected_previous_response_id
|
|
3772
|
+
and request_state.fresh_upstream_request_is_retry_safe
|
|
3773
|
+
and request_state.fresh_upstream_request_text
|
|
3774
|
+
)
|
|
3775
|
+
)
|
|
3776
|
+
)
|
|
3777
|
+
if transport_only_unanchored_replay:
|
|
3778
|
+
return False
|
|
3779
|
+
if _websocket_request_can_replay_before_visible_output(request_state):
|
|
3780
|
+
return True
|
|
3781
|
+
if (
|
|
3782
|
+
clean_close_retry_max_count <= 0
|
|
3783
|
+
or request_state.verified_stale_anchor_replay
|
|
3784
|
+
or request_state.replay_count != 1
|
|
3785
|
+
or request_state.response_event_count != 0
|
|
3786
|
+
or request_state.clean_close_replay_count >= clean_close_retry_max_count
|
|
3787
|
+
or _classify_upstream_close(
|
|
3788
|
+
session.last_upstream_close_code,
|
|
3789
|
+
response_events_seen=request_state.response_event_count,
|
|
3790
|
+
)
|
|
3791
|
+
!= "clean"
|
|
3792
|
+
):
|
|
3793
|
+
return False
|
|
3794
|
+
return _websocket_request_can_replay_before_visible_output(
|
|
3795
|
+
request_state,
|
|
3796
|
+
allow_clean_close_retry=True,
|
|
3797
|
+
)
|
|
3798
|
+
|
|
3799
|
+
fresh_hard_request_account_switch_candidate = False
|
|
3800
|
+
proof_gated_continuity_replay_candidate = False
|
|
3801
|
+
if session.key.strength == "hard":
|
|
3802
|
+
async with session.pending_lock:
|
|
3803
|
+
retryable_candidates = [
|
|
3804
|
+
request_state
|
|
3805
|
+
for request_state in session.pending_requests
|
|
3806
|
+
if not request_state.draining_until_terminal and request_is_retryable(request_state)
|
|
3807
|
+
]
|
|
3808
|
+
if len(retryable_candidates) == 1:
|
|
3809
|
+
candidate = retryable_candidates[0]
|
|
3810
|
+
fresh_hard_request_account_switch_candidate = (
|
|
3811
|
+
candidate.previous_response_id is None
|
|
3812
|
+
and not candidate.hard_continuity_anchor
|
|
3813
|
+
and not candidate.proxy_injected_previous_response_id
|
|
3814
|
+
and not candidate.file_required_preferred_account
|
|
3815
|
+
and candidate.response_event_count == 0
|
|
3816
|
+
and candidate.replay_count == 0
|
|
3817
|
+
)
|
|
3818
|
+
proof_gated_continuity_replay_candidate = (
|
|
3819
|
+
candidate.previous_response_id is not None
|
|
3820
|
+
and candidate.fresh_upstream_request_is_retry_safe
|
|
3821
|
+
and bool(candidate.fresh_upstream_request_text)
|
|
3822
|
+
and candidate.response_event_count == 0
|
|
3823
|
+
and candidate.replay_count == 0
|
|
3824
|
+
)
|
|
3825
|
+
if not await self._http_bridge_precreated_retry_allowed(
|
|
3826
|
+
session,
|
|
3827
|
+
allow_fresh_hard_account_switch=fresh_hard_request_account_switch_candidate,
|
|
3828
|
+
allow_proof_gated_continuity_replay=proof_gated_continuity_replay_candidate,
|
|
3829
|
+
claimed_lease_out=admission_claimed_leases,
|
|
3830
|
+
):
|
|
3831
|
+
return False
|
|
3832
|
+
|
|
3833
|
+
account_neutral_recovery = is_http_bridge_account_neutral_replay(
|
|
3834
|
+
kind=session.key.affinity_kind,
|
|
3835
|
+
key=session.key.affinity_key,
|
|
3836
|
+
)
|
|
3837
|
+
hard_owner_bound = _http_bridge_key_strength(session.key) == "hard"
|
|
3838
|
+
async with session.pending_lock:
|
|
3839
|
+
if request_state is not None:
|
|
3840
|
+
if (
|
|
3841
|
+
request_state not in session.pending_requests
|
|
3842
|
+
or any(pending_request is not request_state for pending_request in session.pending_requests)
|
|
3843
|
+
or request_state.draining_until_terminal
|
|
3844
|
+
or not _http_bridge_request_counts_against_queue(request_state)
|
|
3845
|
+
or not request_is_retryable(request_state)
|
|
3846
|
+
):
|
|
3847
|
+
return False
|
|
3848
|
+
else:
|
|
3849
|
+
retryable_requests = [
|
|
3850
|
+
request_state
|
|
3851
|
+
for request_state in session.pending_requests
|
|
3852
|
+
if not request_state.draining_until_terminal and request_is_retryable(request_state)
|
|
3853
|
+
]
|
|
3854
|
+
if len(retryable_requests) != 1:
|
|
3855
|
+
return False
|
|
3856
|
+
request_state = retryable_requests[0]
|
|
3857
|
+
# An accepted lifecycle is replayed only when it is the sole
|
|
3858
|
+
# request on the socket (the terminal path's other-pending
|
|
3859
|
+
# guard, ``_websocket_accepted_replay_candidate``). Reconnecting
|
|
3860
|
+
# it would strand every sibling still bound to the dead
|
|
3861
|
+
# upstream, so both fail closed with ``stream_incomplete`` as
|
|
3862
|
+
# they did before accepted replays existed.
|
|
3863
|
+
if (
|
|
3864
|
+
request_state.response_id is not None
|
|
3865
|
+
and not request_state.awaiting_response_created
|
|
3866
|
+
and any(pending_request is not request_state for pending_request in session.pending_requests)
|
|
3867
|
+
):
|
|
3868
|
+
return False
|
|
3869
|
+
# A request that already saw response.created released the session
|
|
3870
|
+
# create gate. Its replay must hold the gate again before it can
|
|
3871
|
+
# own the pre-created identity, and never waits for a contended one.
|
|
3872
|
+
if not await _claim_websocket_replay_create_gate(request_state, session.response_create_gate):
|
|
3873
|
+
return False
|
|
3874
|
+
if retry_send_baselines is not None:
|
|
3875
|
+
# The send-attempt baseline: a retried request already carries
|
|
3876
|
+
# prior attempts, so the release keys on advancement past
|
|
3877
|
+
# this point, not on a zero count.
|
|
3878
|
+
retry_send_baselines.append((request_state, request_state.response_create_attempt_count))
|
|
3879
|
+
model_fallback_replay = request_state.precreated_replay_reason == _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
|
|
3880
|
+
if request_state.previous_response_id is not None and not (
|
|
3881
|
+
request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text
|
|
3882
|
+
):
|
|
3883
|
+
# Once a continuation is pending upstream, reconnecting without
|
|
3884
|
+
# replay cannot complete the current request, while replaying it
|
|
3885
|
+
# is unsafe without upstream idempotency guarantees. Proxy-
|
|
3886
|
+
# injected anchors and proof-gated client full resends are
|
|
3887
|
+
# equivalent to the client's own retry once the anchor is
|
|
3888
|
+
# stripped. The latter remains pinned to the current owner.
|
|
3889
|
+
return False
|
|
3890
|
+
close_classification = _classify_upstream_close(
|
|
3891
|
+
session.last_upstream_close_code,
|
|
3892
|
+
response_events_seen=request_state.response_event_count,
|
|
3893
|
+
)
|
|
3894
|
+
close_generation = session.last_upstream_close_generation
|
|
3895
|
+
hard_session_affinity = session.key.strength == "hard"
|
|
3896
|
+
fresh_hard_request_account_switch_allowed = (
|
|
3897
|
+
hard_session_affinity
|
|
3898
|
+
and request_state.previous_response_id is None
|
|
3899
|
+
and not request_state.hard_continuity_anchor
|
|
3900
|
+
and not request_state.proxy_injected_previous_response_id
|
|
3901
|
+
and not request_state.file_required_preferred_account
|
|
3902
|
+
)
|
|
3903
|
+
clean_close_hard_continuation = (
|
|
3904
|
+
close_classification == "clean"
|
|
3905
|
+
and hard_session_affinity
|
|
3906
|
+
and request_state.previous_response_id is not None
|
|
3907
|
+
)
|
|
3908
|
+
clean_close_hard_continuity_anchor = (
|
|
3909
|
+
close_classification == "clean" and hard_session_affinity and request_state.hard_continuity_anchor
|
|
3910
|
+
)
|
|
3911
|
+
clean_close_retry_for_current_close = (
|
|
3912
|
+
close_classification == "clean"
|
|
3913
|
+
and request_state.clean_close_retry_close_generation != close_generation
|
|
3914
|
+
and not request_state.clean_close_retry_in_progress
|
|
3915
|
+
)
|
|
3916
|
+
additional_clean_close_retry = (
|
|
3917
|
+
clean_close_retry_for_current_close
|
|
3918
|
+
and request_state.replay_count == 1
|
|
3919
|
+
and request_state.response_event_count == 0
|
|
3920
|
+
and request_state.clean_close_replay_count < clean_close_retry_max_count
|
|
3921
|
+
)
|
|
3922
|
+
if request_state.replay_count >= 1 and not additional_clean_close_retry:
|
|
3923
|
+
return False
|
|
3924
|
+
account_bound_replay = False
|
|
3925
|
+
if request_state.previous_response_id is not None:
|
|
3926
|
+
require_preferred_reconnect = False
|
|
3927
|
+
if account_neutral_recovery:
|
|
3928
|
+
request_state.preferred_account_id = session.account.id
|
|
3929
|
+
switch_text = None
|
|
3930
|
+
else:
|
|
3931
|
+
switch_text = _prepare_websocket_request_state_for_account_switch(request_state)
|
|
3932
|
+
if switch_text is None:
|
|
3933
|
+
# The retained full body may be retry-safe for continuity
|
|
3934
|
+
# while still naming an account-scoped uploaded file. In
|
|
3935
|
+
# that case retry on the same owner-bound anchor instead of
|
|
3936
|
+
# letting visible-output replay strip the anchor and migrate.
|
|
3937
|
+
fresh_retry_safe = request_state.fresh_upstream_request_is_retry_safe
|
|
3938
|
+
request_state.fresh_upstream_request_is_retry_safe = False
|
|
3939
|
+
try:
|
|
3940
|
+
request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state)
|
|
3941
|
+
finally:
|
|
3942
|
+
request_state.fresh_upstream_request_is_retry_safe = fresh_retry_safe
|
|
3943
|
+
if request_text is None:
|
|
3944
|
+
return False
|
|
3945
|
+
require_preferred_reconnect = request_state.preferred_account_id is not None
|
|
3946
|
+
else:
|
|
3947
|
+
request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state)
|
|
3948
|
+
if request_text is None:
|
|
3949
|
+
return False
|
|
3950
|
+
if not hard_owner_bound:
|
|
3951
|
+
request_state.excluded_account_ids.add(session.account.id)
|
|
3952
|
+
else:
|
|
3953
|
+
# Account-scoped uploaded files cannot be replayed on a
|
|
3954
|
+
# different owner. Keep the preferred account mandatory for
|
|
3955
|
+
# both silent recovery and clean-close recovery.
|
|
3956
|
+
candidate_text = (
|
|
3957
|
+
request_state.fresh_upstream_request_text
|
|
3958
|
+
if request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text
|
|
3959
|
+
else request_state.request_text
|
|
3960
|
+
)
|
|
3961
|
+
# The send boundary decorates durable operations with
|
|
3962
|
+
# codex_lb_operation_id after selection. Keep that operation
|
|
3963
|
+
# identity on its owner unless a dedicated rebind path has
|
|
3964
|
+
# already replaced the operation ID.
|
|
3965
|
+
candidate_portable = request_state.operation_id is None and (
|
|
3966
|
+
_websocket_request_text_is_account_neutral_fresh_replay(candidate_text)
|
|
3967
|
+
)
|
|
3968
|
+
request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state)
|
|
3969
|
+
if request_text is None or request_text != candidate_text:
|
|
3970
|
+
return False
|
|
3971
|
+
account_bound_replay = not candidate_portable
|
|
3972
|
+
require_preferred_reconnect = (
|
|
3973
|
+
account_neutral_recovery or account_bound_replay or request_state.file_required_preferred_account
|
|
3974
|
+
)
|
|
3975
|
+
if account_neutral_recovery or account_bound_replay:
|
|
3976
|
+
request_state.preferred_account_id = session.account.id
|
|
3977
|
+
elif not request_state.file_required_preferred_account:
|
|
3978
|
+
if hard_owner_bound and not model_fallback_replay and not fresh_hard_request_account_switch_allowed:
|
|
3979
|
+
request_state.preferred_account_id = session.account.id
|
|
3980
|
+
else:
|
|
3981
|
+
request_state.preferred_account_id = None
|
|
3982
|
+
# An accepted replay whose session affinity may resolve
|
|
3983
|
+
# a hard sticky owner reconnects unexcluded: the owner
|
|
3984
|
+
# is the only account selection can return, so the
|
|
3985
|
+
# exclusion would spin on ``hard_affinity_saturated``
|
|
3986
|
+
# until the bridge request budget ran out.
|
|
3987
|
+
if model_fallback_replay or _http_bridge_accepted_replay_may_exclude_account(
|
|
3988
|
+
request_state, session
|
|
3989
|
+
):
|
|
3990
|
+
request_state.excluded_account_ids.add(session.account.id)
|
|
3991
|
+
if session.account.id in request_state.excluded_account_ids:
|
|
3992
|
+
session.upstream_turn_state = None
|
|
3993
|
+
session.downstream_turn_state = None
|
|
3994
|
+
session.headers = {
|
|
3995
|
+
key: value for key, value in session.headers.items() if key.lower() != "x-codex-turn-state"
|
|
3996
|
+
}
|
|
3997
|
+
if close_classification == "clean":
|
|
3998
|
+
if not clean_close_retry_for_current_close:
|
|
3999
|
+
return False
|
|
4000
|
+
request_state.clean_close_retry_in_progress = True
|
|
4001
|
+
request_state.clean_close_retry_result = None
|
|
4002
|
+
request_state.clean_close_retry_close_generation = close_generation
|
|
4003
|
+
if additional_clean_close_retry:
|
|
4004
|
+
request_state.clean_close_replay_count += 1
|
|
4005
|
+
retry_jitter_seconds = (
|
|
4006
|
+
self._http_bridge_clean_close_retry_jitter_seconds() if additional_clean_close_retry else 0.0
|
|
4007
|
+
)
|
|
4008
|
+
retry_event = "retry_precreated_clean_close" if additional_clean_close_retry else "retry_precreated"
|
|
4009
|
+
_log_http_bridge_event(
|
|
4010
|
+
retry_event,
|
|
4011
|
+
session.key,
|
|
4012
|
+
account_id=session.account.id,
|
|
4013
|
+
model=session.request_model,
|
|
4014
|
+
detail="accepted_lifecycle_replay" if request_state.replay_downstream_response_id is not None else None,
|
|
4015
|
+
pending_count=1,
|
|
4016
|
+
cache_key_family=session.key.affinity_kind,
|
|
4017
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
4018
|
+
)
|
|
4019
|
+
reconnect_reader_kwargs = {"restart_reader": True} if restart_reader else {}
|
|
4020
|
+
try:
|
|
4021
|
+
if retry_jitter_seconds > 0:
|
|
4022
|
+
logger.info(
|
|
4023
|
+
"HTTP bridge clean-close retry jitter request_id=%s sleep_seconds=%.3f",
|
|
4024
|
+
request_state.request_id,
|
|
4025
|
+
retry_jitter_seconds,
|
|
4026
|
+
)
|
|
4027
|
+
await scheduler_for(self).sleep(retry_jitter_seconds)
|
|
4028
|
+
request_deadline = request_state.bridge_request_deadline
|
|
4029
|
+
if request_deadline is None:
|
|
4030
|
+
request_deadline = request_state.started_at + _http_bridge_request_budget_seconds(
|
|
4031
|
+
_service_get_settings()
|
|
4032
|
+
)
|
|
4033
|
+
now_monotonic = clock.monotonic()
|
|
4034
|
+
async with session.pending_lock:
|
|
4035
|
+
request_still_owned = (
|
|
4036
|
+
request_state in session.pending_requests and not request_state.draining_until_terminal
|
|
4037
|
+
)
|
|
4038
|
+
if not request_still_owned or now_monotonic >= request_deadline:
|
|
4039
|
+
logger.info(
|
|
4040
|
+
"HTTP bridge clean-close retry abandoned after jitter request_id=%s "
|
|
4041
|
+
"still_owned=%s deadline_expired=%s",
|
|
4042
|
+
request_state.request_id,
|
|
4043
|
+
request_still_owned,
|
|
4044
|
+
now_monotonic >= request_deadline,
|
|
4045
|
+
)
|
|
4046
|
+
request_state.clean_close_retry_result = False
|
|
4047
|
+
return False
|
|
4048
|
+
# A fresh hard-session replay may select a replacement account.
|
|
4049
|
+
# The admission lease is account-scoped, so release the old
|
|
4050
|
+
# account's lease before reconnecting; the post-reconnect path
|
|
4051
|
+
# below acquires a lease for the account actually selected.
|
|
4052
|
+
if fresh_hard_request_account_switch_allowed:
|
|
4053
|
+
await self._release_request_state_account_response_create_lease(request_state)
|
|
4054
|
+
if hard_owner_bound and not model_fallback_replay and not fresh_hard_request_account_switch_allowed:
|
|
4055
|
+
await self._reconnect_http_bridge_session(
|
|
4056
|
+
session,
|
|
4057
|
+
request_state=request_state,
|
|
4058
|
+
require_same_account=True,
|
|
4059
|
+
**reconnect_reader_kwargs,
|
|
4060
|
+
)
|
|
4061
|
+
elif require_preferred_reconnect:
|
|
4062
|
+
await self._reconnect_http_bridge_session(
|
|
4063
|
+
session,
|
|
4064
|
+
request_state=request_state,
|
|
4065
|
+
require_same_account=account_neutral_recovery or account_bound_replay,
|
|
4066
|
+
require_preferred_account=True,
|
|
4067
|
+
**reconnect_reader_kwargs,
|
|
4068
|
+
)
|
|
4069
|
+
elif clean_close_hard_continuation or clean_close_hard_continuity_anchor:
|
|
4070
|
+
await self._reconnect_http_bridge_session(
|
|
4071
|
+
session,
|
|
4072
|
+
request_state=request_state,
|
|
4073
|
+
# Continuity anchors (previous_response_id and turn-state)
|
|
4074
|
+
# are account-bound. Do not migrate them while recovering a
|
|
4075
|
+
# clean handoff close.
|
|
4076
|
+
require_same_account=True,
|
|
4077
|
+
**reconnect_reader_kwargs,
|
|
4078
|
+
)
|
|
4079
|
+
else:
|
|
4080
|
+
await self._reconnect_http_bridge_session(
|
|
4081
|
+
session,
|
|
4082
|
+
request_state=request_state,
|
|
4083
|
+
**reconnect_reader_kwargs,
|
|
4084
|
+
)
|
|
4085
|
+
if request_state.account_response_create_lease is None:
|
|
4086
|
+
current_settings = await _service_get_settings_cache().get()
|
|
4087
|
+
request_state.account_response_create_lease = (
|
|
4088
|
+
await self._acquire_account_response_create_lease_or_overload(
|
|
4089
|
+
account_id=session.account.id,
|
|
4090
|
+
request_id=request_state.request_log_id or request_state.request_id,
|
|
4091
|
+
surface="http_bridge",
|
|
4092
|
+
concurrency_caps=effective_account_concurrency_caps(current_settings),
|
|
4093
|
+
)
|
|
4094
|
+
)
|
|
4095
|
+
request_state.account_response_create_release = self._load_balancer.release_account_lease
|
|
4096
|
+
enforce_capacity_retry_deadline = request_state.response_create_admission_reacquire_required
|
|
4097
|
+
if enforce_capacity_retry_deadline:
|
|
4098
|
+
retry_deadline = request_state.bridge_request_deadline
|
|
4099
|
+
if retry_deadline is None:
|
|
4100
|
+
retry_deadline = request_state.started_at + _http_bridge_request_budget_seconds(
|
|
4101
|
+
_service_get_settings()
|
|
4102
|
+
)
|
|
4103
|
+
remaining_retry_budget_seconds = retry_deadline - clock.monotonic()
|
|
4104
|
+
if remaining_retry_budget_seconds <= 0:
|
|
4105
|
+
request_state.response_create_admission_reacquire_required = False
|
|
4106
|
+
await self._release_request_state_account_response_create_lease(request_state)
|
|
4107
|
+
return False
|
|
4108
|
+
try:
|
|
4109
|
+
request_state.response_create_admission = await scheduler_for(self).wait_for(
|
|
4110
|
+
self._get_work_admission().acquire_response_create(),
|
|
4111
|
+
timeout=remaining_retry_budget_seconds,
|
|
4112
|
+
)
|
|
4113
|
+
except TimeoutError:
|
|
4114
|
+
request_state.response_create_admission_reacquire_required = False
|
|
4115
|
+
await self._release_request_state_account_response_create_lease(request_state)
|
|
4116
|
+
return False
|
|
4117
|
+
request_state.response_create_admission_reacquire_required = False
|
|
4118
|
+
if clock.monotonic() >= retry_deadline:
|
|
4119
|
+
request_state.response_create_admission.release()
|
|
4120
|
+
request_state.response_create_admission = None
|
|
4121
|
+
await self._release_request_state_account_response_create_lease(request_state)
|
|
4122
|
+
return False
|
|
4123
|
+
async with session.pending_lock:
|
|
4124
|
+
retry_consumer_attached = (
|
|
4125
|
+
request_state in session.pending_requests
|
|
4126
|
+
and request_state.event_queue is not None
|
|
4127
|
+
and not request_state.draining_until_terminal
|
|
4128
|
+
)
|
|
4129
|
+
if not retry_consumer_attached:
|
|
4130
|
+
request_state.response_create_admission.release()
|
|
4131
|
+
request_state.response_create_admission = None
|
|
4132
|
+
await self._release_request_state_account_response_create_lease(request_state)
|
|
4133
|
+
return False
|
|
4134
|
+
request_text = self._http_bridge_text_with_account_installation_id(session, request_state, request_text)
|
|
4135
|
+
await _send_http_bridge_request_text_with_archive_id(session, request_state, request_text, clock=clock)
|
|
4136
|
+
session.last_used_at = clock.monotonic()
|
|
4137
|
+
request_state.clean_close_retry_result = True
|
|
4138
|
+
return True
|
|
4139
|
+
except asyncio.CancelledError:
|
|
4140
|
+
request_state.clean_close_retry_result = False
|
|
4141
|
+
raise
|
|
4142
|
+
except UpstreamWebSocketTransportError:
|
|
4143
|
+
request_state.clean_close_retry_result = False
|
|
4144
|
+
raise
|
|
4145
|
+
except Exception as exc:
|
|
4146
|
+
request_state.clean_close_retry_result = False
|
|
4147
|
+
(
|
|
4148
|
+
request_state.error_http_status_override,
|
|
4149
|
+
request_state.error_code_override,
|
|
4150
|
+
request_state.error_message_override,
|
|
4151
|
+
request_state.error_type_override,
|
|
4152
|
+
request_state.error_param_override,
|
|
4153
|
+
) = _http_bridge_precreated_retry_failure_error(exc)
|
|
4154
|
+
if isinstance(exc, ProxyResponseError):
|
|
4155
|
+
logger.info(
|
|
4156
|
+
"HTTP bridge pre-created retry failed with terminal proxy error code=%s message=%s",
|
|
4157
|
+
request_state.error_code_override,
|
|
4158
|
+
request_state.error_message_override,
|
|
4159
|
+
)
|
|
4160
|
+
else:
|
|
4161
|
+
logger.warning("HTTP bridge pre-created retry failed", exc_info=True)
|
|
4162
|
+
return False
|
|
4163
|
+
finally:
|
|
4164
|
+
request_state.clean_close_retry_in_progress = False
|
|
4165
|
+
|
|
4166
|
+
async def _retry_http_bridge_precreated_auth_request(
|
|
4167
|
+
self: Any,
|
|
4168
|
+
session: "_HTTPBridgeSession",
|
|
4169
|
+
request_state: _WebSocketRequestState,
|
|
4170
|
+
*,
|
|
4171
|
+
error_message: str | None,
|
|
4172
|
+
) -> Literal["not_replayable", "retried", "failed"]:
|
|
4173
|
+
clock = clock_for(self)
|
|
4174
|
+
permanent_failure_code = _websocket_auth_failure_permanent_code(error_message)
|
|
4175
|
+
bound_to_current_account = request_state.replay_required_account_id == session.account.id
|
|
4176
|
+
if bound_to_current_account and (
|
|
4177
|
+
_websocket_auth_failure_requires_reauth(error_message)
|
|
4178
|
+
or request_state.auth_replay_counts_by_account.get(session.account.id, 0) > 0
|
|
4179
|
+
):
|
|
4180
|
+
failure_code = permanent_failure_code or _WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE
|
|
4181
|
+
await self._load_balancer.mark_permanent_failure(session.account, failure_code)
|
|
4182
|
+
setattr(request_state, "account_health_error_handled", True)
|
|
4183
|
+
request_state.force_refresh_account_id = None
|
|
4184
|
+
request_state.preferred_account_id = None
|
|
4185
|
+
request_state.excluded_account_ids.add(session.account.id)
|
|
4186
|
+
return "not_replayable"
|
|
4187
|
+
request_text = _prepare_websocket_request_state_for_auth_replay(
|
|
4188
|
+
request_state,
|
|
4189
|
+
current_account_id=session.account.id,
|
|
4190
|
+
)
|
|
4191
|
+
if request_text is None:
|
|
4192
|
+
await self._load_balancer.mark_permanent_failure(session.account, permanent_failure_code)
|
|
4193
|
+
setattr(request_state, "account_health_error_handled", True)
|
|
4194
|
+
request_state.force_refresh_account_id = None
|
|
4195
|
+
request_state.preferred_account_id = None
|
|
4196
|
+
request_state.excluded_account_ids.add(session.account.id)
|
|
4197
|
+
return "not_replayable"
|
|
4198
|
+
|
|
4199
|
+
if _websocket_auth_failure_requires_reauth(error_message):
|
|
4200
|
+
failure_code = permanent_failure_code
|
|
4201
|
+
elif request_state.auth_replay_counts_by_account.get(session.account.id, 0) == 0:
|
|
4202
|
+
failure_code = None
|
|
4203
|
+
request_state.auth_replay_counts_by_account[session.account.id] = 1
|
|
4204
|
+
request_state.force_refresh_account_id = session.account.id
|
|
4205
|
+
request_state.preferred_account_id = session.account.id
|
|
4206
|
+
else:
|
|
4207
|
+
failure_code = _WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE
|
|
4208
|
+
|
|
4209
|
+
if failure_code is not None:
|
|
4210
|
+
await self._load_balancer.mark_permanent_failure(session.account, failure_code)
|
|
4211
|
+
request_state.force_refresh_account_id = None
|
|
4212
|
+
request_state.preferred_account_id = None
|
|
4213
|
+
request_state.excluded_account_ids.add(session.account.id)
|
|
4214
|
+
if is_http_bridge_account_neutral_replay(
|
|
4215
|
+
kind=session.key.affinity_kind,
|
|
4216
|
+
key=session.key.affinity_key,
|
|
4217
|
+
):
|
|
4218
|
+
setattr(request_state, "account_health_error_handled", True)
|
|
4219
|
+
return "not_replayable"
|
|
4220
|
+
|
|
4221
|
+
async with session.pending_lock:
|
|
4222
|
+
if request_state not in session.pending_requests:
|
|
4223
|
+
session.pending_requests.appendleft(request_state)
|
|
4224
|
+
session.queued_request_count += 1
|
|
4225
|
+
|
|
4226
|
+
_log_http_bridge_event(
|
|
4227
|
+
"retry_precreated_auth",
|
|
4228
|
+
session.key,
|
|
4229
|
+
account_id=session.account.id,
|
|
4230
|
+
model=session.request_model,
|
|
4231
|
+
pending_count=await self._http_bridge_pending_count(session),
|
|
4232
|
+
cache_key_family=session.key.affinity_kind,
|
|
4233
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
4234
|
+
)
|
|
4235
|
+
try:
|
|
4236
|
+
await self._reconnect_http_bridge_session(
|
|
4237
|
+
session,
|
|
4238
|
+
request_state=request_state,
|
|
4239
|
+
require_same_account=(
|
|
4240
|
+
bound_to_current_account
|
|
4241
|
+
or is_http_bridge_account_neutral_replay(
|
|
4242
|
+
kind=session.key.affinity_kind,
|
|
4243
|
+
key=session.key.affinity_key,
|
|
4244
|
+
)
|
|
4245
|
+
),
|
|
4246
|
+
require_preferred_account=bound_to_current_account,
|
|
4247
|
+
)
|
|
4248
|
+
request_text = self._http_bridge_text_with_account_installation_id(session, request_state, request_text)
|
|
4249
|
+
await _send_http_bridge_request_text_with_archive_id(session, request_state, request_text, clock=clock)
|
|
4250
|
+
session.last_used_at = clock.monotonic()
|
|
4251
|
+
return "retried"
|
|
4252
|
+
except UpstreamWebSocketTransportError:
|
|
4253
|
+
raise
|
|
4254
|
+
except Exception as exc:
|
|
4255
|
+
(
|
|
4256
|
+
request_state.error_http_status_override,
|
|
4257
|
+
request_state.error_code_override,
|
|
4258
|
+
request_state.error_message_override,
|
|
4259
|
+
request_state.error_type_override,
|
|
4260
|
+
request_state.error_param_override,
|
|
4261
|
+
) = _http_bridge_precreated_retry_failure_error(exc)
|
|
4262
|
+
if isinstance(exc, ProxyResponseError):
|
|
4263
|
+
logger.info(
|
|
4264
|
+
"HTTP bridge pre-created auth retry failed with terminal proxy error code=%s message=%s",
|
|
4265
|
+
request_state.error_code_override,
|
|
4266
|
+
request_state.error_message_override,
|
|
4267
|
+
)
|
|
4268
|
+
else:
|
|
4269
|
+
logger.warning("HTTP bridge pre-created auth retry failed", exc_info=True)
|
|
4270
|
+
return "failed"
|
|
4271
|
+
|
|
4272
|
+
async def _retry_http_bridge_security_work_request(
|
|
4273
|
+
self: Any,
|
|
4274
|
+
session: "_HTTPBridgeSession",
|
|
4275
|
+
request_state: _WebSocketRequestState,
|
|
4276
|
+
) -> bool:
|
|
4277
|
+
clock = clock_for(self)
|
|
4278
|
+
if session.account.security_work_authorized:
|
|
4279
|
+
return False
|
|
4280
|
+
if request_state.response_id is not None:
|
|
4281
|
+
return False
|
|
4282
|
+
if request_state.replay_count >= 1:
|
|
4283
|
+
return False
|
|
4284
|
+
if is_http_bridge_account_neutral_replay(
|
|
4285
|
+
kind=session.key.affinity_kind,
|
|
4286
|
+
key=session.key.affinity_key,
|
|
4287
|
+
):
|
|
4288
|
+
return False
|
|
4289
|
+
if request_state.file_required_preferred_account:
|
|
4290
|
+
return False
|
|
4291
|
+
if not _websocket_request_can_replay_before_visible_output(request_state):
|
|
4292
|
+
return False
|
|
4293
|
+
retry_text = _prepare_websocket_request_state_for_account_switch(request_state)
|
|
4294
|
+
if retry_text is None:
|
|
4295
|
+
return False
|
|
4296
|
+
|
|
4297
|
+
owner_account_id = session.account.id
|
|
4298
|
+
previous_replay_count = request_state.replay_count
|
|
4299
|
+
previous_response_id = request_state.response_id
|
|
4300
|
+
previous_response_event_count = request_state.response_event_count
|
|
4301
|
+
previous_upstream_model_output_seen = request_state.upstream_model_output_seen
|
|
4302
|
+
previous_affinity_policy = request_state.affinity_policy
|
|
4303
|
+
previous_affinity_observation = request_state.affinity_observation
|
|
4304
|
+
previous_session_affinity = session.affinity
|
|
4305
|
+
previous_session_codex_session = session.codex_session
|
|
4306
|
+
previous_session_upstream_turn_state = session.upstream_turn_state
|
|
4307
|
+
previous_session_downstream_turn_state = session.downstream_turn_state
|
|
4308
|
+
previous_session_turn_state_aliases = set(session.downstream_turn_state_aliases)
|
|
4309
|
+
previous_session_turn_state_alias_registration_generations = dict(
|
|
4310
|
+
session.turn_state_alias_registration_generations
|
|
4311
|
+
)
|
|
4312
|
+
previous_session_headers = session.headers
|
|
4313
|
+
request_state.preferred_account_id = None
|
|
4314
|
+
request_state.excluded_account_ids.add(owner_account_id)
|
|
4315
|
+
request_state.affinity_policy = replace(
|
|
4316
|
+
request_state.affinity_policy,
|
|
4317
|
+
key=None,
|
|
4318
|
+
kind=None,
|
|
4319
|
+
reallocate_sticky=True,
|
|
4320
|
+
)
|
|
4321
|
+
if previous_affinity_observation is not None:
|
|
4322
|
+
request_state.affinity_observation = AffinityObservation.retaining_source(
|
|
4323
|
+
previous_affinity_observation.source, request_state.affinity_policy
|
|
4324
|
+
)
|
|
4325
|
+
replacement_session_affinity = replace(
|
|
4326
|
+
session.affinity,
|
|
4327
|
+
key=None,
|
|
4328
|
+
kind=None,
|
|
4329
|
+
reallocate_sticky=True,
|
|
4330
|
+
codex_session_source=None,
|
|
4331
|
+
)
|
|
4332
|
+
|
|
4333
|
+
request_state.replay_count += 1
|
|
4334
|
+
request_state.response_id = None
|
|
4335
|
+
request_state.response_event_count = 0
|
|
4336
|
+
request_state.upstream_model_output_seen = False
|
|
4337
|
+
request_state.deferred_reasoning_downstream_texts = []
|
|
4338
|
+
request_state.awaiting_response_created = True
|
|
4339
|
+
if retry_text != request_state.request_text:
|
|
4340
|
+
_bind_http_bridge_proxy_injected_anchor(
|
|
4341
|
+
self,
|
|
4342
|
+
request_state,
|
|
4343
|
+
response_id=None,
|
|
4344
|
+
proxy_injected=False,
|
|
4345
|
+
)
|
|
4346
|
+
request_state.request_text = retry_text
|
|
4347
|
+
|
|
4348
|
+
async with session.pending_lock:
|
|
4349
|
+
if request_state not in session.pending_requests:
|
|
4350
|
+
session.pending_requests.append(request_state)
|
|
4351
|
+
session.queued_request_count += 1
|
|
4352
|
+
|
|
4353
|
+
_log_http_bridge_event(
|
|
4354
|
+
"retry_security_work_authorized",
|
|
4355
|
+
session.key,
|
|
4356
|
+
account_id=session.account.id,
|
|
4357
|
+
model=session.request_model,
|
|
4358
|
+
pending_count=await self._http_bridge_pending_count(session),
|
|
4359
|
+
cache_key_family=session.key.affinity_kind,
|
|
4360
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
4361
|
+
)
|
|
4362
|
+
reconnected = False
|
|
4363
|
+
operation_rebound_for_retry = False
|
|
4364
|
+
security_retry_send_started = False
|
|
4365
|
+
|
|
4366
|
+
def mark_security_retry_send_started() -> None:
|
|
4367
|
+
nonlocal security_retry_send_started
|
|
4368
|
+
security_retry_send_started = True
|
|
4369
|
+
|
|
4370
|
+
try:
|
|
4371
|
+
request_state.precreated_replay_account_id = session.account.id
|
|
4372
|
+
await self._release_request_state_account_response_create_lease(request_state)
|
|
4373
|
+
await _call_with_supported_optional_kwargs(
|
|
4374
|
+
self._reconnect_http_bridge_session,
|
|
4375
|
+
session,
|
|
4376
|
+
optional_kwargs={
|
|
4377
|
+
"owner_rebind_affinity": previous_session_affinity,
|
|
4378
|
+
"selection_affinity": replacement_session_affinity,
|
|
4379
|
+
},
|
|
4380
|
+
request_state=request_state,
|
|
4381
|
+
require_security_work_authorized=True,
|
|
4382
|
+
)
|
|
4383
|
+
reconnected = True
|
|
4384
|
+
settings = await _service_get_settings_cache().get()
|
|
4385
|
+
request_state.account_response_create_lease = await self._acquire_account_response_create_lease_or_overload(
|
|
4386
|
+
account_id=session.account.id,
|
|
4387
|
+
request_id=request_state.request_id,
|
|
4388
|
+
surface="http_bridge_security_retry",
|
|
4389
|
+
concurrency_caps=effective_account_concurrency_caps(settings),
|
|
4390
|
+
)
|
|
4391
|
+
request_state.account_response_create_release = self._load_balancer.release_account_lease
|
|
4392
|
+
if session.account.id != owner_account_id:
|
|
4393
|
+
if (
|
|
4394
|
+
previous_session_affinity.codex_session_source in {"session_header", "thread_header"}
|
|
4395
|
+
and previous_session_affinity.selection_key is not None
|
|
4396
|
+
and previous_session_affinity.kind is not None
|
|
4397
|
+
):
|
|
4398
|
+
async with self._repo_factory() as repos:
|
|
4399
|
+
await repos.sticky_sessions.upsert(
|
|
4400
|
+
previous_session_affinity.selection_key,
|
|
4401
|
+
session.account.id,
|
|
4402
|
+
kind=previous_session_affinity.kind,
|
|
4403
|
+
)
|
|
4404
|
+
if (
|
|
4405
|
+
request_state.operation_registered
|
|
4406
|
+
and request_state.operation_id is not None
|
|
4407
|
+
and request_state.operation_fingerprint is not None
|
|
4408
|
+
and session.durable_session_id is not None
|
|
4409
|
+
and session.durable_owner_epoch is not None
|
|
4410
|
+
):
|
|
4411
|
+
record_operation = getattr(self._durable_bridge, "record_operation", None)
|
|
4412
|
+
if not callable(record_operation):
|
|
4413
|
+
raise ProxyResponseError(
|
|
4414
|
+
502,
|
|
4415
|
+
openai_error(
|
|
4416
|
+
"bridge_continuity_persistence_failed",
|
|
4417
|
+
"Security-work recovery operation could not be re-fenced; retry the request.",
|
|
4418
|
+
),
|
|
4419
|
+
)
|
|
4420
|
+
rebound_operation = await record_operation(
|
|
4421
|
+
operation_id=request_state.operation_id,
|
|
4422
|
+
session_id=session.durable_session_id,
|
|
4423
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4424
|
+
owner_epoch=session.durable_owner_epoch,
|
|
4425
|
+
request_fingerprint=request_state.operation_fingerprint,
|
|
4426
|
+
api_key_scope=durable_bridge_api_key_scope(session.key.api_key_id),
|
|
4427
|
+
account_id=session.account.id,
|
|
4428
|
+
model=request_state.model,
|
|
4429
|
+
parent_response_id=request_state.operation_parent_response_id or request_state.previous_response_id,
|
|
4430
|
+
)
|
|
4431
|
+
if rebound_operation is None or getattr(rebound_operation, "state", None) != "submitted":
|
|
4432
|
+
raise ProxyResponseError(
|
|
4433
|
+
502,
|
|
4434
|
+
openai_error(
|
|
4435
|
+
"bridge_continuity_persistence_failed",
|
|
4436
|
+
"Security-work recovery operation could not be re-fenced; retry the request.",
|
|
4437
|
+
),
|
|
4438
|
+
)
|
|
4439
|
+
operation_rebound_for_retry = True
|
|
4440
|
+
retry_text = self._http_bridge_text_with_account_installation_id(session, request_state, retry_text)
|
|
4441
|
+
await _send_http_bridge_request_text_with_archive_id(
|
|
4442
|
+
session,
|
|
4443
|
+
request_state,
|
|
4444
|
+
retry_text,
|
|
4445
|
+
on_send_started=mark_security_retry_send_started,
|
|
4446
|
+
clock=clock,
|
|
4447
|
+
)
|
|
4448
|
+
session.last_used_at = clock.monotonic()
|
|
4449
|
+
return True
|
|
4450
|
+
except UpstreamWebSocketTransportError:
|
|
4451
|
+
raise
|
|
4452
|
+
except Exception as exc:
|
|
4453
|
+
logger.warning("HTTP bridge security-work retry failed", exc_info=True)
|
|
4454
|
+
if (
|
|
4455
|
+
operation_rebound_for_retry
|
|
4456
|
+
and not security_retry_send_started
|
|
4457
|
+
and request_state.operation_id is not None
|
|
4458
|
+
and session.durable_session_id is not None
|
|
4459
|
+
and session.durable_owner_epoch is not None
|
|
4460
|
+
):
|
|
4461
|
+
update_operation = getattr(self._durable_bridge, "update_operation", None)
|
|
4462
|
+
if callable(update_operation):
|
|
4463
|
+
try:
|
|
4464
|
+
restored = await update_operation(
|
|
4465
|
+
operation_id=request_state.operation_id,
|
|
4466
|
+
session_id=session.durable_session_id,
|
|
4467
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4468
|
+
owner_epoch=session.durable_owner_epoch,
|
|
4469
|
+
state="failed",
|
|
4470
|
+
)
|
|
4471
|
+
if not restored:
|
|
4472
|
+
logger.info(
|
|
4473
|
+
"HTTP bridge security retry failed to restore operation fence operation_id=%s",
|
|
4474
|
+
request_state.operation_id,
|
|
4475
|
+
)
|
|
4476
|
+
except Exception:
|
|
4477
|
+
logger.warning(
|
|
4478
|
+
"Failed to restore HTTP bridge security retry operation operation_id=%s",
|
|
4479
|
+
request_state.operation_id,
|
|
4480
|
+
exc_info=True,
|
|
4481
|
+
)
|
|
4482
|
+
if isinstance(exc, ProxyResponseError):
|
|
4483
|
+
error = _parse_openai_error(exc.payload)
|
|
4484
|
+
code = _normalize_error_code(error.code if error else None, error.type if error else None)
|
|
4485
|
+
if code == _NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE and request_state.event_queue is not None:
|
|
4486
|
+
await request_state.event_queue.put(
|
|
4487
|
+
format_sse_event(
|
|
4488
|
+
_security_work_advisory_event(
|
|
4489
|
+
code=_NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE,
|
|
4490
|
+
message=_SECURITY_WORK_NO_AUTHORIZED_ACCOUNTS_MESSAGE,
|
|
4491
|
+
request_id=request_state.request_log_id or request_state.request_id,
|
|
4492
|
+
action="forward_original_security_work_error",
|
|
4493
|
+
)
|
|
4494
|
+
)
|
|
4495
|
+
)
|
|
4496
|
+
async with session.pending_lock:
|
|
4497
|
+
if request_state in session.pending_requests:
|
|
4498
|
+
session.pending_requests.remove(request_state)
|
|
4499
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
4500
|
+
if reconnected:
|
|
4501
|
+
# Ownership and the replacement socket were already swapped,
|
|
4502
|
+
# but response.create never reached that socket. Retire the
|
|
4503
|
+
# replacement instead of leaving a live bridge with aliases
|
|
4504
|
+
# rebound to a turn that was never submitted.
|
|
4505
|
+
session.upstream_control.reconnect_requested = True
|
|
4506
|
+
session.upstream_control.retire_after_drain = True
|
|
4507
|
+
await self._retire_http_bridge_after_drain_if_ready(session)
|
|
4508
|
+
else:
|
|
4509
|
+
request_state.replay_count = previous_replay_count
|
|
4510
|
+
request_state.response_id = previous_response_id
|
|
4511
|
+
request_state.response_event_count = previous_response_event_count
|
|
4512
|
+
request_state.upstream_model_output_seen = previous_upstream_model_output_seen
|
|
4513
|
+
request_state.deferred_reasoning_downstream_texts = []
|
|
4514
|
+
request_state.affinity_policy = previous_affinity_policy
|
|
4515
|
+
request_state.affinity_observation = previous_affinity_observation
|
|
4516
|
+
session.affinity = previous_session_affinity
|
|
4517
|
+
session.codex_session = previous_session_codex_session
|
|
4518
|
+
session.upstream_turn_state = previous_session_upstream_turn_state
|
|
4519
|
+
session.downstream_turn_state = previous_session_downstream_turn_state
|
|
4520
|
+
async with self._http_bridge_lock:
|
|
4521
|
+
session.downstream_turn_state_aliases.update(previous_session_turn_state_aliases)
|
|
4522
|
+
session.turn_state_alias_registration_generations.update(
|
|
4523
|
+
previous_session_turn_state_alias_registration_generations
|
|
4524
|
+
)
|
|
4525
|
+
_register_http_bridge_turn_state_aliases_locked(self, session)
|
|
4526
|
+
session.headers = previous_session_headers
|
|
4527
|
+
return False
|
|
4528
|
+
|
|
4529
|
+
async def _claim_http_bridge_replacement_before_swap(
|
|
4530
|
+
self: Any,
|
|
4531
|
+
session: "_HTTPBridgeSession",
|
|
4532
|
+
*,
|
|
4533
|
+
account_id: str,
|
|
4534
|
+
upstream: Any,
|
|
4535
|
+
release_selected_account_lease: Any,
|
|
4536
|
+
owner_rebind_affinity: _AffinityPolicy,
|
|
4537
|
+
) -> None:
|
|
4538
|
+
if account_id == session.account.id:
|
|
4539
|
+
return
|
|
4540
|
+
try:
|
|
4541
|
+
if owner_rebind_affinity.legacy_selection_key is not None:
|
|
4542
|
+
async with self._repo_factory() as repos:
|
|
4543
|
+
# A goal restart abandons only session-header interpretation
|
|
4544
|
+
# of the legacy raw row. Preserve that typed capability here:
|
|
4545
|
+
# omitting it would resurrect the retained turn-state owner
|
|
4546
|
+
# during a later security-authorized replacement.
|
|
4547
|
+
legacy_owner_id = await repos.sticky_sessions.get_account_id(
|
|
4548
|
+
owner_rebind_affinity.legacy_selection_key,
|
|
4549
|
+
# The new thread row may be PROMPT_CACHE, but the raw
|
|
4550
|
+
# compatibility row has always been CODEX_SESSION and
|
|
4551
|
+
# remains durable hard ownership.
|
|
4552
|
+
kind=StickySessionKind.CODEX_SESSION,
|
|
4553
|
+
max_age_seconds=None,
|
|
4554
|
+
continuity_source=(owner_rebind_affinity.legacy_continuity_source or "session_header"),
|
|
4555
|
+
)
|
|
4556
|
+
if legacy_owner_id is not None and legacy_owner_id != account_id:
|
|
4557
|
+
raise ProxyResponseError(
|
|
4558
|
+
502,
|
|
4559
|
+
openai_error(
|
|
4560
|
+
"continuity_owner_conflict",
|
|
4561
|
+
"Security retry conflicts with a legacy continuity owner.",
|
|
4562
|
+
error_type="server_error",
|
|
4563
|
+
),
|
|
4564
|
+
)
|
|
4565
|
+
if session.durable_session_id is not None:
|
|
4566
|
+
await _call_with_supported_optional_kwargs(
|
|
4567
|
+
self._claim_durable_http_bridge_session,
|
|
4568
|
+
session,
|
|
4569
|
+
optional_kwargs={
|
|
4570
|
+
"claim_account_id": account_id,
|
|
4571
|
+
"clear_latest_turn_state": True,
|
|
4572
|
+
},
|
|
4573
|
+
allow_takeover=True,
|
|
4574
|
+
force_owner_epoch_advance=True,
|
|
4575
|
+
)
|
|
4576
|
+
except BaseException:
|
|
4577
|
+
try:
|
|
4578
|
+
await upstream.close()
|
|
4579
|
+
except Exception:
|
|
4580
|
+
logger.debug("Failed to close unclaimed HTTP bridge replacement websocket", exc_info=True)
|
|
4581
|
+
await release_selected_account_lease()
|
|
4582
|
+
raise
|