codexLB-up 1.25.0b10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +10 -0
- app/admin_cli.py +494 -0
- app/cli.py +339 -0
- app/codex_sessions_retag.py +487 -0
- app/core/__init__.py +0 -0
- app/core/audit/__init__.py +0 -0
- app/core/audit/service.py +216 -0
- app/core/audit/types.py +94 -0
- app/core/auth/__init__.py +236 -0
- app/core/auth/api_key_cache.py +65 -0
- app/core/auth/dashboard_access.py +429 -0
- app/core/auth/dashboard_mode.py +171 -0
- app/core/auth/dashboard_session_ttl.py +74 -0
- app/core/auth/dashboard_users_cache.py +120 -0
- app/core/auth/dependencies.py +710 -0
- app/core/auth/external_identity.py +78 -0
- app/core/auth/guardian.py +374 -0
- app/core/auth/models.py +49 -0
- app/core/auth/providers/__init__.py +129 -0
- app/core/auth/providers/oidc.py +838 -0
- app/core/auth/providers/registry.py +118 -0
- app/core/auth/refresh.py +394 -0
- app/core/auth/step_up.py +117 -0
- app/core/auth/totp.py +81 -0
- app/core/balancer/__init__.py +71 -0
- app/core/balancer/logic.py +1612 -0
- app/core/balancer/rendezvous_hash.py +21 -0
- app/core/balancer/types.py +21 -0
- app/core/bootstrap.py +172 -0
- app/core/cache/invalidation.py +426 -0
- app/core/clients/__init__.py +0 -0
- app/core/clients/codex.py +996 -0
- app/core/clients/codex_version.py +187 -0
- app/core/clients/codex_version_snapshot.py +2 -0
- app/core/clients/files.py +534 -0
- app/core/clients/headers.py +29 -0
- app/core/clients/http.py +511 -0
- app/core/clients/model_fetcher.py +276 -0
- app/core/clients/native_egress.py +1304 -0
- app/core/clients/oauth.py +477 -0
- app/core/clients/proxy.py +6089 -0
- app/core/clients/proxy_websocket.py +1411 -0
- app/core/clients/rate_limit_reset_credits.py +412 -0
- app/core/clients/stream_errors.py +12 -0
- app/core/clients/thread_cache_identity.py +347 -0
- app/core/clients/usage.py +467 -0
- app/core/clock.py +170 -0
- app/core/config/__init__.py +0 -0
- app/core/config/background_jobs.py +73 -0
- app/core/config/context_window_overrides.py +166 -0
- app/core/config/dashboard_overrides.py +145 -0
- app/core/config/inheritable.py +47 -0
- app/core/config/key_fingerprint.py +115 -0
- app/core/config/settings.py +872 -0
- app/core/config/settings_cache.py +100 -0
- app/core/config/spool_retention.py +217 -0
- app/core/config/tiers.py +174 -0
- app/core/conversation_archive.py +568 -0
- app/core/crypto.py +37 -0
- app/core/draining.py +11 -0
- app/core/errors.py +365 -0
- app/core/exceptions.py +155 -0
- app/core/handlers/__init__.py +3 -0
- app/core/handlers/exceptions.py +478 -0
- app/core/http_protocol.py +176 -0
- app/core/http_protocol_httptools.py +204 -0
- app/core/ingress_limits.py +17 -0
- app/core/metadata_store.py +18 -0
- app/core/metrics/__init__.py +0 -0
- app/core/metrics/middleware.py +74 -0
- app/core/metrics/prometheus.py +604 -0
- app/core/middleware/__init__.py +25 -0
- app/core/middleware/api_firewall.py +111 -0
- app/core/middleware/app_version.py +37 -0
- app/core/middleware/dashboard_auth_proxy.py +55 -0
- app/core/middleware/dashboard_csrf.py +177 -0
- app/core/middleware/dashboard_gzip.py +41 -0
- app/core/middleware/dashboard_overrides.py +59 -0
- app/core/middleware/firewall_cache.py +91 -0
- app/core/middleware/inflight.py +126 -0
- app/core/middleware/multipart_content_encoding.py +129 -0
- app/core/middleware/path_rewrite.py +148 -0
- app/core/middleware/request_body_limit.py +196 -0
- app/core/middleware/request_decompression.py +209 -0
- app/core/middleware/request_id.py +72 -0
- app/core/middleware/required_capability_http.py +116 -0
- app/core/middleware/trusted_proxy_headers.py +36 -0
- app/core/multipart.py +237 -0
- app/core/multipart_fields.py +91 -0
- app/core/openai/__init__.py +0 -0
- app/core/openai/chat_requests.py +493 -0
- app/core/openai/chat_responses.py +931 -0
- app/core/openai/contracts.py +46 -0
- app/core/openai/exceptions.py +24 -0
- app/core/openai/host_models.py +14 -0
- app/core/openai/images.py +413 -0
- app/core/openai/message_coercion.py +376 -0
- app/core/openai/model_refresh_scheduler.py +423 -0
- app/core/openai/model_registry.py +997 -0
- app/core/openai/model_registry_store.py +548 -0
- app/core/openai/models.py +172 -0
- app/core/openai/parsing.py +97 -0
- app/core/openai/requests.py +1980 -0
- app/core/openai/strict_schema.py +199 -0
- app/core/openai/tool_call_safety.py +99 -0
- app/core/openai/v1_requests.py +187 -0
- app/core/plan_types.py +90 -0
- app/core/prestop.py +202 -0
- app/core/rate_limiter/__init__.py +0 -0
- app/core/rate_limiter/db_rate_limiter.py +215 -0
- app/core/request_locality.py +473 -0
- app/core/resilience/__init__.py +0 -0
- app/core/resilience/backpressure.py +57 -0
- app/core/resilience/bulkhead.py +202 -0
- app/core/resilience/circuit_breaker.py +150 -0
- app/core/resilience/degradation.py +55 -0
- app/core/resilience/loop_lag_monitor.py +60 -0
- app/core/resilience/memory_monitor.py +137 -0
- app/core/resilience/network_recovery.py +256 -0
- app/core/resilience/overload.py +98 -0
- app/core/resilience/toggles.py +114 -0
- app/core/retention/__init__.py +0 -0
- app/core/retention/job.py +240 -0
- app/core/retention/scheduler.py +69 -0
- app/core/runtime_logging.py +601 -0
- app/core/scheduling/__init__.py +0 -0
- app/core/scheduling/leader_election.py +993 -0
- app/core/scheduling/leader_election_handle.py +25 -0
- app/core/server.py +140 -0
- app/core/shutdown.py +326 -0
- app/core/socket_peer.py +22 -0
- app/core/startup.py +41 -0
- app/core/timeout_invariants.py +347 -0
- app/core/tracing/__init__.py +0 -0
- app/core/tracing/otel.py +207 -0
- app/core/types.py +6 -0
- app/core/upstream_proxy/__init__.py +13 -0
- app/core/upstream_proxy/cache.py +130 -0
- app/core/upstream_proxy/resolver.py +189 -0
- app/core/upstream_proxy/types.py +104 -0
- app/core/usage/__init__.py +407 -0
- app/core/usage/depletion.py +109 -0
- app/core/usage/live_hub.py +41 -0
- app/core/usage/live_snapshots.py +174 -0
- app/core/usage/logs.py +212 -0
- app/core/usage/metadata_scheduler.py +118 -0
- app/core/usage/models.py +54 -0
- app/core/usage/pricing.py +576 -0
- app/core/usage/pricing_catalog.py +241 -0
- app/core/usage/pricing_snapshot.json +821 -0
- app/core/usage/quota.py +133 -0
- app/core/usage/refresh_policy.py +21 -0
- app/core/usage/refresh_scheduler.py +713 -0
- app/core/usage/reset_credits_refresh_scheduler.py +435 -0
- app/core/usage/types.py +168 -0
- app/core/utils/__init__.py +0 -0
- app/core/utils/json_guards.py +18 -0
- app/core/utils/locks.py +33 -0
- app/core/utils/masking.py +20 -0
- app/core/utils/proxy_env.py +104 -0
- app/core/utils/request_id.py +65 -0
- app/core/utils/retry.py +77 -0
- app/core/utils/shared_future.py +172 -0
- app/core/utils/sse.py +286 -0
- app/core/utils/time.py +24 -0
- app/db/__init__.py +0 -0
- app/db/account_identity_lock.py +56 -0
- app/db/alembic/__init__.py +1 -0
- app/db/alembic/env.py +70 -0
- app/db/alembic/revision_ids.py +43 -0
- app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
- app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
- app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
- app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
- app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
- app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
- app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
- app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
- app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
- app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
- app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
- app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
- app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
- app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
- app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
- app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
- app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
- app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
- app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
- app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
- app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
- app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
- app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
- app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
- app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
- app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
- app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
- app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
- app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
- app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
- app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
- app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
- app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
- app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
- app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
- app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
- app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
- app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
- app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
- app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
- app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
- app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
- app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
- app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
- app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
- app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
- app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
- app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
- app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
- app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
- app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
- app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
- app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
- app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
- app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
- app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
- app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
- app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
- app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
- app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
- app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
- app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
- app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
- app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
- app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
- app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
- app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
- app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
- app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
- app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
- app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
- app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
- app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
- app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
- app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
- app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
- app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
- app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
- app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
- app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
- app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
- app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
- app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
- app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
- app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
- app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
- app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
- app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
- app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
- app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
- app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
- app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
- app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
- app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
- app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
- app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
- app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
- app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
- app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
- app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
- app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
- app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
- app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
- app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
- app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
- app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
- app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
- app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
- app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
- app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
- app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
- app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
- app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
- app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
- app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
- app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
- app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
- app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
- app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
- app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
- app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
- app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
- app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
- app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
- app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
- app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
- app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
- app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
- app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
- app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
- app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
- app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
- app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
- app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
- app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
- app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
- app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
- app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
- app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
- app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
- app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
- app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
- app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
- app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
- app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
- app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
- app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
- app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
- app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
- app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
- app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
- app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
- app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
- app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
- app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
- app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
- app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
- app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
- app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
- app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
- app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
- app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
- app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
- app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
- app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
- app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
- app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
- app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
- app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
- app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
- app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
- app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
- app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
- app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
- app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
- app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
- app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
- app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
- app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
- app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
- app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
- app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
- app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
- app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
- app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
- app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
- app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
- app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
- app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
- app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
- app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
- app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
- app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
- app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
- app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
- app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
- app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
- app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
- app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
- app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
- app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
- app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
- app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
- app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
- app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
- app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
- app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
- app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
- app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
- app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
- app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
- app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
- app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
- app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
- app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
- app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
- app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
- app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
- app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
- app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
- app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
- app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
- app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
- app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
- app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
- app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
- app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
- app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
- app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
- app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
- app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
- app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
- app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
- app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
- app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
- app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
- app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
- app/db/alembic/versions/__init__.py +1 -0
- app/db/backup.py +64 -0
- app/db/migrate.py +1249 -0
- app/db/migration_lock.py +160 -0
- app/db/migration_url.py +18 -0
- app/db/models.py +3100 -0
- app/db/recover.py +244 -0
- app/db/session.py +1295 -0
- app/db/snapshot.py +46 -0
- app/db/sqlite_lock_retry.py +190 -0
- app/db/sqlite_utils.py +549 -0
- app/dependencies.py +462 -0
- app/main.py +1240 -0
- app/modules/__init__.py +0 -0
- app/modules/accounts/__init__.py +0 -0
- app/modules/accounts/api.py +426 -0
- app/modules/accounts/auth_manager.py +1439 -0
- app/modules/accounts/background_repository.py +160 -0
- app/modules/accounts/deletion.py +540 -0
- app/modules/accounts/mappers.py +654 -0
- app/modules/accounts/refresh_claims.py +352 -0
- app/modules/accounts/repository.py +1679 -0
- app/modules/accounts/schemas.py +308 -0
- app/modules/accounts/service.py +809 -0
- app/modules/accounts/usage_rollup.py +486 -0
- app/modules/accounts/usage_rollup_scheduler.py +75 -0
- app/modules/accounts/usage_time_rollup.py +1037 -0
- app/modules/accounts/usage_time_rollup_read.py +522 -0
- app/modules/api_keys/__init__.py +1 -0
- app/modules/api_keys/api.py +344 -0
- app/modules/api_keys/last_used_coalescer.py +216 -0
- app/modules/api_keys/limit_windows.py +41 -0
- app/modules/api_keys/repository.py +1200 -0
- app/modules/api_keys/reset_scheduler.py +85 -0
- app/modules/api_keys/schemas.py +135 -0
- app/modules/api_keys/service.py +2123 -0
- app/modules/audit/__init__.py +0 -0
- app/modules/audit/api.py +88 -0
- app/modules/audit/repository.py +74 -0
- app/modules/audit/schemas.py +33 -0
- app/modules/audit/service.py +71 -0
- app/modules/auth_providers/__init__.py +0 -0
- app/modules/auth_providers/api.py +127 -0
- app/modules/auth_providers/config.py +343 -0
- app/modules/auth_providers/repository.py +101 -0
- app/modules/auth_providers/schemas.py +88 -0
- app/modules/auth_providers/seed.py +105 -0
- app/modules/auth_providers/service.py +289 -0
- app/modules/automations/__init__.py +1 -0
- app/modules/automations/api.py +369 -0
- app/modules/automations/repository.py +1896 -0
- app/modules/automations/scheduler.py +107 -0
- app/modules/automations/schemas.py +158 -0
- app/modules/automations/service.py +2548 -0
- app/modules/cache_isolation_probe/__init__.py +0 -0
- app/modules/cache_isolation_probe/api.py +188 -0
- app/modules/cache_isolation_probe/prefix.py +144 -0
- app/modules/cache_isolation_probe/schemas.py +94 -0
- app/modules/cache_isolation_probe/sender.py +229 -0
- app/modules/cache_isolation_probe/service.py +525 -0
- app/modules/conversation_archive/__init__.py +1 -0
- app/modules/conversation_archive/api.py +101 -0
- app/modules/conversation_archive/schemas.py +28 -0
- app/modules/conversation_archive/service.py +165 -0
- app/modules/dashboard/__init__.py +1 -0
- app/modules/dashboard/api.py +106 -0
- app/modules/dashboard/builders.py +107 -0
- app/modules/dashboard/repository.py +304 -0
- app/modules/dashboard/schemas.py +148 -0
- app/modules/dashboard/service.py +534 -0
- app/modules/dashboard/timeframes.py +19 -0
- app/modules/dashboard/weekly_pace.py +617 -0
- app/modules/dashboard_auth/__init__.py +1 -0
- app/modules/dashboard_auth/api.py +1344 -0
- app/modules/dashboard_auth/oidc_api.py +753 -0
- app/modules/dashboard_auth/oidc_flows.py +330 -0
- app/modules/dashboard_auth/repository.py +452 -0
- app/modules/dashboard_auth/schemas.py +232 -0
- app/modules/dashboard_auth/service.py +1384 -0
- app/modules/dashboard_roles/__init__.py +0 -0
- app/modules/dashboard_roles/api.py +64 -0
- app/modules/dashboard_roles/repository.py +54 -0
- app/modules/dashboard_roles/schemas.py +29 -0
- app/modules/dashboard_roles/seed.py +80 -0
- app/modules/dashboard_roles/service.py +142 -0
- app/modules/dashboard_users/__init__.py +0 -0
- app/modules/dashboard_users/api.py +278 -0
- app/modules/dashboard_users/break_glass.py +169 -0
- app/modules/dashboard_users/credentials.py +18 -0
- app/modules/dashboard_users/identity_resolver.py +665 -0
- app/modules/dashboard_users/repository.py +725 -0
- app/modules/dashboard_users/schemas.py +117 -0
- app/modules/dashboard_users/service.py +1054 -0
- app/modules/firewall/__init__.py +1 -0
- app/modules/firewall/api.py +78 -0
- app/modules/firewall/repository.py +51 -0
- app/modules/firewall/schemas.py +28 -0
- app/modules/firewall/service.py +93 -0
- app/modules/fleet/__init__.py +1 -0
- app/modules/fleet/api.py +177 -0
- app/modules/fleet/mappers.py +56 -0
- app/modules/fleet/observability.py +372 -0
- app/modules/fleet/schemas.py +119 -0
- app/modules/health/__init__.py +0 -0
- app/modules/health/api.py +246 -0
- app/modules/health/schemas.py +27 -0
- app/modules/limit_warmup/__init__.py +1 -0
- app/modules/limit_warmup/repository.py +159 -0
- app/modules/limit_warmup/service.py +1001 -0
- app/modules/model_sources/__init__.py +1 -0
- app/modules/model_sources/api.py +99 -0
- app/modules/model_sources/catalog.py +307 -0
- app/modules/model_sources/forwarding.py +1435 -0
- app/modules/model_sources/projection.py +55 -0
- app/modules/model_sources/repository.py +201 -0
- app/modules/model_sources/schemas.py +79 -0
- app/modules/model_sources/selection.py +166 -0
- app/modules/model_sources/service.py +236 -0
- app/modules/oauth/__init__.py +0 -0
- app/modules/oauth/api.py +101 -0
- app/modules/oauth/repository.py +238 -0
- app/modules/oauth/schemas.py +49 -0
- app/modules/oauth/service.py +1196 -0
- app/modules/oauth/templates/oauth_success.html +122 -0
- app/modules/proxy/__init__.py +0 -0
- app/modules/proxy/_load_balancer/__init__.py +1 -0
- app/modules/proxy/_load_balancer/error_rate.py +116 -0
- app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
- app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
- app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
- app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
- app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
- app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
- app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
- app/modules/proxy/_load_balancer/tunables.py +97 -0
- app/modules/proxy/_load_balancer/types.py +96 -0
- app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
- app/modules/proxy/_service/__init__.py +6 -0
- app/modules/proxy/_service/api_key_usage.py +745 -0
- app/modules/proxy/_service/clock_budget.py +38 -0
- app/modules/proxy/_service/codex_control.py +611 -0
- app/modules/proxy/_service/compact.py +2211 -0
- app/modules/proxy/_service/file_ops.py +749 -0
- app/modules/proxy/_service/http_bridge/__init__.py +3 -0
- app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
- app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
- app/modules/proxy/_service/http_bridge/activity.py +178 -0
- app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
- app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
- app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
- app/modules/proxy/_service/http_bridge/protocol.py +167 -0
- app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
- app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
- app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
- app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
- app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
- app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
- app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
- app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
- app/modules/proxy/_service/observability.py +410 -0
- app/modules/proxy/_service/rate_limit.py +361 -0
- app/modules/proxy/_service/realtime_live.py +655 -0
- app/modules/proxy/_service/refresh.py +154 -0
- app/modules/proxy/_service/request_log.py +585 -0
- app/modules/proxy/_service/response_create.py +930 -0
- app/modules/proxy/_service/streaming/__init__.py +3 -0
- app/modules/proxy/_service/streaming/helpers.py +1317 -0
- app/modules/proxy/_service/streaming/mixin.py +1100 -0
- app/modules/proxy/_service/streaming/protocol.py +32 -0
- app/modules/proxy/_service/streaming/retry.py +3762 -0
- app/modules/proxy/_service/support.py +2122 -0
- app/modules/proxy/_service/transcribe.py +434 -0
- app/modules/proxy/_service/warmup.py +483 -0
- app/modules/proxy/_service/websocket/__init__.py +3 -0
- app/modules/proxy/_service/websocket/helpers.py +2360 -0
- app/modules/proxy/_service/websocket/mixin.py +7285 -0
- app/modules/proxy/_service/websocket/protocol.py +74 -0
- app/modules/proxy/account_cache.py +229 -0
- app/modules/proxy/account_eligibility.py +87 -0
- app/modules/proxy/additional_model_limits.py +40 -0
- app/modules/proxy/affinity.py +830 -0
- app/modules/proxy/affinity_observation.py +101 -0
- app/modules/proxy/api.py +10254 -0
- app/modules/proxy/api_key_usage.py +113 -0
- app/modules/proxy/cap_partitioning.py +280 -0
- app/modules/proxy/capability_lineage.py +65 -0
- app/modules/proxy/capability_lineage_repository.py +82 -0
- app/modules/proxy/capability_routing.py +253 -0
- app/modules/proxy/continuity.py +112 -0
- app/modules/proxy/downstream_delivery.py +277 -0
- app/modules/proxy/durable_bridge_coordinator.py +1217 -0
- app/modules/proxy/durable_bridge_repository.py +4516 -0
- app/modules/proxy/durable_bridge_runtime.py +9 -0
- app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
- app/modules/proxy/fair_share.py +131 -0
- app/modules/proxy/file_pin_repository.py +210 -0
- app/modules/proxy/helpers.py +439 -0
- app/modules/proxy/http_bridge_event_batcher.py +613 -0
- app/modules/proxy/http_bridge_forwarding.py +797 -0
- app/modules/proxy/http_continuation.py +76 -0
- app/modules/proxy/images_observability.py +62 -0
- app/modules/proxy/images_service.py +1054 -0
- app/modules/proxy/load_balancer.py +3016 -0
- app/modules/proxy/rate_limit_cache.py +46 -0
- app/modules/proxy/replay_safety.py +1063 -0
- app/modules/proxy/repo_bundle.py +31 -0
- app/modules/proxy/request_policy.py +1054 -0
- app/modules/proxy/ring_membership.py +251 -0
- app/modules/proxy/schemas.py +345 -0
- app/modules/proxy/selection_errors.py +41 -0
- app/modules/proxy/service.py +2600 -0
- app/modules/proxy/source_admission.py +125 -0
- app/modules/proxy/source_dispatch.py +999 -0
- app/modules/proxy/sticky_repository.py +831 -0
- app/modules/proxy/thread_anchors.py +491 -0
- app/modules/proxy/tool_call_dedupe.py +802 -0
- app/modules/proxy/types.py +54 -0
- app/modules/proxy/work_admission.py +121 -0
- app/modules/quota_planner/__init__.py +1 -0
- app/modules/quota_planner/api.py +276 -0
- app/modules/quota_planner/logic.py +743 -0
- app/modules/quota_planner/repository.py +722 -0
- app/modules/quota_planner/scheduler.py +212 -0
- app/modules/quota_planner/schemas.py +102 -0
- app/modules/quota_planner/warmup.py +693 -0
- app/modules/rate_limit_reset_credits/__init__.py +0 -0
- app/modules/rate_limit_reset_credits/api.py +601 -0
- app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
- app/modules/rate_limit_reset_credits/store.py +114 -0
- app/modules/reports/__init__.py +1 -0
- app/modules/reports/api.py +134 -0
- app/modules/reports/cache.py +74 -0
- app/modules/reports/filters.py +27 -0
- app/modules/reports/repository.py +525 -0
- app/modules/reports/rollup.py +150 -0
- app/modules/reports/rollup_read.py +148 -0
- app/modules/reports/schemas.py +123 -0
- app/modules/reports/service.py +287 -0
- app/modules/reports/thread_identity.py +256 -0
- app/modules/request_logs/__init__.py +0 -0
- app/modules/request_logs/api.py +224 -0
- app/modules/request_logs/cost_backfill.py +154 -0
- app/modules/request_logs/mappers.py +97 -0
- app/modules/request_logs/repository.py +1765 -0
- app/modules/request_logs/schemas.py +145 -0
- app/modules/request_logs/service.py +348 -0
- app/modules/role_mappings/__init__.py +0 -0
- app/modules/role_mappings/api.py +206 -0
- app/modules/role_mappings/matching.py +55 -0
- app/modules/role_mappings/repository.py +106 -0
- app/modules/role_mappings/schemas.py +67 -0
- app/modules/role_mappings/service.py +334 -0
- app/modules/runtime/__init__.py +1 -0
- app/modules/runtime/api.py +18 -0
- app/modules/runtime/schemas.py +14 -0
- app/modules/runtime/service.py +218 -0
- app/modules/scim/__init__.py +1 -0
- app/modules/scim/api.py +194 -0
- app/modules/scim/dependencies.py +112 -0
- app/modules/scim/errors.py +77 -0
- app/modules/scim/management_api.py +156 -0
- app/modules/scim/repository.py +212 -0
- app/modules/scim/schemas.py +145 -0
- app/modules/scim/service.py +786 -0
- app/modules/scim/tokens.py +43 -0
- app/modules/settings/__init__.py +1 -0
- app/modules/settings/api.py +1677 -0
- app/modules/settings/repository.py +606 -0
- app/modules/settings/schemas.py +496 -0
- app/modules/settings/service.py +769 -0
- app/modules/shared/__init__.py +0 -0
- app/modules/shared/schemas.py +22 -0
- app/modules/sticky_sessions/__init__.py +1 -0
- app/modules/sticky_sessions/api.py +118 -0
- app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
- app/modules/sticky_sessions/schemas.py +78 -0
- app/modules/sticky_sessions/service.py +196 -0
- app/modules/telemetry/__init__.py +5 -0
- app/modules/telemetry/api.py +115 -0
- app/modules/telemetry/clients.py +62 -0
- app/modules/telemetry/consent.py +115 -0
- app/modules/telemetry/scheduler.py +85 -0
- app/modules/telemetry/schemas.py +163 -0
- app/modules/telemetry/sender.py +221 -0
- app/modules/telemetry/snapshot.py +490 -0
- app/modules/usage/__init__.py +0 -0
- app/modules/usage/additional_quota_keys.py +379 -0
- app/modules/usage/api.py +45 -0
- app/modules/usage/background_repository.py +147 -0
- app/modules/usage/builders.py +559 -0
- app/modules/usage/depletion_service.py +277 -0
- app/modules/usage/live_ingest.py +356 -0
- app/modules/usage/mappers.py +29 -0
- app/modules/usage/plan_downgrade_observations.py +488 -0
- app/modules/usage/repository.py +1890 -0
- app/modules/usage/schemas.py +68 -0
- app/modules/usage/service.py +102 -0
- app/modules/usage/updater.py +1488 -0
- app/static/assets/Trans-Cj-J9mVA.js +1 -0
- app/static/assets/access-page-DHzZiXZi.js +1 -0
- app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
- app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
- app/static/assets/account-status-DIsU5sUS.js +1 -0
- app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
- app/static/assets/accounts-page-Bbai8ift.js +3 -0
- app/static/assets/api-Dwa7XxPe.js +1 -0
- app/static/assets/api-client-dugX1Pti.js +55 -0
- app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
- app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
- app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
- app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
- app/static/assets/apis-page-BvXz9c6i.js +2 -0
- app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
- app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
- app/static/assets/automations-page-DQgNAJOR.js +3 -0
- app/static/assets/badge-DqMb9es0.js +1 -0
- app/static/assets/blur-email-HEvKagG2.js +1 -0
- app/static/assets/button-BjVmmiGW.js +1 -0
- app/static/assets/checkbox-apsl9vzk.js +1 -0
- app/static/assets/chevron-left-1d_pHVa5.js +1 -0
- app/static/assets/circle-alert-BS0xd_o1.js +1 -0
- app/static/assets/clsx-DB0hHKMi.js +1 -0
- app/static/assets/colors-BBy8GEtr.js +1 -0
- app/static/assets/copy-button-D7nAvOKR.js +1 -0
- app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
- app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
- app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
- app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
- app/static/assets/donut-chart-Bxc69cra.js +1 -0
- app/static/assets/download-kQteIcsB.js +1 -0
- app/static/assets/ellipsis-B_5OgIre.js +1 -0
- app/static/assets/empty-state-JcA7tc8m.js +1 -0
- app/static/assets/es2015-bMH8YKZT.js +41 -0
- app/static/assets/es6-B264ni6W.js +69 -0
- app/static/assets/external-link-UVAOgMQ0.js +1 -0
- app/static/assets/i18n-BWqZ9Ua2.js +1 -0
- app/static/assets/inbox-B_RM2s1J.js +1 -0
- app/static/assets/index-DIpd264Z.js +23 -0
- app/static/assets/index-DMeLKRJx.css +2 -0
- app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
- app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
- app/static/assets/iso-Bpe5Ihbb.js +1 -0
- app/static/assets/label-CGaLmLNf.js +1 -0
- app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
- app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
- app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
- app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
- app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
- app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
- app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
- app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
- app/static/assets/payload-CIYnGZz9.js +1 -0
- app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
- app/static/assets/report-chart-card-BXDliwcc.js +1 -0
- app/static/assets/reports-page-CGCJeMHG.js +3 -0
- app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
- app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
- app/static/assets/search-HfhqwERt.js +1 -0
- app/static/assets/select-Cl33bz-g.js +1 -0
- app/static/assets/settings-page-mZj01HYl.js +2 -0
- app/static/assets/shim-CZwyM4Q7.js +1 -0
- app/static/assets/skeleton-C_EUc4Rp.js +1 -0
- app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
- app/static/assets/status-badge-CHHA5X6c.js +1 -0
- app/static/assets/switch-CzxgokcM.js +1 -0
- app/static/assets/table-DXaNSQ0G.js +1 -0
- app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
- app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
- app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
- app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
- app/static/assets/trash-D3KyWr1h.js +1 -0
- app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
- app/static/assets/use-accounts-PWlVGw3z.js +1 -0
- app/static/assets/use-auth-CuDlW3iC.js +1 -0
- app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
- app/static/assets/use-model-sources-Bul24JBu.js +1 -0
- app/static/assets/use-models-DUCPUuZg.js +1 -0
- app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
- app/static/assets/useTranslation--56nHhlH.js +1 -0
- app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
- app/static/assets/users-uIC3TIlE.js +1 -0
- app/static/assets/vendor-query-_bHZuxEx.js +1 -0
- app/static/assets/vendor-react-BVL0YZPW.js +9 -0
- app/static/favicon.svg +10 -0
- app/static/fonts/GeistSans-Variable.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
- app/static/index.html +29 -0
- codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
- codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
- codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
- codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
- codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
- config/additional_quota_registry.json +11 -0
|
@@ -0,0 +1,4538 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
from collections.abc import Awaitable, Callable, Iterable
|
|
6
|
+
from dataclasses import replace
|
|
7
|
+
from typing import Any, TypeVar, cast
|
|
8
|
+
|
|
9
|
+
import anyio
|
|
10
|
+
|
|
11
|
+
from app.core.balancer.types import UpstreamError
|
|
12
|
+
from app.core.clients.files import create_file as core_create_file # noqa: F401
|
|
13
|
+
from app.core.clients.files import finalize_file as core_finalize_file # noqa: F401
|
|
14
|
+
from app.core.clients.proxy import CodexControlResponse as CodexControlResponse
|
|
15
|
+
from app.core.clients.proxy import ( # noqa: F401
|
|
16
|
+
ImageFetchSession,
|
|
17
|
+
ProxyResponseError,
|
|
18
|
+
UpstreamProxyRouteTrace,
|
|
19
|
+
_as_image_fetch_session,
|
|
20
|
+
_inline_content_images,
|
|
21
|
+
_inline_input_image_urls,
|
|
22
|
+
_ws_transport_payload_budget_bytes,
|
|
23
|
+
filter_inbound_headers,
|
|
24
|
+
pop_compact_timeout_overrides,
|
|
25
|
+
pop_stream_timeout_overrides,
|
|
26
|
+
pop_transcribe_timeout_overrides,
|
|
27
|
+
push_compact_timeout_overrides,
|
|
28
|
+
push_stream_timeout_overrides,
|
|
29
|
+
push_transcribe_timeout_overrides,
|
|
30
|
+
)
|
|
31
|
+
from app.core.clients.proxy import codex_control_request as core_codex_control_request # noqa: F401
|
|
32
|
+
from app.core.clients.proxy import compact_responses as core_compact_responses # noqa: F401
|
|
33
|
+
from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401
|
|
34
|
+
from app.core.clients.proxy_websocket import (
|
|
35
|
+
UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE,
|
|
36
|
+
UpstreamWebSocketMessage,
|
|
37
|
+
UpstreamWebSocketTransportError,
|
|
38
|
+
is_account_neutral_websocket_error_code,
|
|
39
|
+
)
|
|
40
|
+
from app.core.clock import REAL_CLOCK, REAL_SCHEDULER, Clock, Scheduler, clock_for, scheduler_for
|
|
41
|
+
from app.core.errors import response_failed_event
|
|
42
|
+
from app.core.openai.models import OpenAIEvent
|
|
43
|
+
from app.core.openai.parsing import (
|
|
44
|
+
_LIFECYCLE_EVENT_TYPES,
|
|
45
|
+
classify_event_type,
|
|
46
|
+
parse_sse_event_payload,
|
|
47
|
+
)
|
|
48
|
+
from app.core.types import JsonValue
|
|
49
|
+
from app.core.usage.live_hub import publish_live_usage
|
|
50
|
+
from app.core.usage.live_snapshots import EVENT_MARKER, parse_rate_limit_event_text
|
|
51
|
+
from app.core.utils.request_id import reset_request_id, set_request_id
|
|
52
|
+
from app.core.utils.shared_future import wait_on_shared_future
|
|
53
|
+
from app.core.utils.sse import format_sse_event, format_sse_event_from_text, parse_sse_data_json_text
|
|
54
|
+
from app.db.models import Account
|
|
55
|
+
from app.modules.proxy._service.api_key_usage import (
|
|
56
|
+
_API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS,
|
|
57
|
+
)
|
|
58
|
+
from app.modules.proxy._service.compact import (
|
|
59
|
+
_sticky_key_for_compact_request as _sticky_key_for_compact_request,
|
|
60
|
+
)
|
|
61
|
+
from app.modules.proxy._service.compact import (
|
|
62
|
+
_sticky_key_from_compact_payload as _sticky_key_from_compact_payload,
|
|
63
|
+
)
|
|
64
|
+
from app.modules.proxy._service.http_bridge import helpers as _http_bridge_helpers
|
|
65
|
+
from app.modules.proxy._service.http_bridge.accepted_replay import (
|
|
66
|
+
_http_bridge_accepted_anchored_replay_candidate,
|
|
67
|
+
_http_bridge_accepted_capacity_retry_allowed,
|
|
68
|
+
_stage_websocket_request_state_for_replay,
|
|
69
|
+
)
|
|
70
|
+
from app.modules.proxy._service.http_bridge.helpers import (
|
|
71
|
+
_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
|
|
72
|
+
_await_task_deferring_cancellation,
|
|
73
|
+
_forget_http_bridge_denied_anchor_fence,
|
|
74
|
+
_http_bridge_abandonment_may_settle_circuit,
|
|
75
|
+
_http_bridge_denied_anchor_fence_current_map,
|
|
76
|
+
_http_bridge_denied_anchor_fence_entry,
|
|
77
|
+
_http_bridge_durable_lease_ttl_seconds,
|
|
78
|
+
_http_bridge_event_proves_upstream_liveness,
|
|
79
|
+
_http_bridge_eventless_precreated_deadline,
|
|
80
|
+
_http_bridge_request_budget_seconds,
|
|
81
|
+
_http_bridge_request_counts_against_queue,
|
|
82
|
+
_http_bridge_request_state_holds_safe_replay,
|
|
83
|
+
_http_bridge_retry_circuit_attempt_selection_for_pending_requests,
|
|
84
|
+
_log_http_bridge_event,
|
|
85
|
+
_normalize_http_bridge_error_event,
|
|
86
|
+
_record_http_bridge_denied_anchor_fence,
|
|
87
|
+
_record_http_bridge_stuck_retire,
|
|
88
|
+
_record_http_bridge_unmatched_upstream_liveness,
|
|
89
|
+
_schedule_http_bridge_background_cleanup,
|
|
90
|
+
)
|
|
91
|
+
from app.modules.proxy._service.http_bridge.quarantine import (
|
|
92
|
+
_clear_http_bridge_quarantine,
|
|
93
|
+
_http_bridge_quarantine_clear_fence,
|
|
94
|
+
_record_http_bridge_quarantine_eventless_timeout,
|
|
95
|
+
_record_http_bridge_quarantine_wedged_pending,
|
|
96
|
+
)
|
|
97
|
+
from app.modules.proxy._service.http_bridge.retry_circuit import (
|
|
98
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
|
|
99
|
+
_POISON_ANCHOR_CAPTURE_UNAVAILABLE,
|
|
100
|
+
_http_bridge_anchor_poison_detail,
|
|
101
|
+
)
|
|
102
|
+
from app.modules.proxy._service.http_bridge.service_stubs import (
|
|
103
|
+
_assign_websocket_response_id,
|
|
104
|
+
_await_cancelled_task,
|
|
105
|
+
_build_stream_incomplete_terminal_event_for_request,
|
|
106
|
+
_classify_upstream_close,
|
|
107
|
+
_find_websocket_request_state_by_response_id,
|
|
108
|
+
_http_error_status_from_payload,
|
|
109
|
+
_is_account_neutral_transport_drop,
|
|
110
|
+
_is_missing_tool_output_error,
|
|
111
|
+
_is_previous_response_not_found_error,
|
|
112
|
+
_is_security_work_authorization_required_error,
|
|
113
|
+
_match_websocket_request_state_for_anonymous_event,
|
|
114
|
+
_matching_websocket_request_states_for_missing_tool_output_error,
|
|
115
|
+
_matching_websocket_request_states_for_previous_response_error,
|
|
116
|
+
_maybe_rewrite_websocket_previous_response_not_found_event,
|
|
117
|
+
_pop_matching_websocket_request_states,
|
|
118
|
+
_pop_terminal_websocket_request_state,
|
|
119
|
+
_prepare_websocket_request_state_for_account_switch,
|
|
120
|
+
_previous_response_id_from_not_found_message,
|
|
121
|
+
_release_websocket_response_create_gate,
|
|
122
|
+
_response_output_item_done_tool_call,
|
|
123
|
+
_rewrite_websocket_continuity_corruption_event,
|
|
124
|
+
_rewrite_websocket_downstream_response_id,
|
|
125
|
+
_rewrite_websocket_previous_response_owner_unavailable_event,
|
|
126
|
+
_rewrite_websocket_suppressed_duplicate_tool_call_completion_event,
|
|
127
|
+
_security_work_advisory_event,
|
|
128
|
+
_service_get_settings,
|
|
129
|
+
_service_tier_from_event_payload,
|
|
130
|
+
_upstream_websocket_disconnect_message,
|
|
131
|
+
_websocket_auth_request_can_switch_account,
|
|
132
|
+
_websocket_downstream_response_id,
|
|
133
|
+
_websocket_event_error_code,
|
|
134
|
+
_websocket_event_error_message,
|
|
135
|
+
_websocket_event_error_param,
|
|
136
|
+
_websocket_event_error_type,
|
|
137
|
+
_websocket_owner_pinned_quota_error_code,
|
|
138
|
+
_websocket_precreated_auth_error_code,
|
|
139
|
+
_websocket_precreated_retry_error_code,
|
|
140
|
+
_websocket_response_id,
|
|
141
|
+
)
|
|
142
|
+
from app.modules.proxy._service.observability import (
|
|
143
|
+
_hash_identifier as _hash_identifier,
|
|
144
|
+
)
|
|
145
|
+
from app.modules.proxy._service.observability import (
|
|
146
|
+
_hash_identifier_or_none as _hash_identifier_or_none,
|
|
147
|
+
)
|
|
148
|
+
from app.modules.proxy._service.observability import (
|
|
149
|
+
_interesting_header_keys as _interesting_header_keys,
|
|
150
|
+
)
|
|
151
|
+
from app.modules.proxy._service.observability import (
|
|
152
|
+
_tools_hash as _tools_hash,
|
|
153
|
+
)
|
|
154
|
+
from app.modules.proxy._service.observability import (
|
|
155
|
+
_truncate_identifier as _truncate_identifier,
|
|
156
|
+
)
|
|
157
|
+
from app.modules.proxy._service.support import (
|
|
158
|
+
_ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE,
|
|
159
|
+
_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS,
|
|
160
|
+
_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS,
|
|
161
|
+
_HARD_HTTP_BRIDGE_AFFINITY_KINDS, # noqa: F401
|
|
162
|
+
_MODEL_OUTPUT_EVENT_TYPES,
|
|
163
|
+
_PENDING_TOOL_CALL_ITEM_TYPES,
|
|
164
|
+
_WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401
|
|
165
|
+
_account_capacity_wait_payload,
|
|
166
|
+
_clear_websocket_deferred_reasoning_downstream_texts,
|
|
167
|
+
_clear_websocket_precreated_replay_fallback,
|
|
168
|
+
_clear_websocket_request_error_overrides,
|
|
169
|
+
_DeferredKeyedStreamHealthPenalty,
|
|
170
|
+
_HTTPBridgeCompletedDeliveryScope,
|
|
171
|
+
_HTTPBridgeRetryCircuitAttemptSelection,
|
|
172
|
+
_HTTPBridgeSession,
|
|
173
|
+
_mark_response_create_attempt_observed,
|
|
174
|
+
_pop_websocket_deferred_reasoning_downstream_texts,
|
|
175
|
+
_record_response_event,
|
|
176
|
+
_signal_propagated_capacity_startup_ready,
|
|
177
|
+
_signal_propagated_capacity_startup_wait,
|
|
178
|
+
_websocket_request_can_replay_before_visible_output,
|
|
179
|
+
_websocket_should_defer_reasoning_prelude,
|
|
180
|
+
_WebSocketReceiveTimeout,
|
|
181
|
+
_WebSocketRequestState,
|
|
182
|
+
)
|
|
183
|
+
from app.modules.proxy._service.support import (
|
|
184
|
+
_websocket_route_log_kwargs as _websocket_route_log_kwargs,
|
|
185
|
+
)
|
|
186
|
+
from app.modules.proxy._service.warmup import (
|
|
187
|
+
WarmupExecutionData as WarmupExecutionData,
|
|
188
|
+
)
|
|
189
|
+
from app.modules.proxy._service.warmup import (
|
|
190
|
+
WarmupFailedAccountData as WarmupFailedAccountData,
|
|
191
|
+
)
|
|
192
|
+
from app.modules.proxy._service.warmup import (
|
|
193
|
+
WarmupSkippedAccountData as WarmupSkippedAccountData,
|
|
194
|
+
)
|
|
195
|
+
from app.modules.proxy._service.warmup import (
|
|
196
|
+
WarmupSubmittedAccountData as WarmupSubmittedAccountData,
|
|
197
|
+
)
|
|
198
|
+
from app.modules.proxy._service.warmup import (
|
|
199
|
+
_is_warmup_usage_eligible as _is_warmup_usage_eligible,
|
|
200
|
+
)
|
|
201
|
+
from app.modules.proxy._service.warmup import (
|
|
202
|
+
_materialize_warmup_account as _materialize_warmup_account,
|
|
203
|
+
)
|
|
204
|
+
from app.modules.proxy._service.warmup import (
|
|
205
|
+
_snapshot_warmup_account as _snapshot_warmup_account,
|
|
206
|
+
)
|
|
207
|
+
from app.modules.proxy._service.warmup import (
|
|
208
|
+
_WarmupAccountSnapshot as _WarmupAccountSnapshot,
|
|
209
|
+
)
|
|
210
|
+
from app.modules.proxy._service.warmup import (
|
|
211
|
+
_WarmupSubmitResult as _WarmupSubmitResult,
|
|
212
|
+
)
|
|
213
|
+
from app.modules.proxy._service.warmup import (
|
|
214
|
+
_WarmupUsageSnapshot as _WarmupUsageSnapshot,
|
|
215
|
+
)
|
|
216
|
+
from app.modules.proxy.affinity import (
|
|
217
|
+
_extract_model_class,
|
|
218
|
+
)
|
|
219
|
+
from app.modules.proxy.continuity import is_http_bridge_account_neutral_replay
|
|
220
|
+
from app.modules.proxy.helpers import (
|
|
221
|
+
_normalize_error_code,
|
|
222
|
+
is_upstream_model_capacity_error,
|
|
223
|
+
)
|
|
224
|
+
from app.modules.proxy.tool_call_dedupe import (
|
|
225
|
+
mark_duplicate_tool_call_downstream_event,
|
|
226
|
+
rewrite_parallel_tool_call_text,
|
|
227
|
+
)
|
|
228
|
+
from app.modules.proxy.tool_call_dedupe import (
|
|
229
|
+
response_id_from_payload as tool_call_response_id_from_payload,
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
logger = logging.getLogger("app.modules.proxy.service")
|
|
233
|
+
|
|
234
|
+
_HTTP_BRIDGE_RECOVERY_SETTLEMENT_RETRY_DELAYS = (
|
|
235
|
+
0.25,
|
|
236
|
+
0.5,
|
|
237
|
+
1.0,
|
|
238
|
+
2.0,
|
|
239
|
+
4.0,
|
|
240
|
+
8.0,
|
|
241
|
+
15.0,
|
|
242
|
+
30.0,
|
|
243
|
+
60.0,
|
|
244
|
+
120.0,
|
|
245
|
+
)
|
|
246
|
+
_HTTP_BRIDGE_RECOVERY_SETTLEMENT_LEASE_REFRESH_INTERVAL_SECONDS = 10.0
|
|
247
|
+
_HTTP_BRIDGE_DENIED_ANCHOR_CLEAR_RETRY_DELAYS = (
|
|
248
|
+
0.25,
|
|
249
|
+
0.5,
|
|
250
|
+
1.0,
|
|
251
|
+
2.0,
|
|
252
|
+
4.0,
|
|
253
|
+
8.0,
|
|
254
|
+
15.0,
|
|
255
|
+
30.0,
|
|
256
|
+
60.0,
|
|
257
|
+
)
|
|
258
|
+
# A single missing response.created is not proof that an account is bad: the
|
|
259
|
+
# upstream may have accepted the request while the transport was silent. Only
|
|
260
|
+
# repeated failures on separate bridge retirements are allowed to influence
|
|
261
|
+
# account routing, and the signal expires quickly so a transient upstream
|
|
262
|
+
# incident does not permanently drain an account.
|
|
263
|
+
_HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS = 300.0
|
|
264
|
+
_HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD = 3
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
async def _record_http_bridge_account_timeout_signal(
|
|
268
|
+
service: Any,
|
|
269
|
+
session: "_HTTPBridgeSession",
|
|
270
|
+
*,
|
|
271
|
+
detail: str = _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
|
|
272
|
+
) -> None:
|
|
273
|
+
"""Drain an account after repeated eventless upstream failures.
|
|
274
|
+
|
|
275
|
+
This is deliberately separate from the per-session retry circuit. A
|
|
276
|
+
timeout or abrupt eventless transport drop cannot be replayed safely for a
|
|
277
|
+
continuity-bound turn, but three independent eventless failures are enough
|
|
278
|
+
evidence to keep *new* turns away from that account until its normal
|
|
279
|
+
health probe succeeds.
|
|
280
|
+
"""
|
|
281
|
+
|
|
282
|
+
account_id = session.account.id
|
|
283
|
+
now = clock_for(service).monotonic()
|
|
284
|
+
async with service._http_bridge_account_timeout_lock:
|
|
285
|
+
failures = service._http_bridge_account_timeout_failures.setdefault(account_id, [])
|
|
286
|
+
failures[:] = [
|
|
287
|
+
timestamp for timestamp in failures if now - timestamp < _HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS
|
|
288
|
+
]
|
|
289
|
+
failures.append(now)
|
|
290
|
+
if len(failures) < _HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD:
|
|
291
|
+
return
|
|
292
|
+
# Start a fresh evidence window after applying one health penalty. A
|
|
293
|
+
# continuously failing account should be re-evaluated by normal
|
|
294
|
+
# health-tier logic, not receive an unbounded error-count increase from
|
|
295
|
+
# every pending request on one broken socket.
|
|
296
|
+
failures.clear()
|
|
297
|
+
|
|
298
|
+
try:
|
|
299
|
+
# Health-tier draining starts at two transient errors. Apply exactly
|
|
300
|
+
# that minimum penalty so one threshold event actually removes the
|
|
301
|
+
# account from normal routing without over-counting the incident.
|
|
302
|
+
await service._load_balancer.record_errors(session.account, 2)
|
|
303
|
+
except Exception:
|
|
304
|
+
logger.warning(
|
|
305
|
+
"Failed to record repeated HTTP bridge account timeout account_id=%s",
|
|
306
|
+
account_id,
|
|
307
|
+
exc_info=True,
|
|
308
|
+
)
|
|
309
|
+
else:
|
|
310
|
+
logger.warning(
|
|
311
|
+
"HTTP bridge account temporarily drained after repeated eventless upstream failures "
|
|
312
|
+
"account_id=%s detail=%s threshold=%s window_seconds=%.0f",
|
|
313
|
+
account_id,
|
|
314
|
+
detail,
|
|
315
|
+
_HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD,
|
|
316
|
+
_HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS,
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
async def _update_http_bridge_operation_state(
|
|
321
|
+
service: Any,
|
|
322
|
+
session: "_HTTPBridgeSession",
|
|
323
|
+
request_state: Any,
|
|
324
|
+
*,
|
|
325
|
+
state: str,
|
|
326
|
+
response_id: str | None = None,
|
|
327
|
+
) -> None:
|
|
328
|
+
"""Persist operation outcome without allowing journaling to break streaming."""
|
|
329
|
+
operation_id = getattr(request_state, "operation_id", None)
|
|
330
|
+
session_id = getattr(session, "durable_session_id", None)
|
|
331
|
+
owner_epoch = getattr(session, "durable_owner_epoch", None)
|
|
332
|
+
update_operation = getattr(getattr(service, "_durable_bridge", None), "update_operation", None)
|
|
333
|
+
if not operation_id or session_id is None or owner_epoch is None or not callable(update_operation):
|
|
334
|
+
return
|
|
335
|
+
try:
|
|
336
|
+
marked = await update_operation(
|
|
337
|
+
operation_id=operation_id,
|
|
338
|
+
session_id=session_id,
|
|
339
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
340
|
+
owner_epoch=owner_epoch,
|
|
341
|
+
state=state,
|
|
342
|
+
response_id=response_id,
|
|
343
|
+
)
|
|
344
|
+
if marked and response_id is not None:
|
|
345
|
+
request_state.operation_persisted_response_id = response_id
|
|
346
|
+
if not marked:
|
|
347
|
+
logger.info(
|
|
348
|
+
"HTTP bridge operation outcome owner fence rejected operation_id=%s state=%s",
|
|
349
|
+
operation_id,
|
|
350
|
+
state,
|
|
351
|
+
)
|
|
352
|
+
except Exception:
|
|
353
|
+
logger.warning(
|
|
354
|
+
"Failed to persist HTTP bridge operation outcome operation_id=%s state=%s",
|
|
355
|
+
operation_id,
|
|
356
|
+
state,
|
|
357
|
+
exc_info=True,
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _http_bridge_operation_state_for_event(event_type: str | None) -> str | None:
|
|
362
|
+
return {
|
|
363
|
+
"response.created": "acknowledged",
|
|
364
|
+
"response.completed": "completed",
|
|
365
|
+
"response.incomplete": "incomplete",
|
|
366
|
+
"response.failed": "failed",
|
|
367
|
+
"error": "failed",
|
|
368
|
+
}.get(event_type)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
async def _persist_http_bridge_operation_event(
|
|
372
|
+
service: Any,
|
|
373
|
+
session: "_HTTPBridgeSession",
|
|
374
|
+
request_state: Any,
|
|
375
|
+
event_block: str,
|
|
376
|
+
*,
|
|
377
|
+
terminal: bool = False,
|
|
378
|
+
terminal_state: str | None = None,
|
|
379
|
+
terminal_event_queue: Any | None = None,
|
|
380
|
+
terminal_delivery_scope: _HTTPBridgeCompletedDeliveryScope | None = None,
|
|
381
|
+
terminal_append_barrier: Callable[[], Awaitable[None]] | None = None,
|
|
382
|
+
terminal_delivery_barrier: Callable[[], Awaitable[None]] | None = None,
|
|
383
|
+
) -> bool:
|
|
384
|
+
"""Spool one downstream-visible SSE block for reconnect replay.
|
|
385
|
+
|
|
386
|
+
Return whether terminal failure handling already queued the block.
|
|
387
|
+
"""
|
|
388
|
+
operation_id = getattr(request_state, "operation_id", None)
|
|
389
|
+
session_id = getattr(session, "durable_session_id", None)
|
|
390
|
+
owner_epoch = getattr(session, "durable_owner_epoch", None)
|
|
391
|
+
batcher_enqueue = getattr(getattr(service, "_http_bridge_operation_event_batcher", None), "enqueue", None)
|
|
392
|
+
append_event = getattr(getattr(service, "_durable_bridge", None), "append_operation_event", None)
|
|
393
|
+
if not operation_id or session_id is None or owner_epoch is None:
|
|
394
|
+
return False
|
|
395
|
+
append_barrier_task: asyncio.Task[None] | None = None
|
|
396
|
+
delivery_barrier_task: asyncio.Task[None] | None = None
|
|
397
|
+
terminal_enqueued = False
|
|
398
|
+
deferred_cancellation: asyncio.CancelledError | None = None
|
|
399
|
+
scheduler = scheduler_for(service)
|
|
400
|
+
|
|
401
|
+
async def release_terminal_append_barrier() -> asyncio.CancelledError | None:
|
|
402
|
+
nonlocal append_barrier_task
|
|
403
|
+
if terminal_append_barrier is None:
|
|
404
|
+
return None
|
|
405
|
+
if append_barrier_task is None:
|
|
406
|
+
|
|
407
|
+
async def await_append_barrier() -> None:
|
|
408
|
+
await terminal_append_barrier()
|
|
409
|
+
|
|
410
|
+
append_barrier_task = scheduler.create_task(
|
|
411
|
+
await_append_barrier(),
|
|
412
|
+
name=f"http-bridge-terminal-append-barrier-{operation_id}",
|
|
413
|
+
)
|
|
414
|
+
_, cancellation = await _await_task_deferring_cancellation(append_barrier_task)
|
|
415
|
+
return cancellation
|
|
416
|
+
|
|
417
|
+
async def release_terminal_delivery_barrier() -> asyncio.CancelledError | None:
|
|
418
|
+
nonlocal delivery_barrier_task
|
|
419
|
+
if terminal_delivery_barrier is None:
|
|
420
|
+
return None
|
|
421
|
+
if delivery_barrier_task is None:
|
|
422
|
+
|
|
423
|
+
async def await_delivery_barrier() -> None:
|
|
424
|
+
await terminal_delivery_barrier()
|
|
425
|
+
|
|
426
|
+
delivery_barrier_task = scheduler.create_task(
|
|
427
|
+
await_delivery_barrier(),
|
|
428
|
+
name=f"http-bridge-terminal-delivery-barrier-{operation_id}",
|
|
429
|
+
)
|
|
430
|
+
_, cancellation = await _await_task_deferring_cancellation(delivery_barrier_task)
|
|
431
|
+
return cancellation
|
|
432
|
+
|
|
433
|
+
async def enqueue_terminal_delivery() -> bool:
|
|
434
|
+
if terminal_event_queue is None:
|
|
435
|
+
return False
|
|
436
|
+
await terminal_event_queue.put(event_block)
|
|
437
|
+
await terminal_event_queue.put(None)
|
|
438
|
+
if terminal_delivery_scope is not None:
|
|
439
|
+
async with session.pending_lock:
|
|
440
|
+
terminal_delivery_scope.terminal_enqueued = True
|
|
441
|
+
return True
|
|
442
|
+
|
|
443
|
+
async def enqueue_terminal_delivery_deferring_cancellation() -> tuple[bool, asyncio.CancelledError | None]:
|
|
444
|
+
delivery_task = scheduler.create_task(
|
|
445
|
+
enqueue_terminal_delivery(),
|
|
446
|
+
name=f"http-bridge-terminal-delivery-{operation_id}",
|
|
447
|
+
)
|
|
448
|
+
return await _await_task_deferring_cancellation(delivery_task)
|
|
449
|
+
|
|
450
|
+
try:
|
|
451
|
+
batcher = getattr(service, "_http_bridge_operation_event_batcher", None)
|
|
452
|
+
append_terminal_batch = getattr(batcher, "append_terminal_event", None)
|
|
453
|
+
if terminal and terminal_state is not None and callable(append_terminal_batch):
|
|
454
|
+
instance_id = _service_get_settings().http_responses_session_bridge_instance_id
|
|
455
|
+
expected_response_ids = tuple(
|
|
456
|
+
dict.fromkeys(
|
|
457
|
+
response_identity
|
|
458
|
+
for response_identity in (
|
|
459
|
+
request_state.response_id,
|
|
460
|
+
getattr(request_state, "operation_persisted_response_id", None),
|
|
461
|
+
request_state.replay_downstream_response_id,
|
|
462
|
+
)
|
|
463
|
+
if response_identity is not None
|
|
464
|
+
)
|
|
465
|
+
)
|
|
466
|
+
expected_response_id = expected_response_ids[0] if expected_response_ids else None
|
|
467
|
+
alternate_expected_response_id = expected_response_ids[1] if len(expected_response_ids) > 1 else None
|
|
468
|
+
response_id = _websocket_downstream_response_id(request_state)
|
|
469
|
+
|
|
470
|
+
async def append_terminal_batch_capturing_error() -> tuple[Any | None, Exception | None]:
|
|
471
|
+
try:
|
|
472
|
+
return (
|
|
473
|
+
await append_terminal_batch(
|
|
474
|
+
operation_id=operation_id,
|
|
475
|
+
session_id=session_id,
|
|
476
|
+
instance_id=instance_id,
|
|
477
|
+
owner_epoch=owner_epoch,
|
|
478
|
+
event_text=event_block,
|
|
479
|
+
max_bytes=int(
|
|
480
|
+
getattr(
|
|
481
|
+
_service_get_settings(),
|
|
482
|
+
"http_responses_session_bridge_operation_event_spool_max_bytes",
|
|
483
|
+
2 * 1024 * 1024,
|
|
484
|
+
)
|
|
485
|
+
),
|
|
486
|
+
state=terminal_state,
|
|
487
|
+
response_id=response_id,
|
|
488
|
+
),
|
|
489
|
+
None,
|
|
490
|
+
)
|
|
491
|
+
except Exception as exc:
|
|
492
|
+
# Return the optional-spool failure as data so the canonical
|
|
493
|
+
# defer helper can also return a caller-cancellation marker.
|
|
494
|
+
return None, exc
|
|
495
|
+
|
|
496
|
+
append_task = scheduler.create_task(
|
|
497
|
+
append_terminal_batch_capturing_error(),
|
|
498
|
+
name=f"http-bridge-terminal-append-{operation_id}",
|
|
499
|
+
)
|
|
500
|
+
# Counted grouped siblings wait on this barrier; release it even when
|
|
501
|
+
# append raises so gather(..., return_exceptions=True) cannot strand them.
|
|
502
|
+
try:
|
|
503
|
+
(append_result, append_error), deferred_cancellation = await _await_task_deferring_cancellation(
|
|
504
|
+
append_task
|
|
505
|
+
)
|
|
506
|
+
finally:
|
|
507
|
+
barrier_cancellation = await release_terminal_append_barrier()
|
|
508
|
+
deferred_cancellation = deferred_cancellation or barrier_cancellation
|
|
509
|
+
if append_error is not None:
|
|
510
|
+
raise append_error
|
|
511
|
+
persisted = bool(append_result)
|
|
512
|
+
if not persisted:
|
|
513
|
+
logger.info("HTTP bridge terminal event spool became incomplete operation_id=%s", operation_id)
|
|
514
|
+
settlement_required = bool(getattr(append_result, "settlement_required", False))
|
|
515
|
+
if settlement_required:
|
|
516
|
+
terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation()
|
|
517
|
+
deferred_cancellation = deferred_cancellation or delivery_cancellation
|
|
518
|
+
if terminal_delivery_barrier is not None:
|
|
519
|
+
if not terminal_enqueued:
|
|
520
|
+
terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation()
|
|
521
|
+
deferred_cancellation = deferred_cancellation or delivery_cancellation
|
|
522
|
+
barrier_cancellation = await release_terminal_delivery_barrier()
|
|
523
|
+
deferred_cancellation = deferred_cancellation or barrier_cancellation
|
|
524
|
+
if settlement_required:
|
|
525
|
+
settle_terminal_batch = getattr(batcher, "settle_terminal_event", None)
|
|
526
|
+
|
|
527
|
+
async def settle_terminal_append_failure() -> None:
|
|
528
|
+
if callable(settle_terminal_batch):
|
|
529
|
+
await settle_terminal_batch(
|
|
530
|
+
operation_id=operation_id,
|
|
531
|
+
session_id=session_id,
|
|
532
|
+
instance_id=instance_id,
|
|
533
|
+
owner_epoch=owner_epoch,
|
|
534
|
+
state=terminal_state,
|
|
535
|
+
expected_response_id=expected_response_id,
|
|
536
|
+
alternate_expected_response_id=alternate_expected_response_id,
|
|
537
|
+
response_id=response_id,
|
|
538
|
+
)
|
|
539
|
+
else:
|
|
540
|
+
await _update_http_bridge_operation_state(
|
|
541
|
+
service,
|
|
542
|
+
session,
|
|
543
|
+
request_state,
|
|
544
|
+
state=terminal_state,
|
|
545
|
+
response_id=response_id,
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
settlement_task = scheduler.create_task(
|
|
549
|
+
settle_terminal_append_failure(),
|
|
550
|
+
name=f"http-bridge-terminal-settlement-{operation_id}",
|
|
551
|
+
)
|
|
552
|
+
_, settlement_cancellation = await _await_task_deferring_cancellation(settlement_task)
|
|
553
|
+
deferred_cancellation = deferred_cancellation or settlement_cancellation
|
|
554
|
+
if deferred_cancellation is not None:
|
|
555
|
+
if not terminal_enqueued:
|
|
556
|
+
await enqueue_terminal_delivery_deferring_cancellation()
|
|
557
|
+
raise deferred_cancellation
|
|
558
|
+
return terminal_enqueued
|
|
559
|
+
if callable(batcher_enqueue):
|
|
560
|
+
await batcher_enqueue(
|
|
561
|
+
operation_id=operation_id,
|
|
562
|
+
session_id=session_id,
|
|
563
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
564
|
+
owner_epoch=owner_epoch,
|
|
565
|
+
event_text=event_block,
|
|
566
|
+
terminal=terminal,
|
|
567
|
+
)
|
|
568
|
+
return False
|
|
569
|
+
if not callable(append_event):
|
|
570
|
+
return False
|
|
571
|
+
persisted = await append_event(
|
|
572
|
+
operation_id=operation_id,
|
|
573
|
+
session_id=session_id,
|
|
574
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
575
|
+
owner_epoch=owner_epoch,
|
|
576
|
+
event_text=event_block,
|
|
577
|
+
max_bytes=int(
|
|
578
|
+
getattr(
|
|
579
|
+
_service_get_settings(),
|
|
580
|
+
"http_responses_session_bridge_operation_event_spool_max_bytes",
|
|
581
|
+
2 * 1024 * 1024,
|
|
582
|
+
)
|
|
583
|
+
),
|
|
584
|
+
)
|
|
585
|
+
if not persisted:
|
|
586
|
+
logger.info("HTTP bridge operation event spool became incomplete operation_id=%s", operation_id)
|
|
587
|
+
if terminal and terminal_state is not None:
|
|
588
|
+
await _update_http_bridge_operation_state(
|
|
589
|
+
service,
|
|
590
|
+
session,
|
|
591
|
+
request_state,
|
|
592
|
+
state=terminal_state,
|
|
593
|
+
response_id=_websocket_downstream_response_id(request_state),
|
|
594
|
+
)
|
|
595
|
+
return False
|
|
596
|
+
except Exception:
|
|
597
|
+
# The upstream result is still delivered. A reconnect can only replay
|
|
598
|
+
# when every event was durably persisted, so never fail a live stream
|
|
599
|
+
# because the optional spool is unavailable.
|
|
600
|
+
logger.warning("Failed to persist HTTP bridge operation event operation_id=%s", operation_id, exc_info=True)
|
|
601
|
+
barrier_cancellation = await release_terminal_append_barrier()
|
|
602
|
+
deferred_cancellation = deferred_cancellation or barrier_cancellation
|
|
603
|
+
if terminal_event_queue is not None and terminal and not terminal_enqueued:
|
|
604
|
+
try:
|
|
605
|
+
terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation()
|
|
606
|
+
deferred_cancellation = deferred_cancellation or delivery_cancellation
|
|
607
|
+
except Exception:
|
|
608
|
+
logger.debug(
|
|
609
|
+
"Failed to enqueue HTTP bridge terminal after spool error operation_id=%s",
|
|
610
|
+
operation_id,
|
|
611
|
+
exc_info=True,
|
|
612
|
+
)
|
|
613
|
+
barrier_cancellation = await release_terminal_delivery_barrier()
|
|
614
|
+
deferred_cancellation = deferred_cancellation or barrier_cancellation
|
|
615
|
+
if deferred_cancellation is not None:
|
|
616
|
+
raise deferred_cancellation
|
|
617
|
+
return terminal_enqueued
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
async def _wait_for_http_bridge_recovery_settlement_retry(
|
|
621
|
+
service: Any,
|
|
622
|
+
*,
|
|
623
|
+
session_id: str,
|
|
624
|
+
owner_epoch: int,
|
|
625
|
+
api_key_id: str | None,
|
|
626
|
+
delay_seconds: float,
|
|
627
|
+
) -> None:
|
|
628
|
+
remaining = max(0.0, delay_seconds)
|
|
629
|
+
while remaining > 0:
|
|
630
|
+
await scheduler_for(service).sleep(
|
|
631
|
+
min(remaining, _HTTP_BRIDGE_RECOVERY_SETTLEMENT_LEASE_REFRESH_INTERVAL_SECONDS)
|
|
632
|
+
)
|
|
633
|
+
remaining -= _HTTP_BRIDGE_RECOVERY_SETTLEMENT_LEASE_REFRESH_INTERVAL_SECONDS
|
|
634
|
+
try:
|
|
635
|
+
await service._durable_bridge.renew_live_session(
|
|
636
|
+
session_id=session_id,
|
|
637
|
+
api_key_id=api_key_id,
|
|
638
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
639
|
+
owner_epoch=owner_epoch,
|
|
640
|
+
lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(),
|
|
641
|
+
)
|
|
642
|
+
except Exception:
|
|
643
|
+
logger.debug("Failed to refresh HTTP bridge lease during settlement backoff", exc_info=True)
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
async def _retry_http_bridge_recovery_settlement(
|
|
647
|
+
service: Any,
|
|
648
|
+
session: Any,
|
|
649
|
+
*,
|
|
650
|
+
session_id: str,
|
|
651
|
+
api_key_id: str | None,
|
|
652
|
+
instance_id: str,
|
|
653
|
+
owner_epoch: int,
|
|
654
|
+
request_fingerprint: str,
|
|
655
|
+
response_id: str | None,
|
|
656
|
+
release_origin_lease: bool,
|
|
657
|
+
) -> None:
|
|
658
|
+
"""Keep a response-observed journal row fenced until durable settlement succeeds."""
|
|
659
|
+
|
|
660
|
+
for delay_seconds in _HTTP_BRIDGE_RECOVERY_SETTLEMENT_RETRY_DELAYS:
|
|
661
|
+
await _wait_for_http_bridge_recovery_settlement_retry(
|
|
662
|
+
service,
|
|
663
|
+
session_id=session_id,
|
|
664
|
+
owner_epoch=owner_epoch,
|
|
665
|
+
api_key_id=api_key_id,
|
|
666
|
+
delay_seconds=delay_seconds,
|
|
667
|
+
)
|
|
668
|
+
try:
|
|
669
|
+
marked = await service._durable_bridge.mark_recovery_attempt_replayed(
|
|
670
|
+
session_id=session_id,
|
|
671
|
+
api_key_id=api_key_id,
|
|
672
|
+
instance_id=instance_id,
|
|
673
|
+
owner_epoch=owner_epoch,
|
|
674
|
+
request_fingerprint=request_fingerprint,
|
|
675
|
+
response_id=response_id,
|
|
676
|
+
)
|
|
677
|
+
if marked and (release_origin_lease or getattr(session, "closed", False)):
|
|
678
|
+
try:
|
|
679
|
+
await service._durable_bridge.release_live_session(
|
|
680
|
+
session_id=session_id,
|
|
681
|
+
instance_id=instance_id,
|
|
682
|
+
owner_epoch=owner_epoch,
|
|
683
|
+
draining=False,
|
|
684
|
+
)
|
|
685
|
+
except Exception:
|
|
686
|
+
logger.debug("Failed to release HTTP bridge recovery origin lease", exc_info=True)
|
|
687
|
+
if marked:
|
|
688
|
+
return
|
|
689
|
+
logger.warning("HTTP bridge recovery settlement owner fence rejected; retrying")
|
|
690
|
+
except Exception:
|
|
691
|
+
continue
|
|
692
|
+
logger.error(
|
|
693
|
+
"HTTP bridge recovery settlement retry budget exhausted session_id=%s fingerprint=%s",
|
|
694
|
+
_hash_identifier(session_id),
|
|
695
|
+
_hash_identifier(request_fingerprint),
|
|
696
|
+
)
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
def _schedule_http_bridge_recovery_settlement_retry(
|
|
700
|
+
service: Any,
|
|
701
|
+
session: Any,
|
|
702
|
+
**kwargs: Any,
|
|
703
|
+
) -> None:
|
|
704
|
+
_schedule_http_bridge_background_cleanup(
|
|
705
|
+
service,
|
|
706
|
+
_retry_http_bridge_recovery_settlement(service, session, **kwargs),
|
|
707
|
+
name=f"http-bridge-recovery-settlement-{_hash_identifier(kwargs['request_fingerprint'])}",
|
|
708
|
+
error_message="HTTP bridge recovery settlement retry failed",
|
|
709
|
+
attribute=("_http_bridge_recovery_session_id", kwargs["session_id"]),
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
async def _retry_denied_http_bridge_anchor_clear(
|
|
714
|
+
service: Any,
|
|
715
|
+
session: Any,
|
|
716
|
+
*,
|
|
717
|
+
session_id: str | None,
|
|
718
|
+
api_key_id: str | None,
|
|
719
|
+
instance_id: str,
|
|
720
|
+
owner_epoch: int | None,
|
|
721
|
+
response_id: str,
|
|
722
|
+
durable_cleared: bool = False,
|
|
723
|
+
) -> None:
|
|
724
|
+
"""Retry a transient durable clear under the original owner fence."""
|
|
725
|
+
if session_id is None or owner_epoch is None:
|
|
726
|
+
await _retry_denied_http_bridge_anchor_local_cleanup(
|
|
727
|
+
service,
|
|
728
|
+
session,
|
|
729
|
+
response_id=response_id,
|
|
730
|
+
owner_key=f"local:{id(session)}",
|
|
731
|
+
)
|
|
732
|
+
return
|
|
733
|
+
for delay_seconds in _HTTP_BRIDGE_DENIED_ANCHOR_CLEAR_RETRY_DELAYS:
|
|
734
|
+
if session.durable_session_id != session_id or session.durable_owner_epoch != owner_epoch:
|
|
735
|
+
return
|
|
736
|
+
await _wait_for_http_bridge_recovery_settlement_retry(
|
|
737
|
+
service,
|
|
738
|
+
session_id=session_id,
|
|
739
|
+
owner_epoch=owner_epoch,
|
|
740
|
+
api_key_id=api_key_id,
|
|
741
|
+
delay_seconds=delay_seconds,
|
|
742
|
+
)
|
|
743
|
+
if not durable_cleared:
|
|
744
|
+
try:
|
|
745
|
+
cleared = await service._durable_bridge.clear_live_session_response_anchor_if_matches(
|
|
746
|
+
session_id=session_id,
|
|
747
|
+
api_key_id=api_key_id,
|
|
748
|
+
instance_id=instance_id,
|
|
749
|
+
owner_epoch=owner_epoch,
|
|
750
|
+
response_id=response_id,
|
|
751
|
+
)
|
|
752
|
+
except asyncio.CancelledError:
|
|
753
|
+
raise
|
|
754
|
+
except Exception:
|
|
755
|
+
logger.warning("Retrying denied HTTP bridge response-anchor clear after durable failure", exc_info=True)
|
|
756
|
+
continue
|
|
757
|
+
if cleared is None:
|
|
758
|
+
# A clean no-match means the captured owner fence or denied
|
|
759
|
+
# response is already gone. It is a terminal bookkeeping
|
|
760
|
+
# outcome, not a transient durable failure; keep the local
|
|
761
|
+
# alias and denial fence because ownership may have advanced,
|
|
762
|
+
# but do not spend the remaining retry budget on it.
|
|
763
|
+
return
|
|
764
|
+
durable_cleared = True
|
|
765
|
+
# Owner rebinding can happen while the lease-renewal backoff is
|
|
766
|
+
# sleeping. Serialize the final ownership check with rebinders before
|
|
767
|
+
# touching the alias or the process-local denial fence.
|
|
768
|
+
async with session.lifecycle_lock:
|
|
769
|
+
if session.durable_session_id != session_id or session.durable_owner_epoch != owner_epoch:
|
|
770
|
+
return
|
|
771
|
+
try:
|
|
772
|
+
unregister_succeeded = await service._unregister_http_bridge_previous_response_id(
|
|
773
|
+
session,
|
|
774
|
+
response_id,
|
|
775
|
+
expected_durable_session_id=session_id,
|
|
776
|
+
expected_durable_owner_epoch=owner_epoch,
|
|
777
|
+
)
|
|
778
|
+
except asyncio.CancelledError:
|
|
779
|
+
raise
|
|
780
|
+
except Exception:
|
|
781
|
+
logger.warning(
|
|
782
|
+
"Retrying denied HTTP bridge response-alias unregister after cleanup failure",
|
|
783
|
+
exc_info=True,
|
|
784
|
+
)
|
|
785
|
+
continue
|
|
786
|
+
if unregister_succeeded is False:
|
|
787
|
+
return
|
|
788
|
+
if not _forget_http_bridge_denied_anchor_fence(
|
|
789
|
+
service,
|
|
790
|
+
response_id,
|
|
791
|
+
owner_key=session_id,
|
|
792
|
+
owner_epoch=owner_epoch,
|
|
793
|
+
):
|
|
794
|
+
return
|
|
795
|
+
session.denied_proxy_injected_anchor_ids.discard(response_id)
|
|
796
|
+
session.denied_proxy_injected_anchor_cleanup_pending.discard(response_id)
|
|
797
|
+
session.denied_proxy_injected_anchor_generation += 1
|
|
798
|
+
return
|
|
799
|
+
logger.error(
|
|
800
|
+
"Denied HTTP bridge response-anchor clear retry budget exhausted session_id=%s response_id=%s",
|
|
801
|
+
_hash_identifier(session_id),
|
|
802
|
+
_hash_identifier(response_id),
|
|
803
|
+
)
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
async def _retry_denied_http_bridge_anchor_local_cleanup(
|
|
807
|
+
service: Any,
|
|
808
|
+
session: Any,
|
|
809
|
+
*,
|
|
810
|
+
response_id: str,
|
|
811
|
+
owner_key: str,
|
|
812
|
+
) -> None:
|
|
813
|
+
"""Retry process-local alias cleanup for a session without a durable owner."""
|
|
814
|
+
for delay_seconds in _HTTP_BRIDGE_DENIED_ANCHOR_CLEAR_RETRY_DELAYS:
|
|
815
|
+
if delay_seconds:
|
|
816
|
+
await scheduler_for(service).sleep(delay_seconds)
|
|
817
|
+
async with session.lifecycle_lock:
|
|
818
|
+
if session.durable_session_id is not None or session.durable_owner_epoch is not None:
|
|
819
|
+
return
|
|
820
|
+
entry = _http_bridge_denied_anchor_fence_entry(service, response_id)
|
|
821
|
+
try:
|
|
822
|
+
unregister_succeeded = await service._unregister_http_bridge_previous_response_id(
|
|
823
|
+
session,
|
|
824
|
+
response_id,
|
|
825
|
+
)
|
|
826
|
+
except asyncio.CancelledError:
|
|
827
|
+
raise
|
|
828
|
+
except Exception:
|
|
829
|
+
logger.warning(
|
|
830
|
+
"Retrying local denied HTTP bridge response-alias unregister after cleanup failure",
|
|
831
|
+
exc_info=True,
|
|
832
|
+
)
|
|
833
|
+
continue
|
|
834
|
+
if unregister_succeeded is False:
|
|
835
|
+
continue
|
|
836
|
+
# The local session can lose its durable identity after a
|
|
837
|
+
# successor has already claimed the same response id. Remove
|
|
838
|
+
# this session's alias even when the process-local fence now
|
|
839
|
+
# belongs to that durable successor; only forget the fence when
|
|
840
|
+
# this local owner still owns it.
|
|
841
|
+
if entry is not None and entry.owner_key == owner_key:
|
|
842
|
+
_forget_http_bridge_denied_anchor_fence(
|
|
843
|
+
service,
|
|
844
|
+
response_id,
|
|
845
|
+
owner_key=owner_key,
|
|
846
|
+
owner_epoch=None,
|
|
847
|
+
)
|
|
848
|
+
session.denied_proxy_injected_anchor_ids.discard(response_id)
|
|
849
|
+
session.denied_proxy_injected_anchor_cleanup_pending.discard(response_id)
|
|
850
|
+
session.denied_proxy_injected_anchor_generation += 1
|
|
851
|
+
return
|
|
852
|
+
logger.error(
|
|
853
|
+
"Denied HTTP bridge local response-anchor cleanup retry budget exhausted response_id=%s",
|
|
854
|
+
_hash_identifier(response_id),
|
|
855
|
+
)
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
def _schedule_denied_http_bridge_anchor_clear_retry(
|
|
859
|
+
service: Any,
|
|
860
|
+
session: Any,
|
|
861
|
+
*,
|
|
862
|
+
response_id: str,
|
|
863
|
+
session_id: str | None = None,
|
|
864
|
+
api_key_id: str | None = None,
|
|
865
|
+
instance_id: str | None = None,
|
|
866
|
+
owner_epoch: int | None = None,
|
|
867
|
+
durable_cleared: bool = False,
|
|
868
|
+
) -> None:
|
|
869
|
+
session_id = session.durable_session_id if session_id is None else session_id
|
|
870
|
+
owner_epoch = session.durable_owner_epoch if owner_epoch is None else owner_epoch
|
|
871
|
+
api_key_id = session.key.api_key_id if api_key_id is None else api_key_id
|
|
872
|
+
instance_id = (
|
|
873
|
+
_service_get_settings().http_responses_session_bridge_instance_id if instance_id is None else instance_id
|
|
874
|
+
)
|
|
875
|
+
if instance_id is None:
|
|
876
|
+
return
|
|
877
|
+
_schedule_http_bridge_background_cleanup(
|
|
878
|
+
service,
|
|
879
|
+
_retry_denied_http_bridge_anchor_clear(
|
|
880
|
+
service,
|
|
881
|
+
session,
|
|
882
|
+
session_id=session_id,
|
|
883
|
+
api_key_id=api_key_id,
|
|
884
|
+
instance_id=instance_id,
|
|
885
|
+
owner_epoch=owner_epoch,
|
|
886
|
+
response_id=response_id,
|
|
887
|
+
durable_cleared=durable_cleared,
|
|
888
|
+
),
|
|
889
|
+
name=f"http-bridge-denied-anchor-clear-{_hash_identifier(response_id)}",
|
|
890
|
+
error_message="HTTP bridge denied-anchor clear retry failed",
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
T = TypeVar("T")
|
|
895
|
+
_TEXT_DELTA_EVENT_TYPES = frozenset({"response.output_text.delta", "response.refusal.delta"})
|
|
896
|
+
_UNSUPPORTED_DURABLE_TOOL_CALL_ITEM_TYPES = frozenset(
|
|
897
|
+
{
|
|
898
|
+
"computer_call",
|
|
899
|
+
"mcp_approval_request",
|
|
900
|
+
}
|
|
901
|
+
)
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
def _record_http_bridge_tool_call_lifecycle(
|
|
905
|
+
request_state: _WebSocketRequestState,
|
|
906
|
+
*,
|
|
907
|
+
event_type: str | None,
|
|
908
|
+
payload: dict[str, JsonValue] | None,
|
|
909
|
+
) -> None:
|
|
910
|
+
if event_type not in {"response.output_item.added", "response.output_item.done"}:
|
|
911
|
+
return
|
|
912
|
+
item = payload.get("item") if isinstance(payload, dict) else None
|
|
913
|
+
if not isinstance(item, dict):
|
|
914
|
+
request_state.tool_call_manifest_invalid = True
|
|
915
|
+
return
|
|
916
|
+
item_type = item.get("type")
|
|
917
|
+
if not isinstance(item_type, str):
|
|
918
|
+
request_state.tool_call_manifest_invalid = True
|
|
919
|
+
return
|
|
920
|
+
if item_type in _UNSUPPORTED_DURABLE_TOOL_CALL_ITEM_TYPES:
|
|
921
|
+
# These calls require client-provided continuation state but are not
|
|
922
|
+
# representable by the direct function/custom/apply-patch replay proof.
|
|
923
|
+
# Persisting only a parallel supported call would make a partial suffix
|
|
924
|
+
# look complete, so keep the whole durable manifest unknown.
|
|
925
|
+
request_state.tool_call_manifest_invalid = True
|
|
926
|
+
return
|
|
927
|
+
if item_type not in _PENDING_TOOL_CALL_ITEM_TYPES:
|
|
928
|
+
return
|
|
929
|
+
call_id = item.get("call_id")
|
|
930
|
+
if not isinstance(call_id, str) or not call_id:
|
|
931
|
+
request_state.tool_call_manifest_invalid = True
|
|
932
|
+
return
|
|
933
|
+
target = (
|
|
934
|
+
request_state.added_tool_call_types
|
|
935
|
+
if event_type == "response.output_item.added"
|
|
936
|
+
else request_state.pending_tool_call_types
|
|
937
|
+
)
|
|
938
|
+
existing = target.get(call_id)
|
|
939
|
+
if existing is not None:
|
|
940
|
+
request_state.tool_call_manifest_invalid = True
|
|
941
|
+
return
|
|
942
|
+
target[call_id] = item_type
|
|
943
|
+
|
|
944
|
+
|
|
945
|
+
def _response_completed_tool_call_types(payload: dict[str, JsonValue] | None) -> dict[str, str] | None:
|
|
946
|
+
response = payload.get("response") if isinstance(payload, dict) else None
|
|
947
|
+
output = response.get("output") if isinstance(response, dict) else None
|
|
948
|
+
if not isinstance(output, list):
|
|
949
|
+
return None
|
|
950
|
+
result: dict[str, str] = {}
|
|
951
|
+
for item in output:
|
|
952
|
+
if not isinstance(item, dict):
|
|
953
|
+
return None
|
|
954
|
+
item_type = item.get("type")
|
|
955
|
+
if not isinstance(item_type, str):
|
|
956
|
+
return None
|
|
957
|
+
if item_type in _UNSUPPORTED_DURABLE_TOOL_CALL_ITEM_TYPES:
|
|
958
|
+
return None
|
|
959
|
+
if item_type not in _PENDING_TOOL_CALL_ITEM_TYPES:
|
|
960
|
+
continue
|
|
961
|
+
call_id = item.get("call_id")
|
|
962
|
+
if not isinstance(call_id, str) or not call_id:
|
|
963
|
+
return None
|
|
964
|
+
existing = result.get(call_id)
|
|
965
|
+
if existing is not None:
|
|
966
|
+
return None
|
|
967
|
+
result[call_id] = item_type
|
|
968
|
+
return result
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
def _durable_pending_tool_call_manifest(
|
|
972
|
+
request_state: _WebSocketRequestState,
|
|
973
|
+
payload: dict[str, JsonValue] | None,
|
|
974
|
+
) -> dict[str, str] | None:
|
|
975
|
+
terminal_calls = _response_completed_tool_call_types(payload)
|
|
976
|
+
if request_state.tool_call_manifest_invalid or terminal_calls is None:
|
|
977
|
+
return None
|
|
978
|
+
if request_state.added_tool_call_types != request_state.pending_tool_call_types:
|
|
979
|
+
return None
|
|
980
|
+
if terminal_calls and terminal_calls != request_state.pending_tool_call_types:
|
|
981
|
+
return None
|
|
982
|
+
return dict(request_state.pending_tool_call_types)
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
_SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE = "security_work_authorization_required"
|
|
986
|
+
_SECURITY_WORK_RETRY_MESSAGE = (
|
|
987
|
+
"Upstream flagged this request as possible cybersecurity work. "
|
|
988
|
+
"codex-lb is retrying on an account marked as authorized for security work."
|
|
989
|
+
)
|
|
990
|
+
|
|
991
|
+
|
|
992
|
+
def _relinquish_http_bridge_capacity_wait_ownership(
|
|
993
|
+
session: "_HTTPBridgeSession",
|
|
994
|
+
request_state: _WebSocketRequestState,
|
|
995
|
+
claimed_terminal_request_states: list[_WebSocketRequestState],
|
|
996
|
+
) -> None:
|
|
997
|
+
"""Pop a capacity-wait request from pending ownership and record the settlement claim.
|
|
998
|
+
|
|
999
|
+
The selected-model capacity branch reserves its request *in* pending
|
|
1000
|
+
ownership while it waits (a younger submit must not take its queue slot),
|
|
1001
|
+
so the terminal pop that normally records the ``"claimed"`` marker never
|
|
1002
|
+
ran for it. Once the branch gives that ownership up -- the replay was
|
|
1003
|
+
refused or failed, or the request never had a consumer -- this bookkeeping
|
|
1004
|
+
continuation is the request's sole settlement owner exactly like a popped
|
|
1005
|
+
terminal, and an abort between here and ``_finalize_terminal_settlement``
|
|
1006
|
+
must reach the shielded abort settlement (issue #1594) instead of leaking
|
|
1007
|
+
the API-key reservation, its heartbeat, and the re-claimed create gate. The
|
|
1008
|
+
caller holds ``session.pending_lock``.
|
|
1009
|
+
"""
|
|
1010
|
+
if request_state not in session.pending_requests:
|
|
1011
|
+
return
|
|
1012
|
+
session.pending_requests.remove(request_state)
|
|
1013
|
+
if _http_bridge_request_counts_against_queue(request_state):
|
|
1014
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
1015
|
+
request_state.terminal_settlement_phase = "claimed"
|
|
1016
|
+
if request_state not in claimed_terminal_request_states:
|
|
1017
|
+
claimed_terminal_request_states.append(request_state)
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
async def _wait_before_http_bridge_model_capacity_retry(
|
|
1021
|
+
request_state: _WebSocketRequestState | None,
|
|
1022
|
+
*,
|
|
1023
|
+
emit_keepalives: bool,
|
|
1024
|
+
error_message: str | None,
|
|
1025
|
+
cancel_when_detached: bool = False,
|
|
1026
|
+
signal_startup_wait: bool = True,
|
|
1027
|
+
scheduler: Scheduler = REAL_SCHEDULER,
|
|
1028
|
+
clock: Clock = REAL_CLOCK,
|
|
1029
|
+
) -> bool:
|
|
1030
|
+
"""Sleep the bounded selected-model capacity delay before a bridge replay.
|
|
1031
|
+
|
|
1032
|
+
A propagated-error stream (``emit_keepalives=False``) normally signals the
|
|
1033
|
+
pre-response startup wait so its startup probe keeps waiting instead of
|
|
1034
|
+
timing out. An accepted request already committed its response start, so
|
|
1035
|
+
the wait branch passes ``signal_startup_wait=False`` for it: re-signalling
|
|
1036
|
+
would re-arm the startup wait behind a stream that is already open (spec:
|
|
1037
|
+
"Accepted public streams are replayed without keepalives").
|
|
1038
|
+
"""
|
|
1039
|
+
if request_state is None or not is_upstream_model_capacity_error(error_message):
|
|
1040
|
+
return True
|
|
1041
|
+
|
|
1042
|
+
deadline = request_state.bridge_request_deadline
|
|
1043
|
+
if deadline is None:
|
|
1044
|
+
deadline = request_state.started_at + _http_bridge_request_budget_seconds(_service_get_settings())
|
|
1045
|
+
remaining_budget_seconds = max(0.0, deadline - clock.monotonic())
|
|
1046
|
+
if remaining_budget_seconds <= 0:
|
|
1047
|
+
return False
|
|
1048
|
+
|
|
1049
|
+
sleep_seconds = min(_ACCOUNT_SELECTION_RECOVERY_DEFAULT_SLEEP_SECONDS, remaining_budget_seconds)
|
|
1050
|
+
request_state.account_capacity_waiting = True
|
|
1051
|
+
request_state.account_capacity_wait_reason = error_message
|
|
1052
|
+
request_state.account_capacity_wait_started_at = request_state.account_capacity_wait_started_at or clock.monotonic()
|
|
1053
|
+
request_state.account_capacity_wait_retry_after_seconds = sleep_seconds
|
|
1054
|
+
request_state.account_capacity_wait_suppress_keepalive = not emit_keepalives
|
|
1055
|
+
if not emit_keepalives and signal_startup_wait:
|
|
1056
|
+
_signal_http_bridge_capacity_startup_wait(request_state)
|
|
1057
|
+
try:
|
|
1058
|
+
remaining_sleep_seconds = sleep_seconds
|
|
1059
|
+
keepalive_countdown_seconds = 0.0
|
|
1060
|
+
while remaining_sleep_seconds > 0:
|
|
1061
|
+
if cancel_when_detached and request_state.event_queue is None:
|
|
1062
|
+
return False
|
|
1063
|
+
if emit_keepalives and keepalive_countdown_seconds <= 0 and request_state.event_queue is not None:
|
|
1064
|
+
await request_state.event_queue.put(
|
|
1065
|
+
format_sse_event(
|
|
1066
|
+
_account_capacity_wait_payload(
|
|
1067
|
+
request_state,
|
|
1068
|
+
request_id=request_state.request_log_id or request_state.request_id,
|
|
1069
|
+
reason=error_message,
|
|
1070
|
+
retry_after_seconds=remaining_sleep_seconds,
|
|
1071
|
+
now=clock.monotonic(),
|
|
1072
|
+
)
|
|
1073
|
+
)
|
|
1074
|
+
)
|
|
1075
|
+
keepalive_countdown_seconds = _ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS
|
|
1076
|
+
chunk_seconds = min(
|
|
1077
|
+
remaining_sleep_seconds,
|
|
1078
|
+
(
|
|
1079
|
+
_WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS
|
|
1080
|
+
if cancel_when_detached
|
|
1081
|
+
else _ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS
|
|
1082
|
+
),
|
|
1083
|
+
)
|
|
1084
|
+
await scheduler.sleep(chunk_seconds)
|
|
1085
|
+
remaining_sleep_seconds -= chunk_seconds
|
|
1086
|
+
keepalive_countdown_seconds -= chunk_seconds
|
|
1087
|
+
return (not cancel_when_detached or request_state.event_queue is not None) and clock.monotonic() < deadline
|
|
1088
|
+
finally:
|
|
1089
|
+
request_state.account_capacity_waiting = False
|
|
1090
|
+
if emit_keepalives:
|
|
1091
|
+
request_state.account_capacity_wait_suppress_keepalive = False
|
|
1092
|
+
request_state.account_capacity_wait_reason = None
|
|
1093
|
+
request_state.account_capacity_wait_retry_after_seconds = None
|
|
1094
|
+
|
|
1095
|
+
|
|
1096
|
+
async def _release_http_bridge_model_capacity_retry_admission(
|
|
1097
|
+
request_state: _WebSocketRequestState,
|
|
1098
|
+
) -> None:
|
|
1099
|
+
"""Release shared capacity while retaining the session create gate."""
|
|
1100
|
+
if request_state.response_create_admission is not None:
|
|
1101
|
+
request_state.response_create_admission.release()
|
|
1102
|
+
request_state.response_create_admission = None
|
|
1103
|
+
request_state.response_create_admission_reacquire_required = True
|
|
1104
|
+
account_response_create_lease = request_state.account_response_create_lease
|
|
1105
|
+
account_response_create_release = request_state.account_response_create_release
|
|
1106
|
+
request_state.account_response_create_lease = None
|
|
1107
|
+
request_state.account_response_create_release = None
|
|
1108
|
+
if account_response_create_lease is not None and account_response_create_release is not None:
|
|
1109
|
+
await account_response_create_release(account_response_create_lease)
|
|
1110
|
+
|
|
1111
|
+
|
|
1112
|
+
def _signal_http_bridge_model_capacity_retry_ready(
|
|
1113
|
+
request_state: _WebSocketRequestState,
|
|
1114
|
+
*,
|
|
1115
|
+
waited_for_model_capacity_retry: bool,
|
|
1116
|
+
retried: bool,
|
|
1117
|
+
) -> None:
|
|
1118
|
+
if waited_for_model_capacity_retry and retried and request_state.propagate_http_errors:
|
|
1119
|
+
if request_state.capacity_startup_wait_event is not None:
|
|
1120
|
+
request_state.capacity_startup_wait_event.clear()
|
|
1121
|
+
if request_state.capacity_startup_ready_event is not None:
|
|
1122
|
+
request_state.capacity_startup_ready_event.set()
|
|
1123
|
+
_signal_propagated_capacity_startup_ready()
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
def _signal_http_bridge_capacity_startup_wait(request_state: _WebSocketRequestState) -> None:
|
|
1127
|
+
if request_state.capacity_startup_ready_event is not None:
|
|
1128
|
+
request_state.capacity_startup_ready_event.clear()
|
|
1129
|
+
if request_state.capacity_startup_wait_event is not None:
|
|
1130
|
+
request_state.capacity_startup_wait_event.set()
|
|
1131
|
+
_signal_propagated_capacity_startup_wait()
|
|
1132
|
+
|
|
1133
|
+
|
|
1134
|
+
def _archive_http_bridge_upstream_text(
|
|
1135
|
+
session: "_HTTPBridgeSession",
|
|
1136
|
+
text: str,
|
|
1137
|
+
request_state: "_WebSocketRequestState | None",
|
|
1138
|
+
) -> None:
|
|
1139
|
+
_archive_http_bridge_upstream_message(
|
|
1140
|
+
session,
|
|
1141
|
+
UpstreamWebSocketMessage(kind="text", text=text),
|
|
1142
|
+
request_state,
|
|
1143
|
+
)
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def _archive_http_bridge_upstream_message(
|
|
1147
|
+
session: "_HTTPBridgeSession",
|
|
1148
|
+
message: UpstreamWebSocketMessage,
|
|
1149
|
+
request_state: "_WebSocketRequestState | None",
|
|
1150
|
+
) -> None:
|
|
1151
|
+
if request_state is None or request_state.archive_request_id is None:
|
|
1152
|
+
archive_request_id = None
|
|
1153
|
+
else:
|
|
1154
|
+
archive_request_id = request_state.archive_request_id
|
|
1155
|
+
archive_received = getattr(session.upstream, "archive_received", None)
|
|
1156
|
+
if not callable(archive_received):
|
|
1157
|
+
return
|
|
1158
|
+
token = set_request_id(archive_request_id)
|
|
1159
|
+
try:
|
|
1160
|
+
archive_received(message)
|
|
1161
|
+
finally:
|
|
1162
|
+
reset_request_id(token)
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
async def _http_bridge_receive_timeout_with_eventless_deadline(
|
|
1166
|
+
session: "_HTTPBridgeSession",
|
|
1167
|
+
receive_timeout: _WebSocketReceiveTimeout | None,
|
|
1168
|
+
*,
|
|
1169
|
+
now: float,
|
|
1170
|
+
stuck_gate_retire_after_seconds: float,
|
|
1171
|
+
) -> _WebSocketReceiveTimeout | None:
|
|
1172
|
+
if session.closed:
|
|
1173
|
+
return receive_timeout
|
|
1174
|
+
async with session.pending_lock:
|
|
1175
|
+
deadlines = [
|
|
1176
|
+
deadline
|
|
1177
|
+
for request_state in session.pending_requests
|
|
1178
|
+
if (
|
|
1179
|
+
deadline := _http_bridge_eventless_precreated_deadline(
|
|
1180
|
+
request_state,
|
|
1181
|
+
stuck_gate_retire_after_seconds=stuck_gate_retire_after_seconds,
|
|
1182
|
+
)
|
|
1183
|
+
)
|
|
1184
|
+
is not None
|
|
1185
|
+
]
|
|
1186
|
+
if not deadlines:
|
|
1187
|
+
return receive_timeout
|
|
1188
|
+
eventless_timeout = _WebSocketReceiveTimeout(
|
|
1189
|
+
timeout_seconds=max(0.0, min(deadlines) - now),
|
|
1190
|
+
error_code=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
|
|
1191
|
+
error_message="Upstream did not acknowledge response.create before the client-safe deadline",
|
|
1192
|
+
fail_all_pending=True,
|
|
1193
|
+
)
|
|
1194
|
+
if receive_timeout is None or eventless_timeout.timeout_seconds <= receive_timeout.timeout_seconds:
|
|
1195
|
+
return eventless_timeout
|
|
1196
|
+
return receive_timeout
|
|
1197
|
+
|
|
1198
|
+
|
|
1199
|
+
async def _cancel_http_bridge_reader_child(
|
|
1200
|
+
task: asyncio.Task[Any] | None,
|
|
1201
|
+
*,
|
|
1202
|
+
label: str,
|
|
1203
|
+
scheduler_owner: Any,
|
|
1204
|
+
cleanup_tasks: set[asyncio.Task[None]] | None = None,
|
|
1205
|
+
) -> bool:
|
|
1206
|
+
if task is None:
|
|
1207
|
+
return True
|
|
1208
|
+
if task.done():
|
|
1209
|
+
try:
|
|
1210
|
+
task.result()
|
|
1211
|
+
except asyncio.CancelledError:
|
|
1212
|
+
pass
|
|
1213
|
+
except Exception:
|
|
1214
|
+
logger.debug("HTTP bridge reader child already failed during cleanup label=%s", label, exc_info=True)
|
|
1215
|
+
return True
|
|
1216
|
+
try:
|
|
1217
|
+
return bool(
|
|
1218
|
+
await _await_cancelled_task(
|
|
1219
|
+
task,
|
|
1220
|
+
label=label,
|
|
1221
|
+
cleanup_tasks=cleanup_tasks,
|
|
1222
|
+
scheduler=scheduler_for(scheduler_owner),
|
|
1223
|
+
)
|
|
1224
|
+
)
|
|
1225
|
+
except Exception:
|
|
1226
|
+
logger.debug("Failed to cancel HTTP bridge reader child label=%s", label, exc_info=True)
|
|
1227
|
+
return task.done()
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
async def _clear_durable_http_bridge_response_anchor(
|
|
1231
|
+
service: Any,
|
|
1232
|
+
session: "_HTTPBridgeSession",
|
|
1233
|
+
) -> None:
|
|
1234
|
+
"""Invalidate a durable proxy-injected anchor that proved eventless.
|
|
1235
|
+
|
|
1236
|
+
Runs while ``session`` still owns the durable row (before retirement
|
|
1237
|
+
releases the lease), so the fenced write lands under the session's own
|
|
1238
|
+
owner epoch instead of silently losing the fence to a released owner.
|
|
1239
|
+
"""
|
|
1240
|
+
if session.durable_session_id is None or session.durable_owner_epoch is None:
|
|
1241
|
+
return
|
|
1242
|
+
try:
|
|
1243
|
+
lookup = await service._durable_bridge.clear_live_session_response_anchor(
|
|
1244
|
+
session_id=session.durable_session_id,
|
|
1245
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
1246
|
+
owner_epoch=session.durable_owner_epoch,
|
|
1247
|
+
)
|
|
1248
|
+
except Exception:
|
|
1249
|
+
logger.warning("Failed to clear durable HTTP bridge response anchor after stuck timeout", exc_info=True)
|
|
1250
|
+
return
|
|
1251
|
+
if lookup is None or lookup.owner_epoch != session.durable_owner_epoch or lookup.latest_response_id is not None:
|
|
1252
|
+
# None means the durable row is gone entirely (e.g. purged); an
|
|
1253
|
+
# epoch or anchor mismatch means a newer owner already claimed the
|
|
1254
|
+
# session before this fenced write executed. Either way, the anchor
|
|
1255
|
+
# was never actually cleared, so do not report an invalidation that
|
|
1256
|
+
# did not happen.
|
|
1257
|
+
return
|
|
1258
|
+
_log_http_bridge_event(
|
|
1259
|
+
"durable_anchor_invalidated",
|
|
1260
|
+
session.key,
|
|
1261
|
+
account_id=session.account.id,
|
|
1262
|
+
model=session.request_model,
|
|
1263
|
+
detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
|
|
1264
|
+
cache_key_family=session.key.affinity_kind,
|
|
1265
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1266
|
+
)
|
|
1267
|
+
|
|
1268
|
+
|
|
1269
|
+
async def _abandon_durable_http_bridge_continuity(
|
|
1270
|
+
service: Any,
|
|
1271
|
+
session: "_HTTPBridgeSession",
|
|
1272
|
+
*,
|
|
1273
|
+
detail: str = "repeated_zero_event_idle_timeout",
|
|
1274
|
+
settle_circuit: bool = False,
|
|
1275
|
+
expected_continuity: object = _POISON_ANCHOR_CAPTURE_UNAVAILABLE,
|
|
1276
|
+
authorized_episode: Any = None,
|
|
1277
|
+
) -> bool:
|
|
1278
|
+
"""Clear durable continuity before retiring a repeatedly poisoned bridge.
|
|
1279
|
+
|
|
1280
|
+
``rebind_session_account(clear_continuity=True)`` is an existing fenced
|
|
1281
|
+
write that clears the durable response/turn anchor and its alias rows while
|
|
1282
|
+
this worker still owns the session. The ordinary retirement path then
|
|
1283
|
+
closes the row and removes the process-local registrations.
|
|
1284
|
+
"""
|
|
1285
|
+
if session.durable_session_id is None or session.durable_owner_epoch is None:
|
|
1286
|
+
return False
|
|
1287
|
+
# Captured BEFORE the continuity clear's await: this is the closest
|
|
1288
|
+
# snapshot to the episode the poison consult validated, and a sibling
|
|
1289
|
+
# replacing the circuit episode while the rebind is in flight must not
|
|
1290
|
+
# be mistaken for the authorizing one — the settle below clears only
|
|
1291
|
+
# the episode this abandonment invalidated.
|
|
1292
|
+
async with service._http_bridge_retry_circuit_lock:
|
|
1293
|
+
# The consulted episode outranks a registry re-read: a sibling
|
|
1294
|
+
# settle can remove the entry between the consult and this lock,
|
|
1295
|
+
# and a None capture would run the settle unfenced — free to clear
|
|
1296
|
+
# a replacement episode opened during the awaits below. The
|
|
1297
|
+
# registry read remains only for legacy callers with no consult.
|
|
1298
|
+
authorizing_state = (
|
|
1299
|
+
authorized_episode
|
|
1300
|
+
if authorized_episode is not None
|
|
1301
|
+
else service._http_bridge_retry_circuits.get(session.key)
|
|
1302
|
+
)
|
|
1303
|
+
abandonment_expected_episode = (
|
|
1304
|
+
(
|
|
1305
|
+
authorizing_state.persisted_updated_at_epoch,
|
|
1306
|
+
authorizing_state.consecutive_failures,
|
|
1307
|
+
authorizing_state.persisted_admission_generation,
|
|
1308
|
+
)
|
|
1309
|
+
if authorizing_state is not None
|
|
1310
|
+
else None
|
|
1311
|
+
)
|
|
1312
|
+
rebind_fence_kwargs: dict[str, Any] = {}
|
|
1313
|
+
if expected_continuity is not _POISON_ANCHOR_CAPTURE_UNAVAILABLE:
|
|
1314
|
+
# Fence the continuity clear on both anchors captured when the
|
|
1315
|
+
# episode was validated: a completion registering a fresh response
|
|
1316
|
+
# anchor or a turn-state write landing in between changes a column
|
|
1317
|
+
# and the fenced write matches nothing.
|
|
1318
|
+
expected_response_id, expected_turn_state = cast("tuple[str | None, str | None]", expected_continuity)
|
|
1319
|
+
rebind_fence_kwargs["expected_latest_response_id"] = expected_response_id
|
|
1320
|
+
rebind_fence_kwargs["expected_latest_turn_state"] = expected_turn_state
|
|
1321
|
+
try:
|
|
1322
|
+
cleared = await service._durable_bridge.rebind_session_account(
|
|
1323
|
+
session_id=session.durable_session_id,
|
|
1324
|
+
api_key_id=session.key.api_key_id,
|
|
1325
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
1326
|
+
owner_epoch=session.durable_owner_epoch,
|
|
1327
|
+
account_id=session.account.id,
|
|
1328
|
+
clear_continuity=True,
|
|
1329
|
+
**rebind_fence_kwargs,
|
|
1330
|
+
)
|
|
1331
|
+
except Exception:
|
|
1332
|
+
logger.warning("Failed to abandon poisoned HTTP bridge continuity", exc_info=True)
|
|
1333
|
+
return False
|
|
1334
|
+
if not cleared:
|
|
1335
|
+
logger.warning(
|
|
1336
|
+
"Durable bridge continuity clear was fenced before poisoned anchor retirement",
|
|
1337
|
+
extra={
|
|
1338
|
+
"session_id": session.durable_session_id,
|
|
1339
|
+
"account_id": session.account.id,
|
|
1340
|
+
},
|
|
1341
|
+
)
|
|
1342
|
+
return False
|
|
1343
|
+
_log_http_bridge_event(
|
|
1344
|
+
"durable_anchor_poisoned",
|
|
1345
|
+
session.key,
|
|
1346
|
+
account_id=session.account.id,
|
|
1347
|
+
model=session.request_model,
|
|
1348
|
+
detail=detail,
|
|
1349
|
+
cache_key_family=session.key.affinity_kind,
|
|
1350
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1351
|
+
)
|
|
1352
|
+
# Settle only when the requests this abandonment covers are actually
|
|
1353
|
+
# stranded. A stale-anchor rejection that still holds a verified full
|
|
1354
|
+
# resend is about to be replayed, and that replay claims the circuit
|
|
1355
|
+
# generation at dispatch (#1863); clearing the circuit under it removes the
|
|
1356
|
+
# fence it depends on, which is why `response.completed` also skips its
|
|
1357
|
+
# clear for such a replay. Settling unconditionally broke five variants of
|
|
1358
|
+
# the stale-owner replay suite, and settling for no funnel caller left the
|
|
1359
|
+
# production wedge cooling for 60s after its anchor was already gone.
|
|
1360
|
+
if not settle_circuit:
|
|
1361
|
+
return True
|
|
1362
|
+
# The circuit was opened by failures against the anchor this call just
|
|
1363
|
+
# removed, so its cooldown is now backing off a cause that no longer
|
|
1364
|
+
# exists. Leaving it running refuses requests that carry no anchor at all:
|
|
1365
|
+
# observed live as a burst of ~25 rejections logged
|
|
1366
|
+
# `reason=retry_circuit_cooldown_continuity_bound previous_response_id=None`
|
|
1367
|
+
# in the 60s after a successful clear, every one of which would have gone
|
|
1368
|
+
# upstream cleanly. A confirmed abandonment is proof the next attempt
|
|
1369
|
+
# cannot repeat that failure, which is the same evidence a completed
|
|
1370
|
+
# response carries, so settle the circuit the same way. A genuinely new
|
|
1371
|
+
# failure re-opens it at the usual threshold.
|
|
1372
|
+
# A sibling completion can race this settle after the continuity clear:
|
|
1373
|
+
# it registers a FRESH anchor and erases its own transitional tombstone,
|
|
1374
|
+
# and writing the tombstone here again would durably 404 every valid
|
|
1375
|
+
# follow-up riding that fresh anchor, with no registration left to erase
|
|
1376
|
+
# it. Re-read the continuity: an anchor that is present and different
|
|
1377
|
+
# from the one this abandonment removed is fresh evidence, and the
|
|
1378
|
+
# settle then writes a plain reset instead of the tombstone.
|
|
1379
|
+
abandonment_settled_detail: str | None = _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
1380
|
+
abandoned_anchor, abandoned_turn_state = (
|
|
1381
|
+
(expected_continuity[0], expected_continuity[1])
|
|
1382
|
+
if isinstance(expected_continuity, tuple) and len(expected_continuity) == 2
|
|
1383
|
+
else (None, None)
|
|
1384
|
+
)
|
|
1385
|
+
|
|
1386
|
+
def _continuity_is_fresh(observed: tuple[str | None, str | None] | None) -> bool:
|
|
1387
|
+
# Fresh evidence in EITHER continuity column: a delta can resolve
|
|
1388
|
+
# through the turn state alone, so a turn-state advance without a
|
|
1389
|
+
# response id is still continuity a tombstone must not fail closed.
|
|
1390
|
+
return observed is not None and (
|
|
1391
|
+
(observed[0] is not None and observed[0] != abandoned_anchor)
|
|
1392
|
+
or (observed[1] is not None and observed[1] != abandoned_turn_state)
|
|
1393
|
+
)
|
|
1394
|
+
|
|
1395
|
+
if session.durable_session_id:
|
|
1396
|
+
try:
|
|
1397
|
+
post_clear_continuity = await service._durable_bridge.session_latest_continuity(
|
|
1398
|
+
session_id=session.durable_session_id
|
|
1399
|
+
)
|
|
1400
|
+
except Exception:
|
|
1401
|
+
# Unknown continuity keeps the tombstone: the stale-tombstone
|
|
1402
|
+
# cost is self-healing (the next completion settles and erases
|
|
1403
|
+
# it), while a missing tombstone silently drops delta context.
|
|
1404
|
+
post_clear_continuity = None
|
|
1405
|
+
if _continuity_is_fresh(post_clear_continuity):
|
|
1406
|
+
abandonment_settled_detail = None
|
|
1407
|
+
circuit_settled = await service._clear_http_bridge_retry_circuit(
|
|
1408
|
+
session,
|
|
1409
|
+
settled_detail=abandonment_settled_detail,
|
|
1410
|
+
settled_detail_authoritative=True,
|
|
1411
|
+
expected_episode=abandonment_expected_episode,
|
|
1412
|
+
)
|
|
1413
|
+
if not circuit_settled:
|
|
1414
|
+
# The clear reloads fresh state on every call, so one immediate
|
|
1415
|
+
# retry covers a transient durable failure or a double CAS miss
|
|
1416
|
+
# before the surviving cooldown is left to expire on its own. The
|
|
1417
|
+
# abandonment itself still reports success: the anchor is gone, the
|
|
1418
|
+
# marker must record that so the episode cannot clear now-empty
|
|
1419
|
+
# continuity again, and only the settlement is owed.
|
|
1420
|
+
circuit_settled = await service._clear_http_bridge_retry_circuit(
|
|
1421
|
+
session,
|
|
1422
|
+
settled_detail=abandonment_settled_detail,
|
|
1423
|
+
settled_detail_authoritative=True,
|
|
1424
|
+
expected_episode=abandonment_expected_episode,
|
|
1425
|
+
)
|
|
1426
|
+
if not circuit_settled:
|
|
1427
|
+
_log_http_bridge_event(
|
|
1428
|
+
"durable_circuit_settle_failed",
|
|
1429
|
+
session.key,
|
|
1430
|
+
account_id=session.account.id,
|
|
1431
|
+
model=session.request_model,
|
|
1432
|
+
detail=detail,
|
|
1433
|
+
cache_key_family=session.key.affinity_kind,
|
|
1434
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1435
|
+
)
|
|
1436
|
+
if circuit_settled and abandonment_settled_detail is not None and session.durable_session_id:
|
|
1437
|
+
# The pre-settle re-read closes most of the window, but a sibling
|
|
1438
|
+
# registration can still land between that read and the settle
|
|
1439
|
+
# write — and its own tombstone cleanup ran BEFORE this settle
|
|
1440
|
+
# wrote a new one. Re-read once more after the write and erase the
|
|
1441
|
+
# tombstone this settle left when fresh continuity now exists; the
|
|
1442
|
+
# fenced detail-only rewrite defers to any newer write, and both
|
|
1443
|
+
# sides reconciling after their own writes makes every
|
|
1444
|
+
# interleaving converge.
|
|
1445
|
+
try:
|
|
1446
|
+
reconcile_continuity = await service._durable_bridge.session_latest_continuity(
|
|
1447
|
+
session_id=session.durable_session_id
|
|
1448
|
+
)
|
|
1449
|
+
except Exception:
|
|
1450
|
+
reconcile_continuity = None
|
|
1451
|
+
if _continuity_is_fresh(reconcile_continuity):
|
|
1452
|
+
await service._http_bridge_reconcile_transitional_tombstone(session)
|
|
1453
|
+
return True
|
|
1454
|
+
|
|
1455
|
+
|
|
1456
|
+
async def _invalidate_denied_http_bridge_anchor(
|
|
1457
|
+
service: Any,
|
|
1458
|
+
session: "_HTTPBridgeSession",
|
|
1459
|
+
*,
|
|
1460
|
+
denied_response_id: str | None,
|
|
1461
|
+
) -> bool:
|
|
1462
|
+
"""Retire an anchor upstream has explicitly denied.
|
|
1463
|
+
|
|
1464
|
+
``previous_response_not_found`` against an anchor the proxy injected is a
|
|
1465
|
+
verdict, not a symptom: the id came from this proxy's own durable record,
|
|
1466
|
+
no client asked for it, and upstream says it does not exist. The poison
|
|
1467
|
+
counter cannot act on that verdict because it only scores reader failures
|
|
1468
|
+
(see ``_HTTP_BRIDGE_ANCHOR_POISON_DETAILS``), so the dead id survives at
|
|
1469
|
+
any threshold and is re-injected into the following turn, where the
|
|
1470
|
+
store-context trim strips the resent history against it and upstream never
|
|
1471
|
+
emits ``response.created``.
|
|
1472
|
+
|
|
1473
|
+
Clearing costs nothing that is not already lost. The next turn simply
|
|
1474
|
+
dispatches unanchored with the history the client sends, which is the
|
|
1475
|
+
client's own replay rather than a server-side one, so no forked child
|
|
1476
|
+
response can be created against a parent this proxy cannot see.
|
|
1477
|
+
|
|
1478
|
+
The durable clear is conditional on the denied id still being the durable
|
|
1479
|
+
latest response, and removes only that response alias. The in-memory clear
|
|
1480
|
+
is unconditional for the same id even when the durable write is fenced,
|
|
1481
|
+
because it strictly removes one way for the denied id to come back. A
|
|
1482
|
+
durable row that survives re-injects the id on a later turn, which is denied
|
|
1483
|
+
in turn and re-enters this path, so the clear is re-attempted rather than
|
|
1484
|
+
lost.
|
|
1485
|
+
|
|
1486
|
+
``closed`` only fences new admissions. Requests admitted before a session
|
|
1487
|
+
closed can still deliver a terminal denial and must publish its provenance
|
|
1488
|
+
and finish the same cleanup.
|
|
1489
|
+
"""
|
|
1490
|
+
if denied_response_id is None:
|
|
1491
|
+
return False
|
|
1492
|
+
sibling_advanced = False
|
|
1493
|
+
async with session.lifecycle_lock:
|
|
1494
|
+
# Serialize publication with the submitter's final tombstone check and
|
|
1495
|
+
# upstream send. A sibling completion can advance the current carrier
|
|
1496
|
+
# while an already-prepared request still holds the denied id; that
|
|
1497
|
+
# request must remain fenced even when there is no current anchor left
|
|
1498
|
+
# to clear.
|
|
1499
|
+
session.denied_proxy_injected_anchor_generation += 1
|
|
1500
|
+
# Retain denial provenance after the session-local tombstone is retired.
|
|
1501
|
+
# A request that began on an absent canonical session otherwise receives
|
|
1502
|
+
# a successor with no local generation and can redispatch this id.
|
|
1503
|
+
durable_session_id = session.durable_session_id
|
|
1504
|
+
durable_owner_epoch = session.durable_owner_epoch
|
|
1505
|
+
durable_api_key_id = session.key.api_key_id
|
|
1506
|
+
durable_instance_id = _service_get_settings().http_responses_session_bridge_instance_id
|
|
1507
|
+
owner_key = durable_session_id if durable_session_id is not None else f"local:{id(session)}"
|
|
1508
|
+
recorded_generation = _record_http_bridge_denied_anchor_fence(
|
|
1509
|
+
service,
|
|
1510
|
+
denied_response_id,
|
|
1511
|
+
owner_key=owner_key,
|
|
1512
|
+
owner_epoch=durable_owner_epoch,
|
|
1513
|
+
)
|
|
1514
|
+
current_fence = _http_bridge_denied_anchor_fence_current_map(service).get(owner_key)
|
|
1515
|
+
recorded_entry = _http_bridge_denied_anchor_fence_entry(service, denied_response_id)
|
|
1516
|
+
record_won_owner_slot = (
|
|
1517
|
+
current_fence == denied_response_id
|
|
1518
|
+
and recorded_entry is not None
|
|
1519
|
+
and recorded_entry.owner_key == owner_key
|
|
1520
|
+
and recorded_entry.generation == recorded_generation
|
|
1521
|
+
)
|
|
1522
|
+
# Keep one current session-local tombstone. Displaced ids remain
|
|
1523
|
+
# fenced in the process ledger while any prepared request pins them;
|
|
1524
|
+
# retaining every historical id here would make the session carrier
|
|
1525
|
+
# grow without bound and duplicate that ledger's ownership fence. A
|
|
1526
|
+
# stale detached predecessor must not erase a successor's tombstone,
|
|
1527
|
+
# so only replace the set when this publication owns the current slot.
|
|
1528
|
+
if record_won_owner_slot:
|
|
1529
|
+
session.denied_proxy_injected_anchor_ids.clear()
|
|
1530
|
+
session.denied_proxy_injected_anchor_ids.add(denied_response_id)
|
|
1531
|
+
elif not session.denied_proxy_injected_anchor_ids:
|
|
1532
|
+
# A detached predecessor may still have a request pinned to its
|
|
1533
|
+
# own session object. Keep that one local tombstone without
|
|
1534
|
+
# growing a second historical slot beside a successor's entry.
|
|
1535
|
+
session.denied_proxy_injected_anchor_ids.add(denied_response_id)
|
|
1536
|
+
# Another request may have completed and advanced the anchor between
|
|
1537
|
+
# the denied dispatch and this frame. Only retire the id that was
|
|
1538
|
+
# refused.
|
|
1539
|
+
if session.last_completed_response_id != denied_response_id:
|
|
1540
|
+
sibling_advanced = True
|
|
1541
|
+
elif hasattr(session, "denied_proxy_injected_anchor_cleanup_pending") and recorded_entry is not None:
|
|
1542
|
+
# Only the current anchor needs durable/alias cleanup. A sibling
|
|
1543
|
+
# that already advanced the carrier leaves a historical fence for
|
|
1544
|
+
# any pinned request, but has no unresolved cleanup to preserve at
|
|
1545
|
+
# session close.
|
|
1546
|
+
session.denied_proxy_injected_anchor_cleanup_pending.add(denied_response_id)
|
|
1547
|
+
cleared = False
|
|
1548
|
+
no_durable_owner = durable_session_id is None or durable_owner_epoch is None
|
|
1549
|
+
if sibling_advanced:
|
|
1550
|
+
if no_durable_owner:
|
|
1551
|
+
_forget_http_bridge_denied_anchor_fence(
|
|
1552
|
+
service,
|
|
1553
|
+
denied_response_id,
|
|
1554
|
+
owner_key=owner_key,
|
|
1555
|
+
owner_epoch=durable_owner_epoch,
|
|
1556
|
+
)
|
|
1557
|
+
getattr(session, "denied_proxy_injected_anchor_cleanup_pending", set()).discard(denied_response_id)
|
|
1558
|
+
return False
|
|
1559
|
+
retry_durable_clear = False
|
|
1560
|
+
unregister_succeeded = False
|
|
1561
|
+
owner_matches_for_cleanup = False
|
|
1562
|
+
unregister_error: BaseException | None = None
|
|
1563
|
+
durable_error: BaseException | None = None
|
|
1564
|
+
try:
|
|
1565
|
+
try:
|
|
1566
|
+
if not no_durable_owner:
|
|
1567
|
+
lookup = await service._durable_bridge.clear_live_session_response_anchor_if_matches(
|
|
1568
|
+
session_id=durable_session_id,
|
|
1569
|
+
api_key_id=durable_api_key_id,
|
|
1570
|
+
instance_id=durable_instance_id,
|
|
1571
|
+
owner_epoch=durable_owner_epoch,
|
|
1572
|
+
response_id=denied_response_id,
|
|
1573
|
+
)
|
|
1574
|
+
cleared = lookup is not None
|
|
1575
|
+
# ``None`` is a clean fenced no-match (the owner epoch or
|
|
1576
|
+
# latest response changed), not a durable failure. Preserve
|
|
1577
|
+
# the local alias and tombstone for the newer owner, but do
|
|
1578
|
+
# not schedule background retries; raised failures below are
|
|
1579
|
+
# the retryable case.
|
|
1580
|
+
except Exception:
|
|
1581
|
+
retry_durable_clear = True
|
|
1582
|
+
logger.warning("Failed to clear denied HTTP bridge response anchor", exc_info=True)
|
|
1583
|
+
finally:
|
|
1584
|
+
try:
|
|
1585
|
+
if cleared or no_durable_owner:
|
|
1586
|
+
async with session.lifecycle_lock:
|
|
1587
|
+
owner_matches_for_cleanup = (
|
|
1588
|
+
session.durable_session_id == durable_session_id
|
|
1589
|
+
and session.durable_owner_epoch == durable_owner_epoch
|
|
1590
|
+
)
|
|
1591
|
+
if owner_matches_for_cleanup:
|
|
1592
|
+
try:
|
|
1593
|
+
unregister_result = await service._unregister_http_bridge_previous_response_id(
|
|
1594
|
+
session,
|
|
1595
|
+
denied_response_id,
|
|
1596
|
+
expected_durable_session_id=durable_session_id,
|
|
1597
|
+
expected_durable_owner_epoch=durable_owner_epoch,
|
|
1598
|
+
)
|
|
1599
|
+
unregister_succeeded = unregister_result is not False
|
|
1600
|
+
except asyncio.CancelledError as exc:
|
|
1601
|
+
unregister_error = exc
|
|
1602
|
+
except Exception as exc:
|
|
1603
|
+
unregister_error = exc
|
|
1604
|
+
retry_durable_clear = True
|
|
1605
|
+
finally:
|
|
1606
|
+
async with session.lifecycle_lock:
|
|
1607
|
+
if session.last_completed_response_id == denied_response_id:
|
|
1608
|
+
session.last_completed_response_id = None
|
|
1609
|
+
session.last_completed_response_account_id = None
|
|
1610
|
+
session.last_completed_input_count = 0
|
|
1611
|
+
session.last_completed_input_prefix_fingerprint = None
|
|
1612
|
+
session.last_pending_tool_calls.clear()
|
|
1613
|
+
if owner_matches_for_cleanup and (cleared or no_durable_owner) and unregister_succeeded:
|
|
1614
|
+
session.denied_proxy_injected_anchor_ids.discard(denied_response_id)
|
|
1615
|
+
session.denied_proxy_injected_anchor_cleanup_pending.discard(denied_response_id)
|
|
1616
|
+
session.denied_proxy_injected_anchor_generation += 1
|
|
1617
|
+
_forget_http_bridge_denied_anchor_fence(
|
|
1618
|
+
service,
|
|
1619
|
+
denied_response_id,
|
|
1620
|
+
owner_key=durable_session_id if durable_session_id is not None else f"local:{id(session)}",
|
|
1621
|
+
owner_epoch=durable_owner_epoch,
|
|
1622
|
+
)
|
|
1623
|
+
if retry_durable_clear:
|
|
1624
|
+
_schedule_denied_http_bridge_anchor_clear_retry(
|
|
1625
|
+
service,
|
|
1626
|
+
session,
|
|
1627
|
+
response_id=denied_response_id,
|
|
1628
|
+
session_id=durable_session_id,
|
|
1629
|
+
api_key_id=durable_api_key_id,
|
|
1630
|
+
instance_id=durable_instance_id,
|
|
1631
|
+
owner_epoch=durable_owner_epoch,
|
|
1632
|
+
durable_cleared=cleared,
|
|
1633
|
+
)
|
|
1634
|
+
except asyncio.CancelledError as exc:
|
|
1635
|
+
durable_error = exc
|
|
1636
|
+
if unregister_error is not None:
|
|
1637
|
+
raise unregister_error
|
|
1638
|
+
if durable_error is not None:
|
|
1639
|
+
raise durable_error
|
|
1640
|
+
return cleared
|
|
1641
|
+
|
|
1642
|
+
|
|
1643
|
+
def _denied_proxy_injected_anchor_id(
|
|
1644
|
+
request_states: Iterable["_WebSocketRequestState"],
|
|
1645
|
+
) -> str | None:
|
|
1646
|
+
"""Choose the anchor a denial may retire, if any.
|
|
1647
|
+
|
|
1648
|
+
Only an anchor shared exclusively by requests that codex-lb injected onto
|
|
1649
|
+
full-resend-shaped payloads can be retired. A client-supplied or delta-only
|
|
1650
|
+
sibling has no other way to convey prior context once its anchor is gone,
|
|
1651
|
+
which is the same rule the expired-anchor path applies before clearing
|
|
1652
|
+
durable continuity.
|
|
1653
|
+
"""
|
|
1654
|
+
requests_by_anchor: dict[str, list[_WebSocketRequestState]] = {}
|
|
1655
|
+
for request_state in request_states:
|
|
1656
|
+
previous_response_id = request_state.previous_response_id
|
|
1657
|
+
if previous_response_id is not None:
|
|
1658
|
+
requests_by_anchor.setdefault(previous_response_id, []).append(request_state)
|
|
1659
|
+
|
|
1660
|
+
safely_retirable_anchors = [
|
|
1661
|
+
previous_response_id
|
|
1662
|
+
for previous_response_id, grouped_request_states in requests_by_anchor.items()
|
|
1663
|
+
if all(
|
|
1664
|
+
request_state.proxy_injected_previous_response_id
|
|
1665
|
+
and request_state.proxy_injected_anchor_had_full_resend_payload
|
|
1666
|
+
for request_state in grouped_request_states
|
|
1667
|
+
)
|
|
1668
|
+
]
|
|
1669
|
+
if len(safely_retirable_anchors) == 1:
|
|
1670
|
+
return safely_retirable_anchors[0]
|
|
1671
|
+
return None
|
|
1672
|
+
|
|
1673
|
+
|
|
1674
|
+
async def _retire_denied_http_bridge_anchor(
|
|
1675
|
+
service: Any,
|
|
1676
|
+
session: "_HTTPBridgeSession",
|
|
1677
|
+
*,
|
|
1678
|
+
request_states: Iterable["_WebSocketRequestState"],
|
|
1679
|
+
) -> None:
|
|
1680
|
+
"""Best-effort retirement of an anchor upstream denied.
|
|
1681
|
+
|
|
1682
|
+
Retirement is bookkeeping. It must never change how the denial itself is
|
|
1683
|
+
delivered downstream, so a failure here is logged and swallowed rather than
|
|
1684
|
+
escaping into terminal-event handling.
|
|
1685
|
+
"""
|
|
1686
|
+
denied_response_id = _denied_proxy_injected_anchor_id(request_states)
|
|
1687
|
+
if denied_response_id is None:
|
|
1688
|
+
return
|
|
1689
|
+
try:
|
|
1690
|
+
await _invalidate_denied_http_bridge_anchor(
|
|
1691
|
+
service,
|
|
1692
|
+
session,
|
|
1693
|
+
denied_response_id=denied_response_id,
|
|
1694
|
+
)
|
|
1695
|
+
except Exception:
|
|
1696
|
+
logger.warning("Failed to retire a denied proxy-injected HTTP bridge anchor", exc_info=True)
|
|
1697
|
+
|
|
1698
|
+
|
|
1699
|
+
class _HTTPBridgeUpstreamEventsMixin:
|
|
1700
|
+
def _spawn_http_bridge_upstream_reader(self: Any, session: "_HTTPBridgeSession") -> asyncio.Task[None]:
|
|
1701
|
+
"""Start the per-session upstream reader on the owner's scheduler."""
|
|
1702
|
+
|
|
1703
|
+
return scheduler_for(self).create_task(self._relay_http_bridge_upstream_messages(session))
|
|
1704
|
+
|
|
1705
|
+
async def _cancel_http_bridge_previous_reader(self: Any, old_reader: asyncio.Task[Any]) -> bool:
|
|
1706
|
+
"""Cancel and drain a replaced upstream reader before a reconnect."""
|
|
1707
|
+
|
|
1708
|
+
return await _await_cancelled_task(
|
|
1709
|
+
old_reader,
|
|
1710
|
+
label="http bridge upstream reader",
|
|
1711
|
+
cleanup_tasks=self._background_cleanup_tasks,
|
|
1712
|
+
scheduler=scheduler_for(self),
|
|
1713
|
+
)
|
|
1714
|
+
|
|
1715
|
+
async def _await_http_bridge_registry_wait(self: Any, future: asyncio.Future[T], *, timeout: float) -> T:
|
|
1716
|
+
"""Await a shared inflight/capacity registry future with a bounded timeout.
|
|
1717
|
+
|
|
1718
|
+
Not ``wait_for(shield(...))``: shield attaches per-waiter callbacks to
|
|
1719
|
+
the shared registry future, which livelocks the event loop under mass
|
|
1720
|
+
timeout (see shared_future.py). The timed wait runs on the owner's
|
|
1721
|
+
scheduler so virtual time can expire it.
|
|
1722
|
+
"""
|
|
1723
|
+
|
|
1724
|
+
return await wait_on_shared_future(future, timeout=timeout, scheduler=scheduler_for(self))
|
|
1725
|
+
|
|
1726
|
+
async def _fail_http_bridge_reader_and_maybe_retire(
|
|
1727
|
+
self: Any,
|
|
1728
|
+
session: "_HTTPBridgeSession",
|
|
1729
|
+
*,
|
|
1730
|
+
error_code: str,
|
|
1731
|
+
error_message: str,
|
|
1732
|
+
penalize_account: bool = True,
|
|
1733
|
+
retire_detail: str | None = None,
|
|
1734
|
+
force_retire: bool = False,
|
|
1735
|
+
upstream_close_code: int | None = None,
|
|
1736
|
+
response_events_seen: int | None = None,
|
|
1737
|
+
transport_classification: str | None = None,
|
|
1738
|
+
retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None,
|
|
1739
|
+
account_neutral_transport_drop: bool = False,
|
|
1740
|
+
) -> bool:
|
|
1741
|
+
scheduler = scheduler_for(self)
|
|
1742
|
+
session.closed = True
|
|
1743
|
+
async with session.pending_lock:
|
|
1744
|
+
failed_pending_count = sum(
|
|
1745
|
+
1
|
|
1746
|
+
for request_state in session.pending_requests
|
|
1747
|
+
if _http_bridge_request_counts_against_queue(request_state)
|
|
1748
|
+
)
|
|
1749
|
+
session.queued_request_count = max(0, session.queued_request_count - failed_pending_count)
|
|
1750
|
+
observed_response_events = max(
|
|
1751
|
+
(getattr(request_state, "response_event_count", 0) for request_state in session.pending_requests),
|
|
1752
|
+
default=0,
|
|
1753
|
+
)
|
|
1754
|
+
pending_request_states = list(session.pending_requests)
|
|
1755
|
+
if retry_circuit_attempt_selection is None:
|
|
1756
|
+
retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
|
|
1757
|
+
pending_request_states
|
|
1758
|
+
)
|
|
1759
|
+
retry_circuit_attempt_kwargs = {
|
|
1760
|
+
"retry_circuit_attempt_selection": retry_circuit_attempt_selection,
|
|
1761
|
+
}
|
|
1762
|
+
# The #1534 wedge shape: a reattached stream that streamed response
|
|
1763
|
+
# events whose ``response.created`` was never assigned. The eventless
|
|
1764
|
+
# watchdog and the durable-anchor clear both key on
|
|
1765
|
+
# ``response_event_count == 0`` and never trip on it, so quarantine
|
|
1766
|
+
# the session here so later requests stop re-attaching to it.
|
|
1767
|
+
_record_http_bridge_quarantine_wedged_pending(self, session, pending_request_states)
|
|
1768
|
+
observed_close_code = (
|
|
1769
|
+
upstream_close_code if upstream_close_code is not None else session.last_upstream_close_code
|
|
1770
|
+
)
|
|
1771
|
+
observed_response_events = (
|
|
1772
|
+
response_events_seen if response_events_seen is not None else observed_response_events
|
|
1773
|
+
)
|
|
1774
|
+
close_classification = (
|
|
1775
|
+
_classify_upstream_close(observed_close_code, response_events_seen=observed_response_events)
|
|
1776
|
+
if observed_close_code is not None
|
|
1777
|
+
else None
|
|
1778
|
+
)
|
|
1779
|
+
_log_http_bridge_event(
|
|
1780
|
+
"reader_failure",
|
|
1781
|
+
session.key,
|
|
1782
|
+
account_id=session.account.id,
|
|
1783
|
+
model=session.request_model,
|
|
1784
|
+
pending_count=failed_pending_count,
|
|
1785
|
+
detail=error_code,
|
|
1786
|
+
error_message=_truncate_identifier(error_message),
|
|
1787
|
+
upstream_close_code=observed_close_code,
|
|
1788
|
+
response_events_seen=observed_response_events,
|
|
1789
|
+
transport_classification=transport_classification
|
|
1790
|
+
or (
|
|
1791
|
+
f"websocket_close_{close_classification}"
|
|
1792
|
+
if close_classification is not None
|
|
1793
|
+
else "websocket_transport_error"
|
|
1794
|
+
),
|
|
1795
|
+
cache_key_family=session.key.affinity_kind,
|
|
1796
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1797
|
+
)
|
|
1798
|
+
# Draining-only requests no longer count against the queue, but their
|
|
1799
|
+
# event-batcher contexts still belong to the disconnected operation
|
|
1800
|
+
# and must be discarded just like ordinary pending requests.
|
|
1801
|
+
operation_states: list[Any] = [
|
|
1802
|
+
request_state for request_state in pending_request_states if getattr(request_state, "operation_id", None)
|
|
1803
|
+
]
|
|
1804
|
+
# Remove the disconnected attempt's in-memory spool before publishing
|
|
1805
|
+
# UNKNOWN/ACKNOWLEDGED state. A same-replica reconnect may reclaim the
|
|
1806
|
+
# operation as soon as that state is visible; discarding afterward
|
|
1807
|
+
# could then delete the replacement attempt's events.
|
|
1808
|
+
discard_operation = getattr(
|
|
1809
|
+
getattr(self, "_http_bridge_operation_event_batcher", None),
|
|
1810
|
+
"discard_operation",
|
|
1811
|
+
None,
|
|
1812
|
+
)
|
|
1813
|
+
if callable(discard_operation):
|
|
1814
|
+
for request_state in operation_states:
|
|
1815
|
+
operation_id = getattr(request_state, "operation_id", None)
|
|
1816
|
+
if operation_id:
|
|
1817
|
+
await discard_operation(operation_id=operation_id)
|
|
1818
|
+
for request_state in operation_states:
|
|
1819
|
+
# A shared websocket can carry several logical response.create
|
|
1820
|
+
# requests. Classify each operation from its own event count;
|
|
1821
|
+
# using the session-wide maximum would mark an eventless
|
|
1822
|
+
# sibling as safely retryable after another request streamed.
|
|
1823
|
+
operation_state = "unknown" if getattr(request_state, "response_event_count", 0) == 0 else "acknowledged"
|
|
1824
|
+
await _update_http_bridge_operation_state(
|
|
1825
|
+
self,
|
|
1826
|
+
session,
|
|
1827
|
+
request_state,
|
|
1828
|
+
state=operation_state,
|
|
1829
|
+
)
|
|
1830
|
+
if force_retire and retire_detail:
|
|
1831
|
+
_log_http_bridge_event(
|
|
1832
|
+
retire_detail,
|
|
1833
|
+
session.key,
|
|
1834
|
+
account_id=session.account.id,
|
|
1835
|
+
model=session.request_model,
|
|
1836
|
+
pending_count=failed_pending_count,
|
|
1837
|
+
detail=retire_detail,
|
|
1838
|
+
cache_key_family=session.key.affinity_kind,
|
|
1839
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1840
|
+
)
|
|
1841
|
+
try:
|
|
1842
|
+
reservations_settled = await self._fail_pending_websocket_requests(
|
|
1843
|
+
account=session.account,
|
|
1844
|
+
account_id_value=session.account.id,
|
|
1845
|
+
pending_requests=session.pending_requests,
|
|
1846
|
+
pending_lock=session.pending_lock,
|
|
1847
|
+
error_code=error_code,
|
|
1848
|
+
error_message=error_message,
|
|
1849
|
+
api_key=None,
|
|
1850
|
+
response_create_gate=session.response_create_gate,
|
|
1851
|
+
penalize_account=penalize_account,
|
|
1852
|
+
)
|
|
1853
|
+
if (
|
|
1854
|
+
failed_pending_count > 0
|
|
1855
|
+
and reservations_settled is not False
|
|
1856
|
+
and observed_response_events == 0
|
|
1857
|
+
and (
|
|
1858
|
+
retire_detail == _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL
|
|
1859
|
+
or account_neutral_transport_drop
|
|
1860
|
+
)
|
|
1861
|
+
):
|
|
1862
|
+
# Only penalize the account after pending-request cleanup has
|
|
1863
|
+
# settled its API-key reservations. A failed release must not
|
|
1864
|
+
# be hidden behind an already-recorded timeout health signal.
|
|
1865
|
+
# Account-neutral abrupt drops share the same windowed signal:
|
|
1866
|
+
# one drop is infrastructure noise, but repeated eventless
|
|
1867
|
+
# drops on the same account remain evidence of an account-side
|
|
1868
|
+
# fault and must still drain it (issue #1754).
|
|
1869
|
+
await _record_http_bridge_account_timeout_signal(
|
|
1870
|
+
self,
|
|
1871
|
+
session,
|
|
1872
|
+
detail=(
|
|
1873
|
+
"eventless_transport_drop"
|
|
1874
|
+
if account_neutral_transport_drop
|
|
1875
|
+
else _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL
|
|
1876
|
+
),
|
|
1877
|
+
)
|
|
1878
|
+
finally:
|
|
1879
|
+
if session.admission_waiter_count > 0 and not force_retire:
|
|
1880
|
+
retry_circuit_detail = None
|
|
1881
|
+
if close_classification == "clean":
|
|
1882
|
+
retry_circuit_detail = "clean_close"
|
|
1883
|
+
elif observed_response_events == 0:
|
|
1884
|
+
retry_circuit_detail = next(
|
|
1885
|
+
(
|
|
1886
|
+
detail
|
|
1887
|
+
for detail in (retire_detail, error_code)
|
|
1888
|
+
if detail in {"stream_incomplete", "stream_idle_timeout", "upstream_keepalive_timeout"}
|
|
1889
|
+
),
|
|
1890
|
+
None,
|
|
1891
|
+
)
|
|
1892
|
+
# Mirror the terminal, grouped, and direct-retirement paths:
|
|
1893
|
+
# a failed request that still holds a verified safe replay is
|
|
1894
|
+
# about to re-dispatch and claim the circuit generation, so it
|
|
1895
|
+
# must not strike the circuit or reach the poison clear here
|
|
1896
|
+
# either. A pre-drain handoff with no states keeps striking.
|
|
1897
|
+
pending_states_present = [state for state in pending_request_states if state is not None]
|
|
1898
|
+
reader_strike_eligible = not pending_states_present or any(
|
|
1899
|
+
not _http_bridge_request_state_holds_safe_replay(state) for state in pending_states_present
|
|
1900
|
+
)
|
|
1901
|
+
reader_retired = False
|
|
1902
|
+
reader_cancellation: asyncio.CancelledError | None = None
|
|
1903
|
+
if failed_pending_count > 0 and retry_circuit_detail is not None and reader_strike_eligible:
|
|
1904
|
+
reader_strike_detail = retry_circuit_detail
|
|
1905
|
+
|
|
1906
|
+
async def _reader_strike_and_clear() -> bool:
|
|
1907
|
+
consecutive_failures = (
|
|
1908
|
+
await self._record_http_bridge_retry_circuit_failure_for_attempt_selection(
|
|
1909
|
+
session,
|
|
1910
|
+
detail=reader_strike_detail,
|
|
1911
|
+
selection=retry_circuit_attempt_selection,
|
|
1912
|
+
)
|
|
1913
|
+
)
|
|
1914
|
+
poison_candidate_detail = await self._http_bridge_effective_anchor_poison_detail(
|
|
1915
|
+
session, reader_strike_detail
|
|
1916
|
+
)
|
|
1917
|
+
poison_episode = None
|
|
1918
|
+
poison_expected_anchor: object = _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1919
|
+
if poison_candidate_detail is not None and observed_response_events == 0:
|
|
1920
|
+
# The consult returns the exact episode it
|
|
1921
|
+
# validated; the marker below scopes to it.
|
|
1922
|
+
poison_episode, poison_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
|
|
1923
|
+
session,
|
|
1924
|
+
consecutive_failures=consecutive_failures,
|
|
1925
|
+
)
|
|
1926
|
+
if poison_candidate_detail is None or poison_episode is None:
|
|
1927
|
+
return False
|
|
1928
|
+
durable_cleared = await _abandon_durable_http_bridge_continuity(
|
|
1929
|
+
self,
|
|
1930
|
+
session,
|
|
1931
|
+
detail=poison_candidate_detail,
|
|
1932
|
+
settle_circuit=_http_bridge_abandonment_may_settle_circuit(pending_request_states),
|
|
1933
|
+
expected_continuity=poison_expected_anchor,
|
|
1934
|
+
authorized_episode=poison_episode,
|
|
1935
|
+
)
|
|
1936
|
+
if not durable_cleared:
|
|
1937
|
+
_log_http_bridge_event(
|
|
1938
|
+
"durable_anchor_poison_clear_failed",
|
|
1939
|
+
session.key,
|
|
1940
|
+
account_id=session.account.id,
|
|
1941
|
+
model=session.request_model,
|
|
1942
|
+
pending_count=session.admission_waiter_count,
|
|
1943
|
+
detail=poison_candidate_detail,
|
|
1944
|
+
cache_key_family=session.key.affinity_kind,
|
|
1945
|
+
model_class=(
|
|
1946
|
+
_extract_model_class(session.request_model) if session.request_model else None
|
|
1947
|
+
),
|
|
1948
|
+
)
|
|
1949
|
+
return False
|
|
1950
|
+
await self._http_bridge_mark_poison_anchor_cleared(session, episode=poison_episode)
|
|
1951
|
+
await self._retire_stale_pending_http_bridge_session(
|
|
1952
|
+
session,
|
|
1953
|
+
detail=poison_candidate_detail,
|
|
1954
|
+
response_events_seen=observed_response_events,
|
|
1955
|
+
# Reader-failure retirement must never revive: the
|
|
1956
|
+
# pending turns were already terminally failed and
|
|
1957
|
+
# this reader is condemned, so a post-suspension
|
|
1958
|
+
# liveness signal (which durable-anchor
|
|
1959
|
+
# rehydration can spoof without upstream evidence)
|
|
1960
|
+
# would only leave a readerless session registered.
|
|
1961
|
+
allow_liveness_revive=False,
|
|
1962
|
+
**retry_circuit_attempt_kwargs,
|
|
1963
|
+
)
|
|
1964
|
+
return True
|
|
1965
|
+
|
|
1966
|
+
# The failed requests are already drained and finalized,
|
|
1967
|
+
# so a cancellation escaping the strike, the episode
|
|
1968
|
+
# consult, or the rebind would leave an at-threshold
|
|
1969
|
+
# poisoned anchor stored with no retirement left to retry
|
|
1970
|
+
# the abandonment. Defer cancellation across the whole
|
|
1971
|
+
# settlement like the grouped path, then re-raise it.
|
|
1972
|
+
reader_settlement_task = scheduler.create_task(
|
|
1973
|
+
_reader_strike_and_clear(),
|
|
1974
|
+
name=f"http-bridge-reader-poison-settlement-{session.durable_session_id}",
|
|
1975
|
+
)
|
|
1976
|
+
reader_retired, reader_cancellation = await _await_task_deferring_cancellation(
|
|
1977
|
+
reader_settlement_task
|
|
1978
|
+
)
|
|
1979
|
+
if reader_retired is True:
|
|
1980
|
+
force_retire = True
|
|
1981
|
+
if reader_cancellation is not None:
|
|
1982
|
+
raise reader_cancellation
|
|
1983
|
+
else:
|
|
1984
|
+
if reader_cancellation is not None:
|
|
1985
|
+
raise reader_cancellation
|
|
1986
|
+
_log_http_bridge_event(
|
|
1987
|
+
"retire_deferred_for_admission_waiter",
|
|
1988
|
+
session.key,
|
|
1989
|
+
account_id=session.account.id,
|
|
1990
|
+
model=session.request_model,
|
|
1991
|
+
pending_count=session.admission_waiter_count,
|
|
1992
|
+
detail=retire_detail or error_code,
|
|
1993
|
+
cache_key_family=session.key.affinity_kind,
|
|
1994
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1995
|
+
)
|
|
1996
|
+
else:
|
|
1997
|
+
if close_classification == "clean" and failed_pending_count > 0:
|
|
1998
|
+
waiterless_retirement = self._retire_stale_pending_http_bridge_session(
|
|
1999
|
+
session,
|
|
2000
|
+
detail=error_code,
|
|
2001
|
+
retry_circuit_detail="clean_close",
|
|
2002
|
+
response_events_seen=observed_response_events,
|
|
2003
|
+
retired_request_count=failed_pending_count,
|
|
2004
|
+
retired_request_states=pending_request_states,
|
|
2005
|
+
# See the poison branch above: reader-failure
|
|
2006
|
+
# retirement never revives a condemned session.
|
|
2007
|
+
allow_liveness_revive=False,
|
|
2008
|
+
**retry_circuit_attempt_kwargs,
|
|
2009
|
+
)
|
|
2010
|
+
else:
|
|
2011
|
+
waiterless_retirement = self._retire_stale_pending_http_bridge_session(
|
|
2012
|
+
session,
|
|
2013
|
+
detail=retire_detail or error_code,
|
|
2014
|
+
response_events_seen=observed_response_events,
|
|
2015
|
+
# ``_fail_pending_websocket_requests`` has already
|
|
2016
|
+
# claimed and drained these states. Carry the count
|
|
2017
|
+
# and the pre-drain state snapshot sampled under
|
|
2018
|
+
# ``pending_lock`` across that ownership transfer, so
|
|
2019
|
+
# normal reader failures still consume one strike and
|
|
2020
|
+
# a drained safe-replay holder still blocks the
|
|
2021
|
+
# settlement. The deferred/poison branch records its
|
|
2022
|
+
# own strike above and intentionally does not pass it.
|
|
2023
|
+
retired_request_count=failed_pending_count,
|
|
2024
|
+
retired_request_states=pending_request_states,
|
|
2025
|
+
# See the poison branch above: reader-failure
|
|
2026
|
+
# retirement never revives a condemned session.
|
|
2027
|
+
allow_liveness_revive=False,
|
|
2028
|
+
**retry_circuit_attempt_kwargs,
|
|
2029
|
+
)
|
|
2030
|
+
# The failed requests are already drained and finalized, so a
|
|
2031
|
+
# relay cancellation escaping this direct retirement would
|
|
2032
|
+
# skip the waiterless strike, the episode consult, the
|
|
2033
|
+
# poisoned-anchor abandonment, and the detach/close work with
|
|
2034
|
+
# no remaining request lifecycle to retry them. Defer
|
|
2035
|
+
# cancellation across the whole retirement like the
|
|
2036
|
+
# admission-waiter branch above, then re-raise it.
|
|
2037
|
+
waiterless_retirement_task = scheduler.create_task(
|
|
2038
|
+
waiterless_retirement,
|
|
2039
|
+
name=f"http-bridge-waiterless-retirement-{session.durable_session_id}",
|
|
2040
|
+
)
|
|
2041
|
+
_waiterless_result, waiterless_cancellation = await _await_task_deferring_cancellation(
|
|
2042
|
+
waiterless_retirement_task
|
|
2043
|
+
)
|
|
2044
|
+
if waiterless_cancellation is not None:
|
|
2045
|
+
raise waiterless_cancellation
|
|
2046
|
+
return force_retire or session.admission_waiter_count == 0
|
|
2047
|
+
|
|
2048
|
+
async def _relay_http_bridge_upstream_messages(
|
|
2049
|
+
self: Any,
|
|
2050
|
+
session: "_HTTPBridgeSession",
|
|
2051
|
+
) -> None:
|
|
2052
|
+
scheduler = scheduler_for(self)
|
|
2053
|
+
clock = clock_for(self)
|
|
2054
|
+
runtime_settings = _service_get_settings()
|
|
2055
|
+
relay_upstream = session.upstream
|
|
2056
|
+
receive_task: asyncio.Task[UpstreamWebSocketMessage] | None = None
|
|
2057
|
+
wakeup_task: asyncio.Task[bool] | None = None
|
|
2058
|
+
reader_failure_retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None
|
|
2059
|
+
try:
|
|
2060
|
+
while True:
|
|
2061
|
+
reader_failure_retry_circuit_attempt_selection = None
|
|
2062
|
+
# Clear before taking the deadline snapshot. A send before the
|
|
2063
|
+
# clear is represented by its timestamp; a send after it leaves
|
|
2064
|
+
# the event set and wakes the persistent receive wait below.
|
|
2065
|
+
session.upstream_reader_wakeup.clear()
|
|
2066
|
+
# The wakeup waiter is reused across iterations while it is
|
|
2067
|
+
# still pending. A set() that landed while the previous
|
|
2068
|
+
# message was being processed completed it; that send is
|
|
2069
|
+
# already represented in the snapshot below, so consume the
|
|
2070
|
+
# fired waiter here without awaiting and re-arm it before the
|
|
2071
|
+
# next wait.
|
|
2072
|
+
if wakeup_task is not None and wakeup_task.done():
|
|
2073
|
+
wakeup_task.result()
|
|
2074
|
+
wakeup_task = None
|
|
2075
|
+
receive_timeout = await self._next_websocket_receive_timeout(
|
|
2076
|
+
session.pending_requests,
|
|
2077
|
+
pending_lock=session.pending_lock,
|
|
2078
|
+
proxy_request_budget_seconds=_http_bridge_request_budget_seconds(runtime_settings),
|
|
2079
|
+
stream_idle_timeout_seconds=runtime_settings.stream_idle_timeout_seconds,
|
|
2080
|
+
)
|
|
2081
|
+
stuck_gate_retire_after_seconds = float(
|
|
2082
|
+
_http_bridge_helpers.HTTP_BRIDGE_STUCK_GATE_RETIRE_AFTER_SECONDS
|
|
2083
|
+
)
|
|
2084
|
+
receive_timeout = await _http_bridge_receive_timeout_with_eventless_deadline(
|
|
2085
|
+
session,
|
|
2086
|
+
receive_timeout,
|
|
2087
|
+
now=clock.monotonic(),
|
|
2088
|
+
stuck_gate_retire_after_seconds=stuck_gate_retire_after_seconds,
|
|
2089
|
+
)
|
|
2090
|
+
if receive_task is None:
|
|
2091
|
+
receive_task = scheduler.create_task(session.upstream.receive())
|
|
2092
|
+
|
|
2093
|
+
message: UpstreamWebSocketMessage | None = None
|
|
2094
|
+
timed_out = False
|
|
2095
|
+
if receive_task.done():
|
|
2096
|
+
message = receive_task.result()
|
|
2097
|
+
receive_task = None
|
|
2098
|
+
elif receive_timeout is not None and receive_timeout.timeout_seconds <= 0:
|
|
2099
|
+
timed_out = True
|
|
2100
|
+
else:
|
|
2101
|
+
# Event.wait() waiters are level-triggered: a waiter
|
|
2102
|
+
# registered before clear() still fires on the next set(),
|
|
2103
|
+
# so a pending waiter stays valid across iterations and is
|
|
2104
|
+
# only re-created after it fires. Cancelling it per
|
|
2105
|
+
# message cost a create_task + sleep(0) + cancel + timed
|
|
2106
|
+
# asyncio.wait round trip; the finally below cancels the
|
|
2107
|
+
# long-lived waiter once at loop exit.
|
|
2108
|
+
if wakeup_task is None:
|
|
2109
|
+
wakeup_task = scheduler.create_task(session.upstream_reader_wakeup.wait())
|
|
2110
|
+
done, _pending = await scheduler.wait(
|
|
2111
|
+
(receive_task, wakeup_task),
|
|
2112
|
+
timeout=receive_timeout.timeout_seconds if receive_timeout is not None else None,
|
|
2113
|
+
return_when=asyncio.FIRST_COMPLETED,
|
|
2114
|
+
)
|
|
2115
|
+
if receive_task in done:
|
|
2116
|
+
message = receive_task.result()
|
|
2117
|
+
receive_task = None
|
|
2118
|
+
elif wakeup_task in done:
|
|
2119
|
+
wakeup_task.result()
|
|
2120
|
+
wakeup_task = None
|
|
2121
|
+
continue
|
|
2122
|
+
else:
|
|
2123
|
+
timed_out = True
|
|
2124
|
+
|
|
2125
|
+
if timed_out:
|
|
2126
|
+
if receive_timeout is None:
|
|
2127
|
+
raise RuntimeError("HTTP bridge reader timed out without a timeout contract")
|
|
2128
|
+
if receive_timeout.error_code == _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL:
|
|
2129
|
+
if receive_task is not None and receive_task.done():
|
|
2130
|
+
continue
|
|
2131
|
+
async with session.lifecycle_lock:
|
|
2132
|
+
# Send-failure cleanup marks the session closed and
|
|
2133
|
+
# disarms the timestamp while holding this lock.
|
|
2134
|
+
# Do not race that caller's terminal settlement.
|
|
2135
|
+
if session.closed:
|
|
2136
|
+
continue
|
|
2137
|
+
now = clock.monotonic()
|
|
2138
|
+
async with session.pending_lock:
|
|
2139
|
+
if receive_task is not None and receive_task.done():
|
|
2140
|
+
continue
|
|
2141
|
+
expired_request_states = [
|
|
2142
|
+
request_state
|
|
2143
|
+
for request_state in session.pending_requests
|
|
2144
|
+
if (
|
|
2145
|
+
deadline := _http_bridge_eventless_precreated_deadline(
|
|
2146
|
+
request_state,
|
|
2147
|
+
stuck_gate_retire_after_seconds=stuck_gate_retire_after_seconds,
|
|
2148
|
+
)
|
|
2149
|
+
)
|
|
2150
|
+
is not None
|
|
2151
|
+
and deadline <= now
|
|
2152
|
+
]
|
|
2153
|
+
if not expired_request_states:
|
|
2154
|
+
continue
|
|
2155
|
+
expired_retry_circuit_attempt_selection = (
|
|
2156
|
+
_http_bridge_retry_circuit_attempt_selection_for_pending_requests(
|
|
2157
|
+
expired_request_states
|
|
2158
|
+
)
|
|
2159
|
+
)
|
|
2160
|
+
reader_failure_retry_circuit_attempt_selection = expired_retry_circuit_attempt_selection
|
|
2161
|
+
pending_count = len(session.pending_requests)
|
|
2162
|
+
# A delta-only request has no other way to
|
|
2163
|
+
# convey prior context once its anchor is
|
|
2164
|
+
# cleared, so only clear anchors codex-lb
|
|
2165
|
+
# injected onto a full-resend-shaped payload.
|
|
2166
|
+
expired_proxy_injected_anchor = any(
|
|
2167
|
+
request_state.proxy_injected_previous_response_id
|
|
2168
|
+
and request_state.proxy_injected_anchor_had_full_resend_payload
|
|
2169
|
+
for request_state in expired_request_states
|
|
2170
|
+
)
|
|
2171
|
+
|
|
2172
|
+
# Do not mutate durable continuity while a receive
|
|
2173
|
+
# may still deliver the response event that proves
|
|
2174
|
+
# this timeout stale. The non-cancellable branch
|
|
2175
|
+
# remains fail-closed, so it clears immediately
|
|
2176
|
+
# before its forced retirement.
|
|
2177
|
+
force_retire = False
|
|
2178
|
+
if receive_task is not None:
|
|
2179
|
+
receive_cancelled = await _cancel_http_bridge_reader_child(
|
|
2180
|
+
receive_task,
|
|
2181
|
+
label="HTTP bridge upstream receive after missing response.created",
|
|
2182
|
+
cleanup_tasks=self._background_cleanup_tasks,
|
|
2183
|
+
scheduler_owner=self,
|
|
2184
|
+
)
|
|
2185
|
+
if receive_task.done() and not receive_task.cancelled():
|
|
2186
|
+
# A response (or a typed receive failure)
|
|
2187
|
+
# won the race with timeout handling. Let
|
|
2188
|
+
# the normal reader path settle it.
|
|
2189
|
+
continue
|
|
2190
|
+
force_retire = not receive_cancelled and not receive_task.cancelled()
|
|
2191
|
+
if not force_retire:
|
|
2192
|
+
receive_task = None
|
|
2193
|
+
async with session.pending_lock:
|
|
2194
|
+
for request_state in session.pending_requests:
|
|
2195
|
+
if request_state.failure_phase_override is None:
|
|
2196
|
+
request_state.failure_phase_override = "upstream"
|
|
2197
|
+
if request_state.failure_detail_override is None:
|
|
2198
|
+
request_state.failure_detail_override = (
|
|
2199
|
+
_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL
|
|
2200
|
+
)
|
|
2201
|
+
if force_retire:
|
|
2202
|
+
# Do not reconnect while the old receive task
|
|
2203
|
+
# still owns the superseded socket. The ordinary
|
|
2204
|
+
# timeout path takes the same fail-closed branch;
|
|
2205
|
+
# retain the explicit account-neutral timeout
|
|
2206
|
+
# classification rather than routing through the
|
|
2207
|
+
# generic reader-crash account penalty path.
|
|
2208
|
+
if expired_proxy_injected_anchor:
|
|
2209
|
+
await _clear_durable_http_bridge_response_anchor(self, session)
|
|
2210
|
+
_record_http_bridge_quarantine_eventless_timeout(self, session)
|
|
2211
|
+
session.closed = True
|
|
2212
|
+
await self._fail_http_bridge_reader_and_maybe_retire(
|
|
2213
|
+
session,
|
|
2214
|
+
error_code="upstream_request_timeout",
|
|
2215
|
+
error_message=receive_timeout.error_message,
|
|
2216
|
+
penalize_account=False,
|
|
2217
|
+
retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
|
|
2218
|
+
force_retire=True,
|
|
2219
|
+
retry_circuit_attempt_selection=expired_retry_circuit_attempt_selection,
|
|
2220
|
+
)
|
|
2221
|
+
break
|
|
2222
|
+
# A successfully cancelled receive cannot deliver
|
|
2223
|
+
# a late response event, so this is the safe point
|
|
2224
|
+
# to persist the durable-anchor invalidation.
|
|
2225
|
+
if expired_proxy_injected_anchor:
|
|
2226
|
+
await _clear_durable_http_bridge_response_anchor(self, session)
|
|
2227
|
+
# Count the eventless retire toward the repeated-
|
|
2228
|
+
# wedge quarantine; the first strike still goes
|
|
2229
|
+
# through the bounded pre-created recovery below.
|
|
2230
|
+
_record_http_bridge_quarantine_eventless_timeout(self, session)
|
|
2231
|
+
_record_http_bridge_stuck_retire(
|
|
2232
|
+
reason=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
|
|
2233
|
+
session=session,
|
|
2234
|
+
)
|
|
2235
|
+
_log_http_bridge_event(
|
|
2236
|
+
"missing_response_created_timeout",
|
|
2237
|
+
session.key,
|
|
2238
|
+
account_id=session.account.id,
|
|
2239
|
+
model=session.request_model,
|
|
2240
|
+
pending_count=pending_count,
|
|
2241
|
+
detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
|
|
2242
|
+
cache_key_family=session.key.affinity_kind,
|
|
2243
|
+
model_class=(
|
|
2244
|
+
_extract_model_class(session.request_model) if session.request_model else None
|
|
2245
|
+
),
|
|
2246
|
+
)
|
|
2247
|
+
# A fresh, self-contained hard request can use the
|
|
2248
|
+
# same bounded pre-created recovery as the idle
|
|
2249
|
+
# timeout path. Keep the session open until the
|
|
2250
|
+
# recovery routine claims the handoff; otherwise
|
|
2251
|
+
# its retry gate would reject the request as
|
|
2252
|
+
# already retired. Continuity-bound requests still
|
|
2253
|
+
# fail closed in _retry_http_bridge_precreated_request.
|
|
2254
|
+
retried = await self._retry_http_bridge_precreated_request(session)
|
|
2255
|
+
if retried:
|
|
2256
|
+
continue
|
|
2257
|
+
session.closed = True
|
|
2258
|
+
await self._fail_http_bridge_reader_and_maybe_retire(
|
|
2259
|
+
session,
|
|
2260
|
+
error_code="upstream_request_timeout",
|
|
2261
|
+
error_message=receive_timeout.error_message,
|
|
2262
|
+
penalize_account=False,
|
|
2263
|
+
retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
|
|
2264
|
+
force_retire=True,
|
|
2265
|
+
retry_circuit_attempt_selection=expired_retry_circuit_attempt_selection,
|
|
2266
|
+
)
|
|
2267
|
+
break
|
|
2268
|
+
|
|
2269
|
+
async with session.pending_lock:
|
|
2270
|
+
retry_circuit_attempt_selection = (
|
|
2271
|
+
_http_bridge_retry_circuit_attempt_selection_for_pending_requests(
|
|
2272
|
+
tuple(session.pending_requests)
|
|
2273
|
+
)
|
|
2274
|
+
)
|
|
2275
|
+
reader_failure_retry_circuit_attempt_selection = retry_circuit_attempt_selection
|
|
2276
|
+
if receive_task is not None:
|
|
2277
|
+
receive_cancelled = await _cancel_http_bridge_reader_child(
|
|
2278
|
+
receive_task,
|
|
2279
|
+
label="HTTP bridge upstream receive after timeout",
|
|
2280
|
+
cleanup_tasks=self._background_cleanup_tasks,
|
|
2281
|
+
scheduler_owner=self,
|
|
2282
|
+
)
|
|
2283
|
+
if not receive_cancelled:
|
|
2284
|
+
raise RuntimeError("HTTP bridge upstream receive did not cancel after timeout")
|
|
2285
|
+
receive_task = None
|
|
2286
|
+
retried = await self._retry_http_bridge_precreated_request(session)
|
|
2287
|
+
if retried:
|
|
2288
|
+
continue
|
|
2289
|
+
async with session.lifecycle_lock:
|
|
2290
|
+
await self._fail_http_bridge_reader_and_maybe_retire(
|
|
2291
|
+
session,
|
|
2292
|
+
error_code=receive_timeout.error_code,
|
|
2293
|
+
error_message=receive_timeout.error_message,
|
|
2294
|
+
retry_circuit_attempt_selection=retry_circuit_attempt_selection,
|
|
2295
|
+
)
|
|
2296
|
+
break
|
|
2297
|
+
|
|
2298
|
+
if message is None:
|
|
2299
|
+
raise RuntimeError("HTTP bridge upstream receive completed without a message")
|
|
2300
|
+
if message.kind == "text" and message.text is not None:
|
|
2301
|
+
session.last_upstream_close_code = None
|
|
2302
|
+
if EVENT_MARKER in message.text:
|
|
2303
|
+
publish_live_usage(
|
|
2304
|
+
parse_rate_limit_event_text(message.text),
|
|
2305
|
+
account_id=session.account.id,
|
|
2306
|
+
chatgpt_account_id=session.account.chatgpt_account_id,
|
|
2307
|
+
)
|
|
2308
|
+
await self._process_http_bridge_upstream_text(
|
|
2309
|
+
session,
|
|
2310
|
+
message.text,
|
|
2311
|
+
message=message,
|
|
2312
|
+
scheduler=scheduler,
|
|
2313
|
+
clock=clock,
|
|
2314
|
+
)
|
|
2315
|
+
if await self._retire_http_bridge_after_drain_if_ready(session):
|
|
2316
|
+
break
|
|
2317
|
+
continue
|
|
2318
|
+
|
|
2319
|
+
async with session.pending_lock:
|
|
2320
|
+
archive_request_state = session.pending_requests[0] if len(session.pending_requests) == 1 else None
|
|
2321
|
+
response_events_seen = max(
|
|
2322
|
+
(request_state.response_event_count for request_state in session.pending_requests),
|
|
2323
|
+
default=0,
|
|
2324
|
+
)
|
|
2325
|
+
# Buffered reasoning preludes are suppressed from
|
|
2326
|
+
# response_event_count on purpose, but they are still
|
|
2327
|
+
# application-layer output: a drop after one is not an
|
|
2328
|
+
# eventless drop for account-health purposes.
|
|
2329
|
+
upstream_output_observed = any(
|
|
2330
|
+
getattr(request_state, "upstream_model_output_seen", False)
|
|
2331
|
+
for request_state in session.pending_requests
|
|
2332
|
+
)
|
|
2333
|
+
accepted_response_pending = any(
|
|
2334
|
+
request_state.response_id is not None and not request_state.awaiting_response_created
|
|
2335
|
+
for request_state in session.pending_requests
|
|
2336
|
+
)
|
|
2337
|
+
reader_failure_retry_circuit_attempt_selection = (
|
|
2338
|
+
_http_bridge_retry_circuit_attempt_selection_for_pending_requests(
|
|
2339
|
+
tuple(session.pending_requests)
|
|
2340
|
+
)
|
|
2341
|
+
)
|
|
2342
|
+
_archive_http_bridge_upstream_message(session, message, archive_request_state)
|
|
2343
|
+
session.last_upstream_close_generation += 1
|
|
2344
|
+
session.last_upstream_close_code = message.close_code
|
|
2345
|
+
retried = False
|
|
2346
|
+
# Account-neutral transport failures do not prove that the
|
|
2347
|
+
# upstream rejected response.create. The request may still be
|
|
2348
|
+
# executing, so replay could duplicate work, billing, or tool
|
|
2349
|
+
# side effects. Clean closes remain eligible for the bounded
|
|
2350
|
+
# pre-created retry circuit maintained by the session.
|
|
2351
|
+
account_neutral = is_account_neutral_websocket_error_code(message.error_code)
|
|
2352
|
+
# Only a terminal transport message (close or error) may replay
|
|
2353
|
+
# an accepted turn: a protocol-invalid binary frame did not end
|
|
2354
|
+
# the socket, so it keeps the pre-created retry semantics only.
|
|
2355
|
+
if not account_neutral and (message.kind in {"close", "error"} or not accepted_response_pending):
|
|
2356
|
+
retried = await self._retry_http_bridge_precreated_request(session)
|
|
2357
|
+
if retried:
|
|
2358
|
+
continue
|
|
2359
|
+
close_classification = (
|
|
2360
|
+
_classify_upstream_close(message.close_code, response_events_seen=response_events_seen)
|
|
2361
|
+
if message.close_code is not None
|
|
2362
|
+
else None
|
|
2363
|
+
)
|
|
2364
|
+
# An abrupt drop with no close frame and no response events is
|
|
2365
|
+
# weaker account-health evidence than a graceful pre-created
|
|
2366
|
+
# close, which is already exempted below. Keep the individual
|
|
2367
|
+
# drop account-neutral; repeated eventless drops still feed
|
|
2368
|
+
# the windowed account drain signal inside the failure path.
|
|
2369
|
+
# Only terminal transport messages qualify: a protocol-invalid
|
|
2370
|
+
# binary frame also carries no close code but did not end the
|
|
2371
|
+
# socket, so it keeps the existing penalty semantics.
|
|
2372
|
+
account_neutral_transport_drop = (
|
|
2373
|
+
message.kind in ("close", "error")
|
|
2374
|
+
and not account_neutral
|
|
2375
|
+
and not upstream_output_observed
|
|
2376
|
+
and _is_account_neutral_transport_drop(
|
|
2377
|
+
message.close_code, response_events_seen=response_events_seen
|
|
2378
|
+
)
|
|
2379
|
+
)
|
|
2380
|
+
async with session.lifecycle_lock:
|
|
2381
|
+
if (
|
|
2382
|
+
session.liveness_settlement_owner == "send"
|
|
2383
|
+
and message.error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE
|
|
2384
|
+
):
|
|
2385
|
+
# A submitter publishes this dedicated claim beside the
|
|
2386
|
+
# failing send while holding lifecycle_lock. ``closed``
|
|
2387
|
+
# alone is only an admission/retirement state and must
|
|
2388
|
+
# never suppress settlement of still-pending siblings.
|
|
2389
|
+
break
|
|
2390
|
+
await self._fail_http_bridge_reader_and_maybe_retire(
|
|
2391
|
+
session,
|
|
2392
|
+
error_code=message.error_code or "stream_incomplete",
|
|
2393
|
+
error_message=_upstream_websocket_disconnect_message(message),
|
|
2394
|
+
upstream_close_code=message.close_code,
|
|
2395
|
+
response_events_seen=response_events_seen,
|
|
2396
|
+
transport_classification=(
|
|
2397
|
+
f"websocket_close_{close_classification}"
|
|
2398
|
+
if close_classification is not None
|
|
2399
|
+
else "websocket_transport_error"
|
|
2400
|
+
),
|
|
2401
|
+
retry_circuit_attempt_selection=reader_failure_retry_circuit_attempt_selection,
|
|
2402
|
+
penalize_account=(
|
|
2403
|
+
not account_neutral
|
|
2404
|
+
and not account_neutral_transport_drop
|
|
2405
|
+
and not (message.kind == "close" and close_classification == "clean")
|
|
2406
|
+
),
|
|
2407
|
+
account_neutral_transport_drop=account_neutral_transport_drop,
|
|
2408
|
+
**(
|
|
2409
|
+
# An admission waiter must not inherit a socket whose
|
|
2410
|
+
# heartbeat already proved it dead. Other failures
|
|
2411
|
+
# preserve the existing deferred-retirement handoff.
|
|
2412
|
+
{"force_retire": True}
|
|
2413
|
+
if message.error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE
|
|
2414
|
+
else {}
|
|
2415
|
+
),
|
|
2416
|
+
)
|
|
2417
|
+
break
|
|
2418
|
+
except asyncio.CancelledError:
|
|
2419
|
+
raise
|
|
2420
|
+
except Exception as exc:
|
|
2421
|
+
if reader_failure_retry_circuit_attempt_selection is None:
|
|
2422
|
+
# A receive/processing exception can jump here before the
|
|
2423
|
+
# ordinary timeout or close branches publish their snapshot.
|
|
2424
|
+
# Capture before waiting for lifecycle ownership so a
|
|
2425
|
+
# concurrent recovery cannot replace the failed physical send.
|
|
2426
|
+
async with session.pending_lock:
|
|
2427
|
+
reader_failure_retry_circuit_attempt_selection = (
|
|
2428
|
+
_http_bridge_retry_circuit_attempt_selection_for_pending_requests(
|
|
2429
|
+
tuple(session.pending_requests)
|
|
2430
|
+
)
|
|
2431
|
+
)
|
|
2432
|
+
logger.warning(
|
|
2433
|
+
"HTTP bridge upstream reader crashed account_id=%s bridge_kind=%s",
|
|
2434
|
+
session.account.id,
|
|
2435
|
+
session.key.affinity_kind,
|
|
2436
|
+
exc_info=True,
|
|
2437
|
+
)
|
|
2438
|
+
error_code = exc.error_code if isinstance(exc, UpstreamWebSocketTransportError) else "stream_incomplete"
|
|
2439
|
+
account_neutral = is_account_neutral_websocket_error_code(error_code)
|
|
2440
|
+
async with session.lifecycle_lock:
|
|
2441
|
+
if not (
|
|
2442
|
+
session.liveness_settlement_owner == "send"
|
|
2443
|
+
and error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE
|
|
2444
|
+
):
|
|
2445
|
+
# Match the message path above when receive() raises while
|
|
2446
|
+
# a concurrent send failure already owns settlement.
|
|
2447
|
+
await self._fail_http_bridge_reader_and_maybe_retire(
|
|
2448
|
+
session,
|
|
2449
|
+
error_code=error_code,
|
|
2450
|
+
error_message=(
|
|
2451
|
+
str(exc)
|
|
2452
|
+
if isinstance(exc, UpstreamWebSocketTransportError)
|
|
2453
|
+
else "HTTP bridge upstream reader crashed before response.completed"
|
|
2454
|
+
),
|
|
2455
|
+
penalize_account=not account_neutral,
|
|
2456
|
+
retry_circuit_attempt_selection=reader_failure_retry_circuit_attempt_selection,
|
|
2457
|
+
# Preserve ordinary crash handoff behavior, but never hand
|
|
2458
|
+
# a heartbeat-expired socket to an admission waiter.
|
|
2459
|
+
**({"force_retire": True} if error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE else {}),
|
|
2460
|
+
)
|
|
2461
|
+
finally:
|
|
2462
|
+
# Mark the socket this reader owned closed before the child
|
|
2463
|
+
# cancellations below suspend: the persistent wakeup waiter is
|
|
2464
|
+
# normally still pending here, and a recovery that replaces the
|
|
2465
|
+
# socket during that suspension clears the flag itself after
|
|
2466
|
+
# swapping ``session.upstream``.
|
|
2467
|
+
if session.upstream is relay_upstream:
|
|
2468
|
+
session.closed = True
|
|
2469
|
+
await _cancel_http_bridge_reader_child(
|
|
2470
|
+
wakeup_task,
|
|
2471
|
+
label="HTTP bridge reader wakeup wait",
|
|
2472
|
+
cleanup_tasks=self._background_cleanup_tasks,
|
|
2473
|
+
scheduler_owner=self,
|
|
2474
|
+
)
|
|
2475
|
+
await _cancel_http_bridge_reader_child(
|
|
2476
|
+
receive_task,
|
|
2477
|
+
label="HTTP bridge upstream receive",
|
|
2478
|
+
cleanup_tasks=self._background_cleanup_tasks,
|
|
2479
|
+
scheduler_owner=self,
|
|
2480
|
+
)
|
|
2481
|
+
|
|
2482
|
+
async def _process_http_bridge_upstream_text(
|
|
2483
|
+
self: Any,
|
|
2484
|
+
session: "_HTTPBridgeSession",
|
|
2485
|
+
text: str,
|
|
2486
|
+
*,
|
|
2487
|
+
message: UpstreamWebSocketMessage | None = None,
|
|
2488
|
+
scheduler: Scheduler | None = None,
|
|
2489
|
+
clock: Clock | None = None,
|
|
2490
|
+
) -> None:
|
|
2491
|
+
# The relay loop resolves the collaborators once per session and passes
|
|
2492
|
+
# them down; the fallback only serves direct callers (tests).
|
|
2493
|
+
if scheduler is None:
|
|
2494
|
+
scheduler = scheduler_for(self)
|
|
2495
|
+
if clock is None:
|
|
2496
|
+
clock = clock_for(self)
|
|
2497
|
+
# One JSON document per websocket text frame: parse it directly instead
|
|
2498
|
+
# of framing it as SSE and running the line parser over it. The
|
|
2499
|
+
# data-only block is what unmatched events relay.
|
|
2500
|
+
event_block = f"data: {text}\n\n"
|
|
2501
|
+
if (
|
|
2502
|
+
message is not None
|
|
2503
|
+
and message.responses_interpreted
|
|
2504
|
+
and message.payload is not None
|
|
2505
|
+
and text.startswith("{")
|
|
2506
|
+
and "\n" not in text
|
|
2507
|
+
and "\r" not in text
|
|
2508
|
+
):
|
|
2509
|
+
# Only this shape uses the direct JSON path in the legacy bridge.
|
|
2510
|
+
# Preserve its SSE-field semantics for whitespace/multiline text.
|
|
2511
|
+
payload = message.payload
|
|
2512
|
+
event_type = message.event_type
|
|
2513
|
+
routing = message.routing
|
|
2514
|
+
else:
|
|
2515
|
+
payload = parse_sse_data_json_text(text)
|
|
2516
|
+
event_type = classify_event_type(payload)
|
|
2517
|
+
routing = None
|
|
2518
|
+
event = parse_sse_event_payload(payload) if event_type in _LIFECYCLE_EVENT_TYPES else None
|
|
2519
|
+
completed_delivery_scope = _HTTPBridgeCompletedDeliveryScope() if event_type == "response.completed" else None
|
|
2520
|
+
claimed_terminal_request_states: list[_WebSocketRequestState] = []
|
|
2521
|
+
try:
|
|
2522
|
+
await self._process_parsed_http_bridge_upstream_event(
|
|
2523
|
+
session,
|
|
2524
|
+
text=text,
|
|
2525
|
+
event_block=event_block,
|
|
2526
|
+
payload=payload,
|
|
2527
|
+
event=event,
|
|
2528
|
+
event_type=event_type,
|
|
2529
|
+
response_id=_websocket_response_id(event, payload, routing=routing),
|
|
2530
|
+
completed_delivery_scope=completed_delivery_scope,
|
|
2531
|
+
claimed_terminal_request_states=claimed_terminal_request_states,
|
|
2532
|
+
scheduler=scheduler,
|
|
2533
|
+
clock=clock,
|
|
2534
|
+
)
|
|
2535
|
+
except BaseException:
|
|
2536
|
+
# Includes CancelledError. A terminal request popped from
|
|
2537
|
+
# session.pending_requests is owned exclusively by this
|
|
2538
|
+
# bookkeeping continuation: the reader's failure path and the
|
|
2539
|
+
# downstream detach backstop only settle requests still in
|
|
2540
|
+
# pending ownership, so an abort here would otherwise leak the
|
|
2541
|
+
# API-key reservation and its heartbeat forever (issue #1594).
|
|
2542
|
+
await self._settle_aborted_http_bridge_terminal_states(
|
|
2543
|
+
session,
|
|
2544
|
+
claimed_terminal_request_states,
|
|
2545
|
+
)
|
|
2546
|
+
raise
|
|
2547
|
+
else:
|
|
2548
|
+
for claimed_request_state in claimed_terminal_request_states:
|
|
2549
|
+
claimed_request_state.terminal_settlement_phase = None
|
|
2550
|
+
finally:
|
|
2551
|
+
if completed_delivery_scope is not None:
|
|
2552
|
+
completed_delivery_scope.active = False
|
|
2553
|
+
|
|
2554
|
+
async def _settle_aborted_http_bridge_terminal_states(
|
|
2555
|
+
self: Any,
|
|
2556
|
+
session: "_HTTPBridgeSession",
|
|
2557
|
+
request_states: list[_WebSocketRequestState],
|
|
2558
|
+
) -> None:
|
|
2559
|
+
"""Settle claimed-but-unfinalized terminal requests after an abort.
|
|
2560
|
+
|
|
2561
|
+
Once terminal-event bookkeeping pops a request from
|
|
2562
|
+
``session.pending_requests`` it is the sole owner of that request's
|
|
2563
|
+
reservation settlement. If the bookkeeping continuation raises or is
|
|
2564
|
+
cancelled before ``_finalize_websocket_request_state`` transfers the
|
|
2565
|
+
settlement, nothing else can reach the request: the orphaned
|
|
2566
|
+
reservation heartbeat would keep refreshing ``updated_at`` and defeat
|
|
2567
|
+
the stale-reservation janitor permanently. Settle those requests here
|
|
2568
|
+
under a shielded scope. Requests a retry branch restored to pending
|
|
2569
|
+
ownership are skipped; they are reachable by ordinary cleanup again.
|
|
2570
|
+
Settlement is idempotent (release only flips ``status == "reserved"``),
|
|
2571
|
+
so overlapping with an already-transferred finalize is safe.
|
|
2572
|
+
"""
|
|
2573
|
+
if not request_states:
|
|
2574
|
+
return
|
|
2575
|
+
with anyio.CancelScope(shield=True):
|
|
2576
|
+
for request_state in request_states:
|
|
2577
|
+
if request_state.terminal_settlement_phase is None:
|
|
2578
|
+
continue
|
|
2579
|
+
async with session.pending_lock:
|
|
2580
|
+
if request_state in session.pending_requests:
|
|
2581
|
+
request_state.terminal_settlement_phase = None
|
|
2582
|
+
continue
|
|
2583
|
+
request_state.terminal_settlement_phase = "abandoned"
|
|
2584
|
+
try:
|
|
2585
|
+
self._cancel_request_state_api_key_reservation_heartbeat(request_state)
|
|
2586
|
+
await self._release_websocket_request_state_reservation(request_state)
|
|
2587
|
+
request_state.api_key_reservation = None
|
|
2588
|
+
request_state.terminal_settlement_phase = None
|
|
2589
|
+
except Exception:
|
|
2590
|
+
# Leave the "abandoned" marker so the downstream detach
|
|
2591
|
+
# backstop can retry the settlement later.
|
|
2592
|
+
logger.warning(
|
|
2593
|
+
"Failed to settle aborted HTTP bridge terminal request request_id=%s account_id=%s",
|
|
2594
|
+
request_state.request_log_id or request_state.request_id,
|
|
2595
|
+
session.account.id,
|
|
2596
|
+
exc_info=True,
|
|
2597
|
+
)
|
|
2598
|
+
# Best effort: unblock the downstream waiter so it observes
|
|
2599
|
+
# end-of-stream instead of waiting for its idle timeout.
|
|
2600
|
+
event_queue = request_state.event_queue
|
|
2601
|
+
if event_queue is not None:
|
|
2602
|
+
try:
|
|
2603
|
+
event_queue.put_nowait(None)
|
|
2604
|
+
except asyncio.QueueFull:
|
|
2605
|
+
pass
|
|
2606
|
+
|
|
2607
|
+
async def _handle_or_defer_precreated_stream_health(
|
|
2608
|
+
self: Any,
|
|
2609
|
+
request_state: _WebSocketRequestState | None,
|
|
2610
|
+
account: Account,
|
|
2611
|
+
error: UpstreamError,
|
|
2612
|
+
code: str,
|
|
2613
|
+
) -> None:
|
|
2614
|
+
"""Write account health now, or defer it until reservation settlement.
|
|
2615
|
+
|
|
2616
|
+
Keyed pre-created retry branches run while the request's API-key
|
|
2617
|
+
reservation is still open. An immediate ``_handle_stream_error`` would
|
|
2618
|
+
mutate account health before the reservation settles, violating the
|
|
2619
|
+
settlement-ordering invariant (api-keys spec: the health write waits
|
|
2620
|
+
for settlement, and unconfirmed settlement leaves health unapplied).
|
|
2621
|
+
Queue the classified write on the request state instead;
|
|
2622
|
+
``_drain_deferred_keyed_stream_health`` applies it after settlement or
|
|
2623
|
+
fallback release commits. Unkeyed requests keep the immediate write.
|
|
2624
|
+
``account_health_error_handled`` is set either way so terminal
|
|
2625
|
+
finalization does not apply a duplicate penalty.
|
|
2626
|
+
"""
|
|
2627
|
+
if request_state is not None and request_state.api_key_reservation is not None:
|
|
2628
|
+
request_state.deferred_keyed_stream_health.append(
|
|
2629
|
+
_DeferredKeyedStreamHealthPenalty(account=account, error=error, code=code)
|
|
2630
|
+
)
|
|
2631
|
+
else:
|
|
2632
|
+
await self._handle_stream_error(account, error, code)
|
|
2633
|
+
if request_state is not None:
|
|
2634
|
+
setattr(request_state, "account_health_error_handled", True)
|
|
2635
|
+
|
|
2636
|
+
async def _process_parsed_http_bridge_upstream_event(
|
|
2637
|
+
self: Any,
|
|
2638
|
+
session: "_HTTPBridgeSession",
|
|
2639
|
+
*,
|
|
2640
|
+
text: str,
|
|
2641
|
+
event_block: str,
|
|
2642
|
+
payload: dict[str, JsonValue] | None,
|
|
2643
|
+
event: OpenAIEvent | None,
|
|
2644
|
+
event_type: str | None,
|
|
2645
|
+
response_id: str | None,
|
|
2646
|
+
completed_delivery_scope: _HTTPBridgeCompletedDeliveryScope | None,
|
|
2647
|
+
claimed_terminal_request_states: list[_WebSocketRequestState],
|
|
2648
|
+
scheduler: Scheduler,
|
|
2649
|
+
clock: Clock,
|
|
2650
|
+
) -> None:
|
|
2651
|
+
original_text = text
|
|
2652
|
+
error_message = _websocket_event_error_message(event_type, payload)
|
|
2653
|
+
is_typeless_error_event = (
|
|
2654
|
+
isinstance(payload, dict)
|
|
2655
|
+
and not isinstance(payload.get("type"), str)
|
|
2656
|
+
and isinstance(payload.get("error"), dict)
|
|
2657
|
+
)
|
|
2658
|
+
is_previous_response_not_found_event = _is_previous_response_not_found_error(
|
|
2659
|
+
code=_normalize_error_code(
|
|
2660
|
+
_websocket_event_error_code(event_type, payload),
|
|
2661
|
+
_websocket_event_error_type(event_type, payload),
|
|
2662
|
+
),
|
|
2663
|
+
param=_websocket_event_error_param(event_type, payload),
|
|
2664
|
+
message=error_message,
|
|
2665
|
+
)
|
|
2666
|
+
is_missing_tool_output_event = _is_missing_tool_output_error(
|
|
2667
|
+
code=_normalize_error_code(
|
|
2668
|
+
_websocket_event_error_code(event_type, payload),
|
|
2669
|
+
_websocket_event_error_type(event_type, payload),
|
|
2670
|
+
),
|
|
2671
|
+
param=_websocket_event_error_param(event_type, payload),
|
|
2672
|
+
message=error_message,
|
|
2673
|
+
)
|
|
2674
|
+
previous_response_id_hint = _previous_response_id_from_not_found_message(error_message)
|
|
2675
|
+
text, payload, event, event_type, event_block = rewrite_parallel_tool_call_text(
|
|
2676
|
+
text,
|
|
2677
|
+
payload,
|
|
2678
|
+
event_block=event_block,
|
|
2679
|
+
event=event,
|
|
2680
|
+
)
|
|
2681
|
+
|
|
2682
|
+
completed_event_queue: asyncio.Queue[str | None] | None = None
|
|
2683
|
+
completed_event_queue_claimed = False
|
|
2684
|
+
async with session.pending_lock:
|
|
2685
|
+
matched_request_state = None
|
|
2686
|
+
created_request_state = None
|
|
2687
|
+
suppress_downstream_event = False
|
|
2688
|
+
deferred_reasoning_prelude_event = False
|
|
2689
|
+
has_other_pending_requests = False
|
|
2690
|
+
grouped_previous_response_request_states: list[_WebSocketRequestState] = []
|
|
2691
|
+
anonymous_event_prefers_draining = event_type not in {"response.failed", "response.incomplete", "error"}
|
|
2692
|
+
if event_type == "response.created":
|
|
2693
|
+
matched_request_state = _assign_websocket_response_id(session.pending_requests, response_id)
|
|
2694
|
+
created_request_state = matched_request_state
|
|
2695
|
+
release_create_gate = matched_request_state is not None
|
|
2696
|
+
elif response_id is not None:
|
|
2697
|
+
matched_request_state = _find_websocket_request_state_by_response_id(
|
|
2698
|
+
session.pending_requests,
|
|
2699
|
+
response_id,
|
|
2700
|
+
)
|
|
2701
|
+
release_create_gate = False
|
|
2702
|
+
elif response_id is None:
|
|
2703
|
+
matched_request_state = _match_websocket_request_state_for_anonymous_event(
|
|
2704
|
+
session.pending_requests,
|
|
2705
|
+
prefer_previous_response_not_found=is_previous_response_not_found_event
|
|
2706
|
+
or is_missing_tool_output_event,
|
|
2707
|
+
previous_response_id_hint=previous_response_id_hint,
|
|
2708
|
+
error_message=error_message,
|
|
2709
|
+
allow_unanchored_previous_response_error=is_previous_response_not_found_event,
|
|
2710
|
+
prefer_draining_requests=anonymous_event_prefers_draining,
|
|
2711
|
+
event_type=event_type,
|
|
2712
|
+
)
|
|
2713
|
+
release_create_gate = False
|
|
2714
|
+
else:
|
|
2715
|
+
release_create_gate = False
|
|
2716
|
+
|
|
2717
|
+
_archive_http_bridge_upstream_text(session, original_text, matched_request_state)
|
|
2718
|
+
pending_request_count = len(session.pending_requests)
|
|
2719
|
+
|
|
2720
|
+
if matched_request_state is not None:
|
|
2721
|
+
# The deferred reasoning prelude intentionally skips ordinary
|
|
2722
|
+
# response-event accounting below, but it still proves that the
|
|
2723
|
+
# physical response.create received an upstream response. Publish
|
|
2724
|
+
# that attempt transition before any later recovery await can
|
|
2725
|
+
# classify the send as eventless.
|
|
2726
|
+
_mark_response_create_attempt_observed(matched_request_state, event_type)
|
|
2727
|
+
session.last_upstream_event_generation += 1
|
|
2728
|
+
now = clock.monotonic()
|
|
2729
|
+
if matched_request_state.latency_first_upstream_event_ms is None:
|
|
2730
|
+
matched_request_state.latency_first_upstream_event_ms = int(
|
|
2731
|
+
max(0.0, now - matched_request_state.started_at) * 1000
|
|
2732
|
+
)
|
|
2733
|
+
if event_type == "response.created" and matched_request_state.latency_response_created_ms is None:
|
|
2734
|
+
matched_request_state.latency_response_created_ms = int(
|
|
2735
|
+
max(0.0, now - matched_request_state.started_at) * 1000
|
|
2736
|
+
)
|
|
2737
|
+
actual_service_tier = _service_tier_from_event_payload(payload)
|
|
2738
|
+
if actual_service_tier is not None:
|
|
2739
|
+
matched_request_state.actual_service_tier = actual_service_tier
|
|
2740
|
+
matched_request_state.service_tier = actual_service_tier
|
|
2741
|
+
_record_http_bridge_tool_call_lifecycle(
|
|
2742
|
+
matched_request_state,
|
|
2743
|
+
event_type=event_type,
|
|
2744
|
+
payload=payload,
|
|
2745
|
+
)
|
|
2746
|
+
completed_tool_call = _response_output_item_done_tool_call(payload)
|
|
2747
|
+
if completed_tool_call is not None:
|
|
2748
|
+
completed_call_id, completed_call_type = completed_tool_call
|
|
2749
|
+
if completed_call_id not in matched_request_state.pending_function_call_ids:
|
|
2750
|
+
matched_request_state.pending_function_call_ids.append(completed_call_id)
|
|
2751
|
+
matched_request_state.pending_tool_call_types[completed_call_id] = completed_call_type
|
|
2752
|
+
if mark_duplicate_tool_call_downstream_event(
|
|
2753
|
+
payload,
|
|
2754
|
+
seen_tool_call_keys=matched_request_state.seen_tool_call_keys,
|
|
2755
|
+
response_id=tool_call_response_id_from_payload(payload) or matched_request_state.request_id,
|
|
2756
|
+
scope_side_effects_by_response_id=False,
|
|
2757
|
+
):
|
|
2758
|
+
matched_request_state.suppressed_duplicate_tool_call = True
|
|
2759
|
+
return
|
|
2760
|
+
if event_type in _TEXT_DELTA_EVENT_TYPES:
|
|
2761
|
+
matched_request_state.downstream_visible = True
|
|
2762
|
+
if event_type == "response.created" and matched_request_state.suppress_next_created_downstream:
|
|
2763
|
+
matched_request_state.suppress_next_created_downstream = False
|
|
2764
|
+
suppress_downstream_event = True
|
|
2765
|
+
elif (
|
|
2766
|
+
event_type == "response.in_progress" and matched_request_state.suppress_next_in_progress_downstream
|
|
2767
|
+
):
|
|
2768
|
+
matched_request_state.suppress_next_in_progress_downstream = False
|
|
2769
|
+
suppress_downstream_event = True
|
|
2770
|
+
if payload is not None:
|
|
2771
|
+
rewritten_payload = _rewrite_websocket_downstream_response_id(payload, matched_request_state)
|
|
2772
|
+
# ``text`` is the serialization ``payload`` was parsed from (or the
|
|
2773
|
+
# tool-call rewrite's compact re-dump). Relaying it verbatim is
|
|
2774
|
+
# only valid while nothing above mutated ``payload`` in place:
|
|
2775
|
+
# ``mark_duplicate_tool_call_downstream_event`` trims partially
|
|
2776
|
+
# duplicated ``multi_tool_use.parallel`` arguments on
|
|
2777
|
+
# ``response.output_item.done`` items without returning a new
|
|
2778
|
+
# object, so those (rare, per-item) events are always
|
|
2779
|
+
# re-serialized. Every other pre-framing step is read-only.
|
|
2780
|
+
if rewritten_payload is not payload or event_type == "response.output_item.done":
|
|
2781
|
+
payload = rewritten_payload
|
|
2782
|
+
event_block = format_sse_event(payload)
|
|
2783
|
+
else:
|
|
2784
|
+
# Identity fast path: nothing changed, so frame the
|
|
2785
|
+
# upstream JSON text instead of serializing the dict again.
|
|
2786
|
+
event_block = format_sse_event_from_text(payload, text)
|
|
2787
|
+
if _websocket_should_defer_reasoning_prelude(matched_request_state, event_type, payload):
|
|
2788
|
+
matched_request_state.deferred_reasoning_downstream_texts.append(event_block)
|
|
2789
|
+
matched_request_state.last_upstream_activity_at = now
|
|
2790
|
+
matched_request_state.upstream_model_output_seen = True
|
|
2791
|
+
suppress_downstream_event = True
|
|
2792
|
+
deferred_reasoning_prelude_event = True
|
|
2793
|
+
elif event_type in _MODEL_OUTPUT_EVENT_TYPES:
|
|
2794
|
+
matched_request_state.upstream_model_output_seen = True
|
|
2795
|
+
if not suppress_downstream_event:
|
|
2796
|
+
matched_request_state.downstream_visible = True
|
|
2797
|
+
|
|
2798
|
+
terminal_request_state = None
|
|
2799
|
+
if event_type in {"response.completed", "response.failed", "response.incomplete", "error"}:
|
|
2800
|
+
early_retry_error_code = _websocket_precreated_retry_error_code(
|
|
2801
|
+
matched_request_state,
|
|
2802
|
+
event_type=event_type,
|
|
2803
|
+
payload=payload,
|
|
2804
|
+
has_other_pending_requests=any(
|
|
2805
|
+
pending_request is not matched_request_state for pending_request in session.pending_requests
|
|
2806
|
+
),
|
|
2807
|
+
)
|
|
2808
|
+
reserve_terminal_for_model_capacity_retry = bool(
|
|
2809
|
+
matched_request_state is not None
|
|
2810
|
+
and early_retry_error_code is not None
|
|
2811
|
+
and early_retry_error_code != _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
|
|
2812
|
+
and not is_previous_response_not_found_event
|
|
2813
|
+
and is_upstream_model_capacity_error(error_message)
|
|
2814
|
+
and _websocket_request_can_replay_before_visible_output(matched_request_state)
|
|
2815
|
+
and _http_bridge_accepted_capacity_retry_allowed(matched_request_state)
|
|
2816
|
+
)
|
|
2817
|
+
if reserve_terminal_for_model_capacity_retry:
|
|
2818
|
+
terminal_request_state = matched_request_state
|
|
2819
|
+
else:
|
|
2820
|
+
terminal_request_state = _pop_terminal_websocket_request_state(
|
|
2821
|
+
session.pending_requests,
|
|
2822
|
+
response_id=response_id,
|
|
2823
|
+
fallback_request_state=matched_request_state,
|
|
2824
|
+
prefer_previous_response_not_found=is_previous_response_not_found_event
|
|
2825
|
+
or is_missing_tool_output_event,
|
|
2826
|
+
previous_response_id_hint=previous_response_id_hint,
|
|
2827
|
+
error_message=error_message,
|
|
2828
|
+
allow_unanchored_previous_response_error=is_previous_response_not_found_event,
|
|
2829
|
+
allow_precreated_terminal_fallback=True,
|
|
2830
|
+
prefer_draining_requests=anonymous_event_prefers_draining,
|
|
2831
|
+
)
|
|
2832
|
+
if terminal_request_state is not None:
|
|
2833
|
+
# Upstream generation ends here; the durable alias, operation,
|
|
2834
|
+
# recovery and circuit-settlement writes below and the
|
|
2835
|
+
# finalizer's settlement are local and must not stretch the
|
|
2836
|
+
# throughput sample's span. A later terminal for the same turn
|
|
2837
|
+
# (capacity retry) replaces it; those rows are not sampled.
|
|
2838
|
+
terminal_request_state.upstream_terminal_at = clock.monotonic()
|
|
2839
|
+
if (
|
|
2840
|
+
matched_request_state is None
|
|
2841
|
+
and terminal_request_state is not None
|
|
2842
|
+
and response_id is not None
|
|
2843
|
+
and event_type == "response.completed"
|
|
2844
|
+
and terminal_request_state.response_id is None
|
|
2845
|
+
):
|
|
2846
|
+
terminal_request_state.response_id = response_id
|
|
2847
|
+
matched_request_state = terminal_request_state
|
|
2848
|
+
elif (
|
|
2849
|
+
matched_request_state is None
|
|
2850
|
+
and terminal_request_state is not None
|
|
2851
|
+
and response_id is not None
|
|
2852
|
+
and terminal_request_state.response_id == response_id
|
|
2853
|
+
):
|
|
2854
|
+
matched_request_state = terminal_request_state
|
|
2855
|
+
if (
|
|
2856
|
+
terminal_request_state is not None
|
|
2857
|
+
and not reserve_terminal_for_model_capacity_retry
|
|
2858
|
+
and _http_bridge_request_counts_against_queue(terminal_request_state)
|
|
2859
|
+
):
|
|
2860
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
2861
|
+
elif is_previous_response_not_found_event or is_missing_tool_output_event:
|
|
2862
|
+
grouped_previous_response_request_states = _pop_matching_websocket_request_states(
|
|
2863
|
+
session.pending_requests,
|
|
2864
|
+
_matching_websocket_request_states_for_previous_response_error(
|
|
2865
|
+
session.pending_requests,
|
|
2866
|
+
previous_response_id_hint=previous_response_id_hint,
|
|
2867
|
+
error_message=error_message,
|
|
2868
|
+
allow_unanchored_previous_response_error=is_previous_response_not_found_event,
|
|
2869
|
+
),
|
|
2870
|
+
)
|
|
2871
|
+
if not grouped_previous_response_request_states and is_missing_tool_output_event:
|
|
2872
|
+
grouped_previous_response_request_states = _pop_matching_websocket_request_states(
|
|
2873
|
+
session.pending_requests,
|
|
2874
|
+
_matching_websocket_request_states_for_missing_tool_output_error(
|
|
2875
|
+
session.pending_requests,
|
|
2876
|
+
),
|
|
2877
|
+
)
|
|
2878
|
+
if grouped_previous_response_request_states:
|
|
2879
|
+
grouped_counted_requests = sum(
|
|
2880
|
+
1
|
|
2881
|
+
for grouped_request_state in grouped_previous_response_request_states
|
|
2882
|
+
if _http_bridge_request_counts_against_queue(grouped_request_state)
|
|
2883
|
+
)
|
|
2884
|
+
session.queued_request_count = max(
|
|
2885
|
+
0,
|
|
2886
|
+
session.queued_request_count - grouped_counted_requests,
|
|
2887
|
+
)
|
|
2888
|
+
if (
|
|
2889
|
+
terminal_request_state is None
|
|
2890
|
+
and event_type == "error"
|
|
2891
|
+
and is_typeless_error_event
|
|
2892
|
+
and not grouped_previous_response_request_states
|
|
2893
|
+
):
|
|
2894
|
+
grouped_previous_response_request_states = list(session.pending_requests)
|
|
2895
|
+
session.pending_requests.clear()
|
|
2896
|
+
if grouped_previous_response_request_states:
|
|
2897
|
+
grouped_counted_requests = sum(
|
|
2898
|
+
1
|
|
2899
|
+
for grouped_request_state in grouped_previous_response_request_states
|
|
2900
|
+
if _http_bridge_request_counts_against_queue(grouped_request_state)
|
|
2901
|
+
)
|
|
2902
|
+
session.queued_request_count = max(
|
|
2903
|
+
0,
|
|
2904
|
+
session.queued_request_count - grouped_counted_requests,
|
|
2905
|
+
)
|
|
2906
|
+
has_other_pending_requests = any(
|
|
2907
|
+
pending_request is not terminal_request_state for pending_request in session.pending_requests
|
|
2908
|
+
)
|
|
2909
|
+
if (
|
|
2910
|
+
event_type == "response.completed"
|
|
2911
|
+
and terminal_request_state is not None
|
|
2912
|
+
and terminal_request_state not in session.pending_requests
|
|
2913
|
+
):
|
|
2914
|
+
completed_event_queue = terminal_request_state.event_queue
|
|
2915
|
+
completed_event_queue_claimed = True
|
|
2916
|
+
if completed_event_queue is not None and completed_delivery_scope is not None:
|
|
2917
|
+
completed_delivery_scope.active = True
|
|
2918
|
+
terminal_request_state.completed_delivery_scope = completed_delivery_scope
|
|
2919
|
+
|
|
2920
|
+
# Every request popped from pending ownership above is now owned
|
|
2921
|
+
# exclusively by this bookkeeping continuation. Record the claim
|
|
2922
|
+
# (still under pending_lock) so an abort before finalization can
|
|
2923
|
+
# settle the reservation instead of leaking it (issue #1594).
|
|
2924
|
+
for claimed_request_state in (terminal_request_state, *grouped_previous_response_request_states):
|
|
2925
|
+
if claimed_request_state is not None and claimed_request_state not in session.pending_requests:
|
|
2926
|
+
claimed_request_state.terminal_settlement_phase = "claimed"
|
|
2927
|
+
if claimed_request_state not in claimed_terminal_request_states:
|
|
2928
|
+
claimed_terminal_request_states.append(claimed_request_state)
|
|
2929
|
+
|
|
2930
|
+
if len(grouped_previous_response_request_states) > 1:
|
|
2931
|
+
session.upstream_control.reconnect_requested = True
|
|
2932
|
+
if is_previous_response_not_found_event:
|
|
2933
|
+
# This branch settles every request that shared the denied
|
|
2934
|
+
# anchor and then returns, so the single-request retirement
|
|
2935
|
+
# below is never reached for a fan-out denial.
|
|
2936
|
+
await _retire_denied_http_bridge_anchor(
|
|
2937
|
+
self,
|
|
2938
|
+
session,
|
|
2939
|
+
request_states=grouped_previous_response_request_states,
|
|
2940
|
+
)
|
|
2941
|
+
grouped_error_reason = (
|
|
2942
|
+
"previous_response_not_found"
|
|
2943
|
+
if is_previous_response_not_found_event
|
|
2944
|
+
else "missing_tool_output"
|
|
2945
|
+
if is_missing_tool_output_event
|
|
2946
|
+
else "stream_incomplete"
|
|
2947
|
+
)
|
|
2948
|
+
grouped_terminal_events = []
|
|
2949
|
+
for grouped_request_state in grouped_previous_response_request_states:
|
|
2950
|
+
grouped_request_state.error_http_status_override = 502
|
|
2951
|
+
(
|
|
2952
|
+
_grouped_downstream_text,
|
|
2953
|
+
grouped_event_block,
|
|
2954
|
+
grouped_event,
|
|
2955
|
+
grouped_payload,
|
|
2956
|
+
grouped_event_type,
|
|
2957
|
+
) = _build_stream_incomplete_terminal_event_for_request(
|
|
2958
|
+
grouped_request_state,
|
|
2959
|
+
reason=grouped_error_reason,
|
|
2960
|
+
)
|
|
2961
|
+
grouped_operation_state = _http_bridge_operation_state_for_event(grouped_event_type)
|
|
2962
|
+
grouped_terminal_events.append(
|
|
2963
|
+
(
|
|
2964
|
+
grouped_request_state,
|
|
2965
|
+
grouped_event_block,
|
|
2966
|
+
grouped_event,
|
|
2967
|
+
grouped_payload,
|
|
2968
|
+
grouped_event_type,
|
|
2969
|
+
grouped_operation_state,
|
|
2970
|
+
)
|
|
2971
|
+
)
|
|
2972
|
+
|
|
2973
|
+
# This grouped settlement returns before the single-request
|
|
2974
|
+
# settlement path below, so without recording here a multi-request
|
|
2975
|
+
# continuity failure fails every grouped request with a synthetic
|
|
2976
|
+
# terminal event and never advances the circuit — leaving the anchor
|
|
2977
|
+
# that failed them reusable. The detail is read back off the built
|
|
2978
|
+
# event so it matches whatever the single-request path would have
|
|
2979
|
+
# recorded for the same reason, and recording precedes the persist
|
|
2980
|
+
# and delivery machinery below for the same reason it does there: a
|
|
2981
|
+
# client resending on observed completion must not outrun it.
|
|
2982
|
+
grouped_poison_strike_failures = 0
|
|
2983
|
+
grouped_poison_detail: str | None = None
|
|
2984
|
+
for (
|
|
2985
|
+
grouped_request_state,
|
|
2986
|
+
_grouped_terminal_block,
|
|
2987
|
+
grouped_terminal_event,
|
|
2988
|
+
_grouped_terminal_payload,
|
|
2989
|
+
_grouped_terminal_event_type,
|
|
2990
|
+
_grouped_terminal_operation_state,
|
|
2991
|
+
) in grouped_terminal_events:
|
|
2992
|
+
# Same admission test the single-request settlement path
|
|
2993
|
+
# applies: a request still holding a verified full resend is
|
|
2994
|
+
# about to be replayed and claims the circuit generation at
|
|
2995
|
+
# dispatch, so charging it here would let two safely
|
|
2996
|
+
# replayable requests open the circuit between them and clear
|
|
2997
|
+
# the anchor both of them could still have used.
|
|
2998
|
+
if grouped_request_state.response_event_count != 0 or _http_bridge_request_state_holds_safe_replay(
|
|
2999
|
+
grouped_request_state
|
|
3000
|
+
):
|
|
3001
|
+
continue
|
|
3002
|
+
if grouped_request_state.request_kind == "prewarm" or grouped_request_state.skip_request_log:
|
|
3003
|
+
# Same exclusion the single-request terminal branch
|
|
3004
|
+
# applies: an internal warmup probe carries no anchor
|
|
3005
|
+
# and proves nothing about the key's continuity, and
|
|
3006
|
+
# one warmup plus one client request must not open the
|
|
3007
|
+
# circuit between them after a single real failure.
|
|
3008
|
+
continue
|
|
3009
|
+
grouped_terminal_error = (
|
|
3010
|
+
grouped_terminal_event.response.error
|
|
3011
|
+
if grouped_terminal_event is not None and grouped_terminal_event.response is not None
|
|
3012
|
+
else None
|
|
3013
|
+
)
|
|
3014
|
+
grouped_terminal_detail = _normalize_error_code(
|
|
3015
|
+
grouped_terminal_error.code if grouped_terminal_error else None,
|
|
3016
|
+
grouped_terminal_error.type if grouped_terminal_error else None,
|
|
3017
|
+
)
|
|
3018
|
+
if grouped_terminal_detail is None:
|
|
3019
|
+
continue
|
|
3020
|
+
grouped_strike_failures = await self._record_http_bridge_retry_circuit_failure(
|
|
3021
|
+
session,
|
|
3022
|
+
detail=grouped_terminal_detail,
|
|
3023
|
+
attempt=grouped_request_state.response_create_attempt,
|
|
3024
|
+
terminal_pre_response_frame=True,
|
|
3025
|
+
)
|
|
3026
|
+
# A fan-out carrying two or more eventless requests advances the
|
|
3027
|
+
# circuit through its threshold inside this loop, so the anchor
|
|
3028
|
+
# that failed all of them is proven dead here just as it is on
|
|
3029
|
+
# the single-request path. Keep the highest count and its poison
|
|
3030
|
+
# detail; discarding them left the grouped branch returning with
|
|
3031
|
+
# the durable anchor still stored.
|
|
3032
|
+
grouped_poison_candidate = await self._http_bridge_effective_anchor_poison_detail(
|
|
3033
|
+
session, grouped_terminal_detail
|
|
3034
|
+
)
|
|
3035
|
+
if grouped_poison_candidate is not None and grouped_strike_failures is not None:
|
|
3036
|
+
if grouped_strike_failures > grouped_poison_strike_failures:
|
|
3037
|
+
grouped_poison_strike_failures = grouped_strike_failures
|
|
3038
|
+
grouped_poison_detail = grouped_poison_candidate
|
|
3039
|
+
|
|
3040
|
+
append_terminal_batch = getattr(
|
|
3041
|
+
getattr(self, "_http_bridge_operation_event_batcher", None),
|
|
3042
|
+
"append_terminal_event",
|
|
3043
|
+
None,
|
|
3044
|
+
)
|
|
3045
|
+
append_participants = {
|
|
3046
|
+
id(grouped_request_state)
|
|
3047
|
+
for grouped_request_state, *_rest in grouped_terminal_events
|
|
3048
|
+
if grouped_request_state.operation_id
|
|
3049
|
+
and session.durable_session_id is not None
|
|
3050
|
+
and session.durable_owner_epoch is not None
|
|
3051
|
+
and callable(append_terminal_batch)
|
|
3052
|
+
}
|
|
3053
|
+
append_ready = asyncio.Event()
|
|
3054
|
+
append_lock = asyncio.Lock()
|
|
3055
|
+
append_arrivals = 0
|
|
3056
|
+
if not append_participants:
|
|
3057
|
+
append_ready.set()
|
|
3058
|
+
|
|
3059
|
+
async def await_all_grouped_appends() -> None:
|
|
3060
|
+
nonlocal append_arrivals
|
|
3061
|
+
async with append_lock:
|
|
3062
|
+
append_arrivals += 1
|
|
3063
|
+
if append_arrivals == len(append_participants):
|
|
3064
|
+
append_ready.set()
|
|
3065
|
+
await append_ready.wait()
|
|
3066
|
+
|
|
3067
|
+
delivery_ready = asyncio.Event()
|
|
3068
|
+
delivery_lock = asyncio.Lock()
|
|
3069
|
+
delivery_arrivals = 0
|
|
3070
|
+
|
|
3071
|
+
async def await_all_grouped_deliveries() -> None:
|
|
3072
|
+
nonlocal delivery_arrivals
|
|
3073
|
+
async with delivery_lock:
|
|
3074
|
+
delivery_arrivals += 1
|
|
3075
|
+
if delivery_arrivals == len(grouped_terminal_events):
|
|
3076
|
+
delivery_ready.set()
|
|
3077
|
+
await delivery_ready.wait()
|
|
3078
|
+
|
|
3079
|
+
async def persist_one_grouped_terminal_event(
|
|
3080
|
+
grouped_terminal_event: tuple[Any, str, OpenAIEvent | None, Any, str | None, str | None],
|
|
3081
|
+
) -> None:
|
|
3082
|
+
(
|
|
3083
|
+
grouped_request_state,
|
|
3084
|
+
grouped_event_block,
|
|
3085
|
+
_grouped_event,
|
|
3086
|
+
_grouped_payload,
|
|
3087
|
+
_grouped_event_type,
|
|
3088
|
+
grouped_operation_state,
|
|
3089
|
+
) = grouped_terminal_event
|
|
3090
|
+
if id(grouped_request_state) in append_participants:
|
|
3091
|
+
await _persist_http_bridge_operation_event(
|
|
3092
|
+
self,
|
|
3093
|
+
session,
|
|
3094
|
+
grouped_request_state,
|
|
3095
|
+
grouped_event_block,
|
|
3096
|
+
terminal=True,
|
|
3097
|
+
terminal_state=grouped_operation_state,
|
|
3098
|
+
terminal_event_queue=grouped_request_state.event_queue,
|
|
3099
|
+
terminal_append_barrier=await_all_grouped_appends,
|
|
3100
|
+
terminal_delivery_barrier=await_all_grouped_deliveries,
|
|
3101
|
+
)
|
|
3102
|
+
else:
|
|
3103
|
+
await append_ready.wait()
|
|
3104
|
+
if grouped_request_state.event_queue is not None:
|
|
3105
|
+
await grouped_request_state.event_queue.put(grouped_event_block)
|
|
3106
|
+
await grouped_request_state.event_queue.put(None)
|
|
3107
|
+
await await_all_grouped_deliveries()
|
|
3108
|
+
await _persist_http_bridge_operation_event(
|
|
3109
|
+
self,
|
|
3110
|
+
session,
|
|
3111
|
+
grouped_request_state,
|
|
3112
|
+
grouped_event_block,
|
|
3113
|
+
terminal=True,
|
|
3114
|
+
terminal_state=grouped_operation_state,
|
|
3115
|
+
)
|
|
3116
|
+
if grouped_operation_state is not None and grouped_operation_state != "failed":
|
|
3117
|
+
await _update_http_bridge_operation_state(
|
|
3118
|
+
self,
|
|
3119
|
+
session,
|
|
3120
|
+
grouped_request_state,
|
|
3121
|
+
state=grouped_operation_state,
|
|
3122
|
+
response_id=_websocket_downstream_response_id(grouped_request_state),
|
|
3123
|
+
)
|
|
3124
|
+
|
|
3125
|
+
async def persist_grouped_terminal_events() -> Exception | None:
|
|
3126
|
+
first_error: Exception | None = None
|
|
3127
|
+
# Spawned through the scheduler (``gather`` would ``ensure_future``
|
|
3128
|
+
# each coroutine into an unowned task) so a simulation owns the
|
|
3129
|
+
# per-request persistence work of a grouped terminal.
|
|
3130
|
+
persistence_results = await asyncio.gather(
|
|
3131
|
+
*(
|
|
3132
|
+
scheduler.create_task(
|
|
3133
|
+
persist_one_grouped_terminal_event(item),
|
|
3134
|
+
name=f"http-bridge-grouped-terminal-persist-{session.durable_session_id}",
|
|
3135
|
+
)
|
|
3136
|
+
for item in grouped_terminal_events
|
|
3137
|
+
),
|
|
3138
|
+
return_exceptions=True,
|
|
3139
|
+
)
|
|
3140
|
+
for persistence_result in persistence_results:
|
|
3141
|
+
if isinstance(persistence_result, Exception) and first_error is None:
|
|
3142
|
+
first_error = persistence_result
|
|
3143
|
+
try:
|
|
3144
|
+
for (
|
|
3145
|
+
grouped_request_state,
|
|
3146
|
+
_grouped_event_block,
|
|
3147
|
+
grouped_event,
|
|
3148
|
+
grouped_payload,
|
|
3149
|
+
grouped_event_type,
|
|
3150
|
+
_grouped_operation_state,
|
|
3151
|
+
) in grouped_terminal_events:
|
|
3152
|
+
try:
|
|
3153
|
+
await self._finalize_websocket_request_state(
|
|
3154
|
+
grouped_request_state,
|
|
3155
|
+
account=session.account,
|
|
3156
|
+
account_id_value=session.account.id,
|
|
3157
|
+
event=grouped_event,
|
|
3158
|
+
event_type=grouped_event_type,
|
|
3159
|
+
payload=grouped_payload,
|
|
3160
|
+
api_key=grouped_request_state.api_key,
|
|
3161
|
+
upstream_control=session.upstream_control,
|
|
3162
|
+
response_create_gate=session.response_create_gate,
|
|
3163
|
+
)
|
|
3164
|
+
except Exception as exc:
|
|
3165
|
+
if first_error is None:
|
|
3166
|
+
first_error = exc
|
|
3167
|
+
except Exception as exc:
|
|
3168
|
+
if first_error is None:
|
|
3169
|
+
first_error = exc
|
|
3170
|
+
try:
|
|
3171
|
+
# Grouped terminal errors settle detached/abandoned requests
|
|
3172
|
+
# (event_queue is None) with no downstream stream finalizer
|
|
3173
|
+
# left to run, so release the now-idle session's account
|
|
3174
|
+
# stream lease here just like the single terminal path below.
|
|
3175
|
+
await self._maybe_release_idle_http_bridge_session_lease(session)
|
|
3176
|
+
except Exception as exc:
|
|
3177
|
+
if first_error is None:
|
|
3178
|
+
first_error = exc
|
|
3179
|
+
return first_error
|
|
3180
|
+
|
|
3181
|
+
grouped_settlement_task = scheduler.create_task(
|
|
3182
|
+
persist_grouped_terminal_events(),
|
|
3183
|
+
name=f"http-bridge-grouped-terminal-settlement-{session.durable_session_id}",
|
|
3184
|
+
)
|
|
3185
|
+
grouped_error, grouped_cancellation = await _await_task_deferring_cancellation(grouped_settlement_task)
|
|
3186
|
+
if grouped_poison_detail is not None and grouped_poison_strike_failures > 0:
|
|
3187
|
+
# The grouped frames have been published by here, so this mirrors
|
|
3188
|
+
# the single-request ordering: the strikes precede delivery, the
|
|
3189
|
+
# durable clear follows it.
|
|
3190
|
+
#
|
|
3191
|
+
# This runs under cancellation too. `_await_task_deferring_
|
|
3192
|
+
# cancellation` has already waited for every grouped frame and
|
|
3193
|
+
# finalization, so the requests this anchor failed are gone and
|
|
3194
|
+
# no later retirement can retry the clear for them; skipping it
|
|
3195
|
+
# here left the poisoned anchor stored for another replica or
|
|
3196
|
+
# for reuse once the local quarantine expired. Defer the
|
|
3197
|
+
# cancellation across the write the same way, then re-raise it
|
|
3198
|
+
# below unchanged. The clear is internally exception-safe.
|
|
3199
|
+
grouped_clear_detail = grouped_poison_detail
|
|
3200
|
+
grouped_clear_strike_failures = grouped_poison_strike_failures
|
|
3201
|
+
|
|
3202
|
+
async def _consult_and_clear_grouped_anchor() -> None:
|
|
3203
|
+
# The episode consult is a durable await of its own: run
|
|
3204
|
+
# it inside the same cancellation-deferred task as the
|
|
3205
|
+
# clear, or a reader cancellation landing mid-consult
|
|
3206
|
+
# escapes with the grouped requests already finalized and
|
|
3207
|
+
# no retirement left to retry the abandonment. The
|
|
3208
|
+
# consult reads the live registered episode, so a
|
|
3209
|
+
# sibling settle during grouped publication vetoes the
|
|
3210
|
+
# clear.
|
|
3211
|
+
grouped_poison_episode, grouped_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
|
|
3212
|
+
session,
|
|
3213
|
+
consecutive_failures=grouped_clear_strike_failures,
|
|
3214
|
+
)
|
|
3215
|
+
if grouped_poison_episode is None:
|
|
3216
|
+
return
|
|
3217
|
+
durable_cleared = await _abandon_durable_http_bridge_continuity(
|
|
3218
|
+
self,
|
|
3219
|
+
session,
|
|
3220
|
+
detail=grouped_clear_detail,
|
|
3221
|
+
expected_continuity=grouped_expected_anchor,
|
|
3222
|
+
authorized_episode=grouped_poison_episode,
|
|
3223
|
+
# A mixed group can hold a member with a verified safe
|
|
3224
|
+
# replay whose dispatch claims the circuit generation;
|
|
3225
|
+
# settling under it would remove the fence it depends
|
|
3226
|
+
# on, same as the retirement funnels.
|
|
3227
|
+
settle_circuit=_http_bridge_abandonment_may_settle_circuit(
|
|
3228
|
+
grouped_previous_response_request_states
|
|
3229
|
+
),
|
|
3230
|
+
)
|
|
3231
|
+
if not durable_cleared:
|
|
3232
|
+
return
|
|
3233
|
+
# A mixed group's abandonment deliberately leaves the
|
|
3234
|
+
# circuit alive for its safe member, so the episode
|
|
3235
|
+
# survives and must remember its anchor was already
|
|
3236
|
+
# cleared — one poisoned anchor is abandoned once. The
|
|
3237
|
+
# marker belongs inside this owned task: a cancellation
|
|
3238
|
+
# landing between the durable clear and the marker write
|
|
3239
|
+
# would otherwise leave the cleared episode unmarked, and
|
|
3240
|
+
# an ordinary load would re-arm quarantine from the
|
|
3241
|
+
# unchanged surviving row after the safe replay had
|
|
3242
|
+
# already recovered the conversation.
|
|
3243
|
+
await self._http_bridge_mark_poison_anchor_cleared(session, episode=grouped_poison_episode)
|
|
3244
|
+
|
|
3245
|
+
grouped_clear_task = scheduler.create_task(
|
|
3246
|
+
_consult_and_clear_grouped_anchor(),
|
|
3247
|
+
name=f"http-bridge-grouped-anchor-clear-{session.durable_session_id}",
|
|
3248
|
+
)
|
|
3249
|
+
_grouped_clear_result, grouped_clear_cancellation = await _await_task_deferring_cancellation(
|
|
3250
|
+
grouped_clear_task
|
|
3251
|
+
)
|
|
3252
|
+
if grouped_cancellation is None:
|
|
3253
|
+
grouped_cancellation = grouped_clear_cancellation
|
|
3254
|
+
if grouped_cancellation is not None:
|
|
3255
|
+
if grouped_error is not None:
|
|
3256
|
+
logger.warning(
|
|
3257
|
+
"Grouped HTTP bridge terminal finalization failed while preserving cancellation error=%r",
|
|
3258
|
+
grouped_error,
|
|
3259
|
+
)
|
|
3260
|
+
raise grouped_cancellation
|
|
3261
|
+
if grouped_error is not None:
|
|
3262
|
+
raise grouped_error
|
|
3263
|
+
return
|
|
3264
|
+
|
|
3265
|
+
if len(grouped_previous_response_request_states) == 1 and terminal_request_state is None:
|
|
3266
|
+
terminal_request_state = grouped_previous_response_request_states[0]
|
|
3267
|
+
|
|
3268
|
+
if not deferred_reasoning_prelude_event:
|
|
3269
|
+
if matched_request_state is terminal_request_state:
|
|
3270
|
+
_record_response_event(matched_request_state, event_type, now=clock.monotonic())
|
|
3271
|
+
else:
|
|
3272
|
+
_record_response_event(matched_request_state, event_type, now=clock.monotonic())
|
|
3273
|
+
_record_response_event(terminal_request_state, event_type, now=clock.monotonic())
|
|
3274
|
+
|
|
3275
|
+
status_request_state = terminal_request_state or matched_request_state
|
|
3276
|
+
if status_request_state is None and is_previous_response_not_found_event:
|
|
3277
|
+
session.upstream_control.reconnect_requested = True
|
|
3278
|
+
return
|
|
3279
|
+
|
|
3280
|
+
if status_request_state is None and pending_request_count:
|
|
3281
|
+
# The bridge multiplexes one upstream connection across pending
|
|
3282
|
+
# requests. An event that reaches none of them is dropped here, and
|
|
3283
|
+
# whatever was waiting for it waits until a timeout fires, so the
|
|
3284
|
+
# drop needs to be visible rather than inferred from a missing
|
|
3285
|
+
# downstream response.
|
|
3286
|
+
#
|
|
3287
|
+
# The frame still proves the upstream transport is alive. Nothing
|
|
3288
|
+
# resets the downstream pre-response silence clock from here (that
|
|
3289
|
+
# clock only sees matched queue items), so record the liveness as an
|
|
3290
|
+
# explicit marker: a later bridge_eventless_timeout with a non-zero
|
|
3291
|
+
# count is a local matching wedge, not a silent upstream.
|
|
3292
|
+
unmatched_liveness_count = _record_http_bridge_unmatched_upstream_liveness(
|
|
3293
|
+
session,
|
|
3294
|
+
event_type=event_type,
|
|
3295
|
+
)
|
|
3296
|
+
logger.warning(
|
|
3297
|
+
"HTTP bridge upstream event matched no pending request account_id=%s bridge_kind=%s "
|
|
3298
|
+
"event_type=%s has_response_id=%s pending_count=%d unmatched_upstream_liveness=%d",
|
|
3299
|
+
session.account.id,
|
|
3300
|
+
session.key.affinity_kind,
|
|
3301
|
+
event_type or "unknown",
|
|
3302
|
+
response_id is not None,
|
|
3303
|
+
pending_request_count,
|
|
3304
|
+
unmatched_liveness_count,
|
|
3305
|
+
)
|
|
3306
|
+
if _http_bridge_event_proves_upstream_liveness(event_type):
|
|
3307
|
+
_log_http_bridge_event(
|
|
3308
|
+
"unmatched_upstream_liveness",
|
|
3309
|
+
session.key,
|
|
3310
|
+
account_id=session.account.id,
|
|
3311
|
+
model=session.request_model,
|
|
3312
|
+
pending_count=pending_request_count,
|
|
3313
|
+
detail=(
|
|
3314
|
+
f"event_type={event_type or 'unknown'} unmatched_upstream_liveness={unmatched_liveness_count}"
|
|
3315
|
+
),
|
|
3316
|
+
cache_key_family=session.key.affinity_kind,
|
|
3317
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
3318
|
+
)
|
|
3319
|
+
|
|
3320
|
+
if status_request_state is not None and event_type not in {
|
|
3321
|
+
"response.completed",
|
|
3322
|
+
"response.failed",
|
|
3323
|
+
"response.incomplete",
|
|
3324
|
+
"error",
|
|
3325
|
+
}:
|
|
3326
|
+
await self._maybe_touch_request_state_api_key_reservation(
|
|
3327
|
+
status_request_state,
|
|
3328
|
+
api_key=status_request_state.api_key,
|
|
3329
|
+
surface="http_bridge",
|
|
3330
|
+
)
|
|
3331
|
+
|
|
3332
|
+
continuity_persistence_failed_after_ack = False
|
|
3333
|
+
if (
|
|
3334
|
+
event_type == "response.completed"
|
|
3335
|
+
and terminal_request_state is not None
|
|
3336
|
+
and terminal_request_state.suppressed_duplicate_tool_call
|
|
3337
|
+
):
|
|
3338
|
+
session.upstream_control.reconnect_requested = True
|
|
3339
|
+
session.closed = True
|
|
3340
|
+
try:
|
|
3341
|
+
await session.upstream.close()
|
|
3342
|
+
except Exception:
|
|
3343
|
+
logger.debug("Failed to close HTTP bridge upstream after suppressed duplicate tool call", exc_info=True)
|
|
3344
|
+
terminal_request_state.error_http_status_override = 502
|
|
3345
|
+
(
|
|
3346
|
+
event,
|
|
3347
|
+
payload,
|
|
3348
|
+
event_type,
|
|
3349
|
+
rewritten_text,
|
|
3350
|
+
) = _rewrite_websocket_suppressed_duplicate_tool_call_completion_event(
|
|
3351
|
+
request_state=terminal_request_state,
|
|
3352
|
+
)
|
|
3353
|
+
event_block = f"data: {rewritten_text}\n\n"
|
|
3354
|
+
|
|
3355
|
+
if (
|
|
3356
|
+
status_request_state is not None
|
|
3357
|
+
and status_request_state.previous_response_id is not None
|
|
3358
|
+
and is_missing_tool_output_event
|
|
3359
|
+
):
|
|
3360
|
+
status_request_state.error_http_status_override = 502
|
|
3361
|
+
event, payload, event_type, rewritten_text = _rewrite_websocket_continuity_corruption_event(
|
|
3362
|
+
request_state=status_request_state,
|
|
3363
|
+
upstream_control=session.upstream_control,
|
|
3364
|
+
reason="missing_tool_output",
|
|
3365
|
+
reconnect_requested=True,
|
|
3366
|
+
original_text=text,
|
|
3367
|
+
)
|
|
3368
|
+
event_block = f"data: {rewritten_text}\n\n"
|
|
3369
|
+
|
|
3370
|
+
if status_request_state is not None and is_previous_response_not_found_event:
|
|
3371
|
+
status_request_state.error_http_status_override = 502
|
|
3372
|
+
status_request_state.previous_response_not_found_rewritten = (
|
|
3373
|
+
response_id is None and not has_other_pending_requests
|
|
3374
|
+
)
|
|
3375
|
+
event, payload, event_type, rewritten_text = _maybe_rewrite_websocket_previous_response_not_found_event(
|
|
3376
|
+
request_state=status_request_state,
|
|
3377
|
+
event=event,
|
|
3378
|
+
payload=payload,
|
|
3379
|
+
event_type=event_type,
|
|
3380
|
+
upstream_control=session.upstream_control,
|
|
3381
|
+
original_text=text,
|
|
3382
|
+
)
|
|
3383
|
+
event_block = f"data: {rewritten_text}\n\n"
|
|
3384
|
+
await _retire_denied_http_bridge_anchor(
|
|
3385
|
+
self,
|
|
3386
|
+
session,
|
|
3387
|
+
request_states=(status_request_state,),
|
|
3388
|
+
)
|
|
3389
|
+
|
|
3390
|
+
retry_error_code = _websocket_precreated_retry_error_code(
|
|
3391
|
+
status_request_state,
|
|
3392
|
+
event_type=event_type,
|
|
3393
|
+
payload=payload,
|
|
3394
|
+
has_other_pending_requests=has_other_pending_requests,
|
|
3395
|
+
)
|
|
3396
|
+
auth_error_code = _websocket_precreated_auth_error_code(
|
|
3397
|
+
status_request_state,
|
|
3398
|
+
event_type=event_type,
|
|
3399
|
+
payload=payload,
|
|
3400
|
+
has_other_pending_requests=has_other_pending_requests,
|
|
3401
|
+
)
|
|
3402
|
+
owner_pinned_quota_error = _websocket_owner_pinned_quota_error_code(
|
|
3403
|
+
status_request_state,
|
|
3404
|
+
event_type=event_type,
|
|
3405
|
+
payload=payload,
|
|
3406
|
+
)
|
|
3407
|
+
retry_error_message = _websocket_event_error_message(event_type, payload)
|
|
3408
|
+
# An accepted anchored request waits only when the pre-created retry
|
|
3409
|
+
# can actually re-send it (proxy-injected anchor); a client-supplied
|
|
3410
|
+
# anchor falls through to the transparent-code branch, which forwards
|
|
3411
|
+
# the capacity terminal unchanged instead of staging a refused retry.
|
|
3412
|
+
wait_for_model_capacity_retry = bool(
|
|
3413
|
+
retry_error_code is not None
|
|
3414
|
+
and retry_error_code != _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
|
|
3415
|
+
and not is_previous_response_not_found_event
|
|
3416
|
+
and status_request_state is not None
|
|
3417
|
+
and is_upstream_model_capacity_error(retry_error_message)
|
|
3418
|
+
and _websocket_request_can_replay_before_visible_output(status_request_state)
|
|
3419
|
+
and _http_bridge_accepted_capacity_retry_allowed(status_request_state)
|
|
3420
|
+
)
|
|
3421
|
+
if (
|
|
3422
|
+
auth_error_code is not None
|
|
3423
|
+
and not is_previous_response_not_found_event
|
|
3424
|
+
and status_request_state is not None
|
|
3425
|
+
):
|
|
3426
|
+
auth_retry_result = await self._retry_http_bridge_precreated_auth_request(
|
|
3427
|
+
session,
|
|
3428
|
+
status_request_state,
|
|
3429
|
+
error_message=_websocket_event_error_message(event_type, payload),
|
|
3430
|
+
)
|
|
3431
|
+
if auth_retry_result == "retried":
|
|
3432
|
+
return
|
|
3433
|
+
if auth_retry_result == "failed":
|
|
3434
|
+
async with session.pending_lock:
|
|
3435
|
+
if status_request_state in session.pending_requests:
|
|
3436
|
+
session.pending_requests.remove(status_request_state)
|
|
3437
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
3438
|
+
if is_http_bridge_account_neutral_replay(
|
|
3439
|
+
kind=session.key.affinity_kind,
|
|
3440
|
+
key=session.key.affinity_key,
|
|
3441
|
+
):
|
|
3442
|
+
_clear_websocket_request_error_overrides(status_request_state)
|
|
3443
|
+
else:
|
|
3444
|
+
status_request_state.error_http_status_override = 502
|
|
3445
|
+
(
|
|
3446
|
+
_downstream_text,
|
|
3447
|
+
event_block,
|
|
3448
|
+
event,
|
|
3449
|
+
payload,
|
|
3450
|
+
event_type,
|
|
3451
|
+
) = _build_stream_incomplete_terminal_event_for_request(status_request_state)
|
|
3452
|
+
elif wait_for_model_capacity_retry and status_request_state is not None and retry_error_code is not None:
|
|
3453
|
+
# Reserve the terminal request again before any await so a younger
|
|
3454
|
+
# submit cannot claim its queue slot while account health is being
|
|
3455
|
+
# updated. A concurrent detach will mark this state as draining.
|
|
3456
|
+
retry_consumer_attached = False
|
|
3457
|
+
# An accepted request has already committed its response start,
|
|
3458
|
+
# so the pre-response startup-wait signals must not fire for it.
|
|
3459
|
+
accepted_lifecycle_replay = (
|
|
3460
|
+
status_request_state.response_id is not None and not status_request_state.awaiting_response_created
|
|
3461
|
+
)
|
|
3462
|
+
async with session.pending_lock:
|
|
3463
|
+
if status_request_state.event_queue is not None:
|
|
3464
|
+
retry_consumer_attached = True
|
|
3465
|
+
if status_request_state not in session.pending_requests:
|
|
3466
|
+
session.pending_requests.appendleft(status_request_state)
|
|
3467
|
+
session.queued_request_count += 1
|
|
3468
|
+
if not await _stage_websocket_request_state_for_replay(
|
|
3469
|
+
status_request_state,
|
|
3470
|
+
create_gate=session.response_create_gate,
|
|
3471
|
+
surface="http_bridge",
|
|
3472
|
+
trigger="capacity_error",
|
|
3473
|
+
):
|
|
3474
|
+
retry_consumer_attached = False
|
|
3475
|
+
if status_request_state.propagate_http_errors and not accepted_lifecycle_replay:
|
|
3476
|
+
_signal_http_bridge_capacity_startup_wait(status_request_state)
|
|
3477
|
+
await self._handle_or_defer_precreated_stream_health(
|
|
3478
|
+
status_request_state,
|
|
3479
|
+
session.account,
|
|
3480
|
+
{"message": retry_error_message or "Upstream error"},
|
|
3481
|
+
retry_error_code,
|
|
3482
|
+
)
|
|
3483
|
+
retry_consumer_attached = (
|
|
3484
|
+
retry_consumer_attached
|
|
3485
|
+
and status_request_state.event_queue is not None
|
|
3486
|
+
and not status_request_state.draining_until_terminal
|
|
3487
|
+
)
|
|
3488
|
+
if retry_consumer_attached:
|
|
3489
|
+
wait_request_had_event_queue = True
|
|
3490
|
+
if (
|
|
3491
|
+
status_request_state.response_create_admission is not None
|
|
3492
|
+
or status_request_state.account_response_create_lease is not None
|
|
3493
|
+
):
|
|
3494
|
+
await _release_http_bridge_model_capacity_retry_admission(status_request_state)
|
|
3495
|
+
status_request_state.awaiting_response_created = True
|
|
3496
|
+
retry_after_wait = await _wait_before_http_bridge_model_capacity_retry(
|
|
3497
|
+
status_request_state,
|
|
3498
|
+
emit_keepalives=not status_request_state.propagate_http_errors,
|
|
3499
|
+
error_message=retry_error_message,
|
|
3500
|
+
cancel_when_detached=True,
|
|
3501
|
+
signal_startup_wait=not accepted_lifecycle_replay,
|
|
3502
|
+
scheduler=scheduler,
|
|
3503
|
+
clock=clock,
|
|
3504
|
+
)
|
|
3505
|
+
if wait_request_had_event_queue and status_request_state.event_queue is None:
|
|
3506
|
+
retry_after_wait = False
|
|
3507
|
+
suppress_capacity_keepalives_until_retry_finishes = (
|
|
3508
|
+
status_request_state.account_capacity_wait_suppress_keepalive
|
|
3509
|
+
)
|
|
3510
|
+
try:
|
|
3511
|
+
retried = retry_after_wait and await self._retry_http_bridge_precreated_request(
|
|
3512
|
+
session,
|
|
3513
|
+
request_state=status_request_state,
|
|
3514
|
+
)
|
|
3515
|
+
if retried:
|
|
3516
|
+
if not accepted_lifecycle_replay:
|
|
3517
|
+
_signal_http_bridge_model_capacity_retry_ready(
|
|
3518
|
+
status_request_state,
|
|
3519
|
+
waited_for_model_capacity_retry=True,
|
|
3520
|
+
retried=True,
|
|
3521
|
+
)
|
|
3522
|
+
return
|
|
3523
|
+
finally:
|
|
3524
|
+
if suppress_capacity_keepalives_until_retry_finishes:
|
|
3525
|
+
status_request_state.account_capacity_wait_suppress_keepalive = False
|
|
3526
|
+
async with session.pending_lock:
|
|
3527
|
+
_relinquish_http_bridge_capacity_wait_ownership(
|
|
3528
|
+
session, status_request_state, claimed_terminal_request_states
|
|
3529
|
+
)
|
|
3530
|
+
if retry_after_wait or not status_request_state.propagate_http_errors:
|
|
3531
|
+
status_request_state.error_http_status_override = 502
|
|
3532
|
+
(
|
|
3533
|
+
_downstream_text,
|
|
3534
|
+
event_block,
|
|
3535
|
+
event,
|
|
3536
|
+
payload,
|
|
3537
|
+
event_type,
|
|
3538
|
+
) = _build_stream_incomplete_terminal_event_for_request(status_request_state)
|
|
3539
|
+
else:
|
|
3540
|
+
async with session.pending_lock:
|
|
3541
|
+
_relinquish_http_bridge_capacity_wait_ownership(
|
|
3542
|
+
session, status_request_state, claimed_terminal_request_states
|
|
3543
|
+
)
|
|
3544
|
+
elif owner_pinned_quota_error is not None and not is_previous_response_not_found_event:
|
|
3545
|
+
await self._handle_or_defer_precreated_stream_health(
|
|
3546
|
+
status_request_state,
|
|
3547
|
+
session.account,
|
|
3548
|
+
{"message": retry_error_message or "Upstream error"},
|
|
3549
|
+
owner_pinned_quota_error,
|
|
3550
|
+
)
|
|
3551
|
+
if (
|
|
3552
|
+
status_request_state is not None
|
|
3553
|
+
and status_request_state.previous_response_id is not None
|
|
3554
|
+
and status_request_state.preferred_account_id is not None
|
|
3555
|
+
):
|
|
3556
|
+
safe_request_text = _prepare_websocket_request_state_for_account_switch(status_request_state)
|
|
3557
|
+
if safe_request_text is not None:
|
|
3558
|
+
previous_upstream_turn_state = session.upstream_turn_state
|
|
3559
|
+
previous_downstream_turn_state = session.downstream_turn_state
|
|
3560
|
+
session.upstream_turn_state = None
|
|
3561
|
+
session.downstream_turn_state = None
|
|
3562
|
+
await self._release_request_state_account_response_create_lease(status_request_state)
|
|
3563
|
+
status_request_state.excluded_account_ids.add(session.account.id)
|
|
3564
|
+
status_request_state.affinity_policy = replace(
|
|
3565
|
+
status_request_state.affinity_policy,
|
|
3566
|
+
reallocate_sticky=True,
|
|
3567
|
+
)
|
|
3568
|
+
status_request_state.request_text = safe_request_text
|
|
3569
|
+
async with session.pending_lock:
|
|
3570
|
+
if status_request_state not in session.pending_requests:
|
|
3571
|
+
session.pending_requests.appendleft(status_request_state)
|
|
3572
|
+
session.queued_request_count += 1
|
|
3573
|
+
status_request_state.awaiting_response_created = True
|
|
3574
|
+
status_request_state.response_id = None
|
|
3575
|
+
retried = await self._retry_http_bridge_precreated_request(session)
|
|
3576
|
+
if retried:
|
|
3577
|
+
return
|
|
3578
|
+
session.upstream_turn_state = previous_upstream_turn_state
|
|
3579
|
+
session.downstream_turn_state = previous_downstream_turn_state
|
|
3580
|
+
async with session.pending_lock:
|
|
3581
|
+
if status_request_state in session.pending_requests:
|
|
3582
|
+
session.pending_requests.remove(status_request_state)
|
|
3583
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
3584
|
+
status_request_state.error_http_status_override = 502
|
|
3585
|
+
(
|
|
3586
|
+
_downstream_text,
|
|
3587
|
+
event_block,
|
|
3588
|
+
event,
|
|
3589
|
+
payload,
|
|
3590
|
+
event_type,
|
|
3591
|
+
) = _build_stream_incomplete_terminal_event_for_request(status_request_state)
|
|
3592
|
+
else:
|
|
3593
|
+
status_request_state.error_http_status_override = 502
|
|
3594
|
+
session.upstream_control.reconnect_requested = True
|
|
3595
|
+
session.upstream_control.retire_after_drain = True
|
|
3596
|
+
event, payload, event_type, rewritten_text = (
|
|
3597
|
+
_rewrite_websocket_previous_response_owner_unavailable_event(
|
|
3598
|
+
request_state=status_request_state,
|
|
3599
|
+
)
|
|
3600
|
+
)
|
|
3601
|
+
event_block = f"data: {rewritten_text}\n\n"
|
|
3602
|
+
elif (
|
|
3603
|
+
retry_error_code == _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
|
|
3604
|
+
and not is_previous_response_not_found_event
|
|
3605
|
+
and status_request_state is not None
|
|
3606
|
+
and _websocket_auth_request_can_switch_account(status_request_state)
|
|
3607
|
+
):
|
|
3608
|
+
rejected_account_id = session.account.id
|
|
3609
|
+
status_request_state.precreated_replay_reason = _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
|
|
3610
|
+
status_request_state.precreated_replay_account_id = rejected_account_id
|
|
3611
|
+
previous_upstream_turn_state = session.upstream_turn_state
|
|
3612
|
+
previous_downstream_turn_state = session.downstream_turn_state
|
|
3613
|
+
previous_headers = session.headers
|
|
3614
|
+
await self._release_request_state_account_response_create_lease(status_request_state)
|
|
3615
|
+
async with session.pending_lock:
|
|
3616
|
+
if status_request_state not in session.pending_requests:
|
|
3617
|
+
session.pending_requests.appendleft(status_request_state)
|
|
3618
|
+
session.queued_request_count += 1
|
|
3619
|
+
status_request_state.awaiting_response_created = True
|
|
3620
|
+
status_request_state.response_id = None
|
|
3621
|
+
retried = await self._retry_http_bridge_precreated_request(session)
|
|
3622
|
+
if retried:
|
|
3623
|
+
logger.info(
|
|
3624
|
+
"Retried HTTP bridge request after account/model rejection "
|
|
3625
|
+
"request_id=%s rejected_account_id=%s model=%s",
|
|
3626
|
+
status_request_state.request_log_id or status_request_state.request_id,
|
|
3627
|
+
rejected_account_id,
|
|
3628
|
+
status_request_state.model,
|
|
3629
|
+
)
|
|
3630
|
+
return
|
|
3631
|
+
replacement_session_selected = session.account.id != rejected_account_id
|
|
3632
|
+
if not replacement_session_selected:
|
|
3633
|
+
session.upstream_turn_state = previous_upstream_turn_state
|
|
3634
|
+
session.downstream_turn_state = previous_downstream_turn_state
|
|
3635
|
+
session.headers = previous_headers
|
|
3636
|
+
async with session.pending_lock:
|
|
3637
|
+
if status_request_state in session.pending_requests:
|
|
3638
|
+
session.pending_requests.remove(status_request_state)
|
|
3639
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
3640
|
+
if replacement_session_selected:
|
|
3641
|
+
# Reconnect may have committed the session to a replacement
|
|
3642
|
+
# account before its replacement lease or send failed. Never
|
|
3643
|
+
# graft the rejected account's turn metadata back onto that
|
|
3644
|
+
# socket; retire the now-unused replacement session instead.
|
|
3645
|
+
session.upstream_control.reconnect_requested = True
|
|
3646
|
+
session.upstream_control.retire_after_drain = True
|
|
3647
|
+
await self._retire_http_bridge_after_drain_if_ready(session)
|
|
3648
|
+
payload = cast(
|
|
3649
|
+
dict[str, JsonValue],
|
|
3650
|
+
dict(
|
|
3651
|
+
response_failed_event(
|
|
3652
|
+
status_request_state.error_code_override or "upstream_unavailable",
|
|
3653
|
+
status_request_state.error_message_override or "HTTP bridge replacement retry failed",
|
|
3654
|
+
error_type=status_request_state.error_type_override or "server_error",
|
|
3655
|
+
response_id=status_request_state.request_id,
|
|
3656
|
+
error_param=status_request_state.error_param_override,
|
|
3657
|
+
)
|
|
3658
|
+
),
|
|
3659
|
+
)
|
|
3660
|
+
event_block = format_sse_event(payload)
|
|
3661
|
+
event = parse_sse_event_payload(payload)
|
|
3662
|
+
event_type = "response.failed"
|
|
3663
|
+
if status_request_state.precreated_replay_reason == _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE:
|
|
3664
|
+
_clear_websocket_precreated_replay_fallback(status_request_state)
|
|
3665
|
+
elif (
|
|
3666
|
+
retry_error_code is not None
|
|
3667
|
+
and retry_error_code != _ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE
|
|
3668
|
+
and not is_previous_response_not_found_event
|
|
3669
|
+
):
|
|
3670
|
+
await self._handle_or_defer_precreated_stream_health(
|
|
3671
|
+
status_request_state,
|
|
3672
|
+
session.account,
|
|
3673
|
+
{"message": retry_error_message or "Upstream error"},
|
|
3674
|
+
retry_error_code,
|
|
3675
|
+
)
|
|
3676
|
+
# Pre-created anchored requests belong to the owner-pinned branch
|
|
3677
|
+
# above; an accepted anchored follow-up with a proxy-injected
|
|
3678
|
+
# anchor and a retry-safe fresh body is replayed here exactly as
|
|
3679
|
+
# the capacity-message wait branch replays it.
|
|
3680
|
+
if status_request_state is not None and (
|
|
3681
|
+
status_request_state.previous_response_id is None
|
|
3682
|
+
or _http_bridge_accepted_anchored_replay_candidate(status_request_state)
|
|
3683
|
+
):
|
|
3684
|
+
async with session.pending_lock:
|
|
3685
|
+
if status_request_state not in session.pending_requests:
|
|
3686
|
+
session.pending_requests.appendleft(status_request_state)
|
|
3687
|
+
session.queued_request_count += 1
|
|
3688
|
+
staged = await _stage_websocket_request_state_for_replay(
|
|
3689
|
+
status_request_state,
|
|
3690
|
+
create_gate=session.response_create_gate,
|
|
3691
|
+
surface="http_bridge",
|
|
3692
|
+
trigger="capacity_error",
|
|
3693
|
+
)
|
|
3694
|
+
retried = staged and await self._retry_http_bridge_precreated_request(session)
|
|
3695
|
+
if retried:
|
|
3696
|
+
return
|
|
3697
|
+
async with session.pending_lock:
|
|
3698
|
+
if status_request_state in session.pending_requests:
|
|
3699
|
+
session.pending_requests.remove(status_request_state)
|
|
3700
|
+
session.queued_request_count = max(0, session.queued_request_count - 1)
|
|
3701
|
+
if staged:
|
|
3702
|
+
# A busy create gate forwards the upstream terminal as-is;
|
|
3703
|
+
# only a replay that was attempted and failed is rewritten.
|
|
3704
|
+
status_request_state.error_http_status_override = 502
|
|
3705
|
+
(
|
|
3706
|
+
_downstream_text,
|
|
3707
|
+
event_block,
|
|
3708
|
+
event,
|
|
3709
|
+
payload,
|
|
3710
|
+
event_type,
|
|
3711
|
+
) = _build_stream_incomplete_terminal_event_for_request(status_request_state)
|
|
3712
|
+
|
|
3713
|
+
completed_usage = (
|
|
3714
|
+
event.response.usage if event_type == "response.completed" and event and event.response else None
|
|
3715
|
+
)
|
|
3716
|
+
completed_empty_prewarm = (
|
|
3717
|
+
event_type == "response.completed"
|
|
3718
|
+
and terminal_request_state is not None
|
|
3719
|
+
and terminal_request_state.request_kind == "prewarm"
|
|
3720
|
+
and completed_usage is not None
|
|
3721
|
+
and completed_usage.output_tokens == 0
|
|
3722
|
+
)
|
|
3723
|
+
|
|
3724
|
+
# The circuit settle and quarantine clear run BEFORE the fresh
|
|
3725
|
+
# anchor is registered. A poison abandonment fences its continuity
|
|
3726
|
+
# clear on the anchor captured when its episode was validated, and
|
|
3727
|
+
# settle-first ordering is what makes that fence airtight: any
|
|
3728
|
+
# consult that runs after this settle is vetoed by the reset row,
|
|
3729
|
+
# and any abandonment authorized before it fences on the old anchor
|
|
3730
|
+
# and cannot touch the one registered below.
|
|
3731
|
+
completion_circuit_settlement_failed = False
|
|
3732
|
+
completion_quarantine_clear_fence: int | None = None
|
|
3733
|
+
completion_pre_settle_poison_detail: str | None = None
|
|
3734
|
+
completion_settles_onto_tombstone = False
|
|
3735
|
+
if (
|
|
3736
|
+
event_type == "response.completed"
|
|
3737
|
+
and terminal_request_state is not None
|
|
3738
|
+
and not terminal_request_state.suppressed_duplicate_tool_call
|
|
3739
|
+
and terminal_request_state.request_kind != "prewarm"
|
|
3740
|
+
and not terminal_request_state.skip_request_log
|
|
3741
|
+
):
|
|
3742
|
+
# Captured before the settle and registration awaits below: a
|
|
3743
|
+
# concurrent request failing during those awaits arms a NEW
|
|
3744
|
+
# same-key quarantine whose evidence this completion does not
|
|
3745
|
+
# disprove, and the clear at the bottom must fence on what was
|
|
3746
|
+
# armed when this response proved the session healthy, not on
|
|
3747
|
+
# whatever owns the key by then.
|
|
3748
|
+
# Adopt any durable-only poison row BEFORE the fences are
|
|
3749
|
+
# captured: the settle's internal load would otherwise arm the
|
|
3750
|
+
# quarantine after the fence read (leaving a healthy key
|
|
3751
|
+
# suppressed for the TTL because the clear cannot match), and a
|
|
3752
|
+
# failed registration would find no pre-settle poison detail to
|
|
3753
|
+
# re-seed.
|
|
3754
|
+
completion_pre_settle_load_succeeded = await self._load_http_bridge_retry_circuit(session)
|
|
3755
|
+
completion_quarantine_clear_fence = _http_bridge_quarantine_clear_fence(self, session.key)
|
|
3756
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
3757
|
+
pre_settle_state = self._http_bridge_retry_circuits.get(session.key)
|
|
3758
|
+
if pre_settle_state is not None:
|
|
3759
|
+
completion_pre_settle_poison_detail = (
|
|
3760
|
+
pre_settle_state.last_detail
|
|
3761
|
+
if _http_bridge_anchor_poison_detail(pre_settle_state.last_detail) is not None
|
|
3762
|
+
else pre_settle_state.owed_poison_detail
|
|
3763
|
+
)
|
|
3764
|
+
# An episode already carrying the fail-closed tombstone
|
|
3765
|
+
# keeps it through the settle: erasing it before the
|
|
3766
|
+
# fresh anchor commits would drop the only durable
|
|
3767
|
+
# marker while the registration can still fail or never
|
|
3768
|
+
# run at all.
|
|
3769
|
+
completion_settles_onto_tombstone = (
|
|
3770
|
+
completion_pre_settle_poison_detail is not None
|
|
3771
|
+
or pre_settle_state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
3772
|
+
)
|
|
3773
|
+
completion_registration_possible = (
|
|
3774
|
+
response_id is not None and matched_request_state is not None and not completed_empty_prewarm
|
|
3775
|
+
)
|
|
3776
|
+
if (
|
|
3777
|
+
not terminal_request_state.verified_stale_anchor_replay
|
|
3778
|
+
and completion_settles_onto_tombstone
|
|
3779
|
+
and not completion_registration_possible
|
|
3780
|
+
):
|
|
3781
|
+
# No usable response id or matched request: the registration
|
|
3782
|
+
# block below never runs, so settling here would replace the
|
|
3783
|
+
# poison row with a zero-count tombstone while the OLD
|
|
3784
|
+
# poisoned anchor stays stored — the next planning load
|
|
3785
|
+
# reads the zero count as a disproved episode, revokes the
|
|
3786
|
+
# quarantine, and a full resend gets the dead anchor
|
|
3787
|
+
# injected. Keep the episode unsettled; a completion that
|
|
3788
|
+
# can actually register a fresh anchor settles it.
|
|
3789
|
+
completion_circuit_settlement_failed = True
|
|
3790
|
+
elif not terminal_request_state.verified_stale_anchor_replay:
|
|
3791
|
+
# A completion replacing a poison episode settles with the
|
|
3792
|
+
# transitional tombstone: a crash or takeover between this
|
|
3793
|
+
# settle and the registration below would otherwise leave
|
|
3794
|
+
# the old poisoned anchor stored while the reset row reads
|
|
3795
|
+
# as disproved. The tombstone is erased once the fresh
|
|
3796
|
+
# anchor commits.
|
|
3797
|
+
circuit_settled = await self._clear_http_bridge_retry_circuit(
|
|
3798
|
+
session,
|
|
3799
|
+
settled_detail=(
|
|
3800
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
3801
|
+
if completion_settles_onto_tombstone
|
|
3802
|
+
else None
|
|
3803
|
+
),
|
|
3804
|
+
)
|
|
3805
|
+
if not completion_pre_settle_load_succeeded:
|
|
3806
|
+
# The fence above was captured off a failed durable read;
|
|
3807
|
+
# the settle's own successful inner load may have armed
|
|
3808
|
+
# the poison quarantine AFTER that capture, and the final
|
|
3809
|
+
# fenced clear would then refuse to remove it — a healthy
|
|
3810
|
+
# key stuck for the whole poison window despite the fresh
|
|
3811
|
+
# anchor about to register. Recapture after the settle:
|
|
3812
|
+
# this still precedes the registration awaits, so a
|
|
3813
|
+
# quarantine armed by a genuinely concurrent strike
|
|
3814
|
+
# during those awaits stays outside the fence.
|
|
3815
|
+
completion_quarantine_clear_fence = _http_bridge_quarantine_clear_fence(self, session.key)
|
|
3816
|
+
if not circuit_settled:
|
|
3817
|
+
# The old poison episode was restored. Its owed clear is
|
|
3818
|
+
# suppressed only once the fresh anchor actually
|
|
3819
|
+
# persists below: suppressing here and then failing the
|
|
3820
|
+
# registration would leave the old poisoned anchor
|
|
3821
|
+
# stored with no funnel willing to clear it after the
|
|
3822
|
+
# process-local quarantine expires.
|
|
3823
|
+
completion_circuit_settlement_failed = True
|
|
3824
|
+
|
|
3825
|
+
# False until a fresh durable anchor actually confirms: a completed
|
|
3826
|
+
# event without a usable response id (or with no matched request)
|
|
3827
|
+
# skips the registration block entirely, and clearing the quarantine
|
|
3828
|
+
# then would leave the old poisoned anchor as the stored one for the
|
|
3829
|
+
# next reattach.
|
|
3830
|
+
completion_anchor_registration_confirmed = False
|
|
3831
|
+
if (
|
|
3832
|
+
response_id is not None
|
|
3833
|
+
and matched_request_state is not None
|
|
3834
|
+
and event_type == "response.completed"
|
|
3835
|
+
and not completed_empty_prewarm
|
|
3836
|
+
):
|
|
3837
|
+
anchor_advance_supersession = None
|
|
3838
|
+
if completion_circuit_settlement_failed:
|
|
3839
|
+
# Applied BEFORE the fresh anchor is published: a concurrent
|
|
3840
|
+
# funnel's consult whose continuity read lands between the
|
|
3841
|
+
# publication and a later suppression would validate the old
|
|
3842
|
+
# poison row and fence its rebind on the anchor just
|
|
3843
|
+
# registered, deleting it. The suppression is transitional —
|
|
3844
|
+
# rolled back below when the registration fails, so the old
|
|
3845
|
+
# anchor never becomes uncleareable.
|
|
3846
|
+
anchor_advance_supersession = await self._http_bridge_suppress_poison_clear_after_anchor_advance(
|
|
3847
|
+
session
|
|
3848
|
+
)
|
|
3849
|
+
alias_registered = await self._register_http_bridge_previous_response_id(
|
|
3850
|
+
session,
|
|
3851
|
+
response_id,
|
|
3852
|
+
input_item_count=(
|
|
3853
|
+
matched_request_state.input_item_count if matched_request_state.input_item_count > 0 else None
|
|
3854
|
+
),
|
|
3855
|
+
input_full_fingerprint=(
|
|
3856
|
+
matched_request_state.input_full_fingerprint if matched_request_state.input_item_count > 0 else None
|
|
3857
|
+
),
|
|
3858
|
+
pending_tool_calls=_durable_pending_tool_call_manifest(matched_request_state, payload),
|
|
3859
|
+
)
|
|
3860
|
+
completion_anchor_registration_confirmed = alias_registered
|
|
3861
|
+
if not alias_registered and anchor_advance_supersession is not None:
|
|
3862
|
+
await self._http_bridge_restore_poison_clear_after_failed_anchor_advance(
|
|
3863
|
+
session, anchor_advance_supersession
|
|
3864
|
+
)
|
|
3865
|
+
if alias_registered and anchor_advance_supersession is not None:
|
|
3866
|
+
await self._http_bridge_promote_transitional_supersession(session, anchor_advance_supersession)
|
|
3867
|
+
if alias_registered and not completion_circuit_settlement_failed and completion_settles_onto_tombstone:
|
|
3868
|
+
# The fresh anchor committed: erase the transitional
|
|
3869
|
+
# tombstone the settle left, fenced on the row's own
|
|
3870
|
+
# values, with one reconcile round covering both a
|
|
3871
|
+
# transient durable blip and a CAS miss under a sticky
|
|
3872
|
+
# concurrent strike.
|
|
3873
|
+
await self._http_bridge_reconcile_transitional_tombstone(session)
|
|
3874
|
+
if (
|
|
3875
|
+
not alias_registered
|
|
3876
|
+
and not completion_circuit_settlement_failed
|
|
3877
|
+
and completion_pre_settle_poison_detail is not None
|
|
3878
|
+
):
|
|
3879
|
+
# The settle-before-registration ordering already zeroed the
|
|
3880
|
+
# durable circuit, but the registration failure means the
|
|
3881
|
+
# OLD poisoned anchor is still the stored one. Without
|
|
3882
|
+
# durable evidence the next planning load revokes the kept
|
|
3883
|
+
# local quarantine as a disproved episode and other replicas
|
|
3884
|
+
# never arm; re-seed the row so the poison protection holds
|
|
3885
|
+
# everywhere until a real recovery lands.
|
|
3886
|
+
await self._http_bridge_restore_poison_row_after_failed_registration(
|
|
3887
|
+
session, completion_pre_settle_poison_detail
|
|
3888
|
+
)
|
|
3889
|
+
if not alias_registered and is_http_bridge_account_neutral_replay(
|
|
3890
|
+
kind=session.key.affinity_kind,
|
|
3891
|
+
key=session.key.affinity_key,
|
|
3892
|
+
):
|
|
3893
|
+
session.upstream_control.reconnect_requested = True
|
|
3894
|
+
session.upstream_control.retire_after_drain = True
|
|
3895
|
+
matched_request_state.error_http_status_override = 502
|
|
3896
|
+
payload = cast(
|
|
3897
|
+
dict[str, JsonValue],
|
|
3898
|
+
dict(
|
|
3899
|
+
response_failed_event(
|
|
3900
|
+
"bridge_continuity_persistence_failed",
|
|
3901
|
+
"Recovered response continuity could not be persisted; retry the request.",
|
|
3902
|
+
response_id=_websocket_downstream_response_id(matched_request_state),
|
|
3903
|
+
)
|
|
3904
|
+
),
|
|
3905
|
+
)
|
|
3906
|
+
event_block = format_sse_event(payload)
|
|
3907
|
+
event = parse_sse_event_payload(payload)
|
|
3908
|
+
event_type = "response.failed"
|
|
3909
|
+
# The upstream response was already acknowledged. The local
|
|
3910
|
+
# alias write failed, so expose a terminal error downstream
|
|
3911
|
+
# but keep the durable operation acknowledged/ambiguous to
|
|
3912
|
+
# prevent an identical retry from dispatching it again.
|
|
3913
|
+
continuity_persistence_failed_after_ack = True
|
|
3914
|
+
completed_usage = None
|
|
3915
|
+
completed_empty_prewarm = False
|
|
3916
|
+
|
|
3917
|
+
if (
|
|
3918
|
+
event_type == "response.completed"
|
|
3919
|
+
and terminal_request_state is not None
|
|
3920
|
+
and not terminal_request_state.suppressed_duplicate_tool_call
|
|
3921
|
+
and terminal_request_state.request_kind != "prewarm"
|
|
3922
|
+
and not terminal_request_state.skip_request_log
|
|
3923
|
+
# A failed circuit settlement leaves the at-threshold poison row
|
|
3924
|
+
# standing; the quarantine keeps covering it so the surviving
|
|
3925
|
+
# cooldown cannot hand the next attach a poisoned injection while
|
|
3926
|
+
# the settle retries at the next opportunity.
|
|
3927
|
+
and not completion_circuit_settlement_failed
|
|
3928
|
+
# A durable alias write that failed leaves the OLD anchor as the
|
|
3929
|
+
# stored one even though this worker's completion succeeded: a
|
|
3930
|
+
# replica or a restart would re-inject it, and the quarantine is
|
|
3931
|
+
# the only protection left once the circuit was settled.
|
|
3932
|
+
and completion_anchor_registration_confirmed
|
|
3933
|
+
):
|
|
3934
|
+
# The quarantine clears only after the fresh anchor persisted:
|
|
3935
|
+
# a failed alias write rewrites the event to response.failed
|
|
3936
|
+
# above, this guard then skips, and the quarantine keeps
|
|
3937
|
+
# covering the old anchor that is still stored. The circuit
|
|
3938
|
+
# settle deliberately ran before registration for the
|
|
3939
|
+
# abandonment fence; the quarantine surviving here is what
|
|
3940
|
+
# protects the partial-failure window that ordering leaves.
|
|
3941
|
+
_clear_http_bridge_quarantine(
|
|
3942
|
+
self,
|
|
3943
|
+
session,
|
|
3944
|
+
key_generation=completion_quarantine_clear_fence,
|
|
3945
|
+
additional_key=terminal_request_state.verified_stale_anchor_retry_circuit_key,
|
|
3946
|
+
additional_key_generation=terminal_request_state.verified_stale_anchor_quarantine_generation,
|
|
3947
|
+
)
|
|
3948
|
+
|
|
3949
|
+
operation_state = _http_bridge_operation_state_for_event(event_type)
|
|
3950
|
+
if operation_state is not None:
|
|
3951
|
+
operation_request_states: list[Any] = []
|
|
3952
|
+
for candidate in (matched_request_state, terminal_request_state):
|
|
3953
|
+
if candidate is not None and candidate not in operation_request_states:
|
|
3954
|
+
operation_request_states.append(candidate)
|
|
3955
|
+
for operation_request_state in operation_request_states:
|
|
3956
|
+
request_operation_state = operation_state
|
|
3957
|
+
if continuity_persistence_failed_after_ack and operation_request_state is matched_request_state:
|
|
3958
|
+
request_operation_state = "acknowledged"
|
|
3959
|
+
if request_operation_state == "failed":
|
|
3960
|
+
# Failure rows are exposed only by the terminal-event
|
|
3961
|
+
# persistence path below, which appends the terminal SSE
|
|
3962
|
+
# block and flips the operation state atomically.
|
|
3963
|
+
continue
|
|
3964
|
+
await _update_http_bridge_operation_state(
|
|
3965
|
+
self,
|
|
3966
|
+
session,
|
|
3967
|
+
operation_request_state,
|
|
3968
|
+
state=request_operation_state,
|
|
3969
|
+
response_id=response_id,
|
|
3970
|
+
)
|
|
3971
|
+
|
|
3972
|
+
recovery_attempt_session_id = (
|
|
3973
|
+
matched_request_state.recovery_attempt_session_id
|
|
3974
|
+
if matched_request_state is not None and matched_request_state.recovery_attempt_session_id is not None
|
|
3975
|
+
else session.durable_session_id
|
|
3976
|
+
)
|
|
3977
|
+
recovery_attempt_owner_epoch = (
|
|
3978
|
+
matched_request_state.recovery_attempt_owner_epoch
|
|
3979
|
+
if matched_request_state is not None and matched_request_state.recovery_attempt_owner_epoch is not None
|
|
3980
|
+
else session.durable_owner_epoch
|
|
3981
|
+
)
|
|
3982
|
+
|
|
3983
|
+
if (
|
|
3984
|
+
isinstance(event_type, str)
|
|
3985
|
+
and (event_type.startswith("response.") or event_type == "error")
|
|
3986
|
+
and matched_request_state is not None
|
|
3987
|
+
and matched_request_state.recovery_attempt_fingerprint is not None
|
|
3988
|
+
and recovery_attempt_session_id is not None
|
|
3989
|
+
and recovery_attempt_owner_epoch is not None
|
|
3990
|
+
and (
|
|
3991
|
+
event_type in {"response.completed", "response.failed", "response.incomplete", "error"}
|
|
3992
|
+
or not matched_request_state.recovery_attempt_event_observed
|
|
3993
|
+
)
|
|
3994
|
+
):
|
|
3995
|
+
settlement_marked = False
|
|
3996
|
+
for settlement_attempt in range(3):
|
|
3997
|
+
try:
|
|
3998
|
+
marked = await self._durable_bridge.mark_recovery_attempt_replayed(
|
|
3999
|
+
session_id=recovery_attempt_session_id,
|
|
4000
|
+
api_key_id=session.key.api_key_id,
|
|
4001
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4002
|
+
owner_epoch=recovery_attempt_owner_epoch,
|
|
4003
|
+
request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
|
|
4004
|
+
response_id=response_id,
|
|
4005
|
+
)
|
|
4006
|
+
if marked:
|
|
4007
|
+
settlement_marked = True
|
|
4008
|
+
break
|
|
4009
|
+
if settlement_attempt == 2:
|
|
4010
|
+
_schedule_http_bridge_recovery_settlement_retry(
|
|
4011
|
+
self,
|
|
4012
|
+
session,
|
|
4013
|
+
session_id=recovery_attempt_session_id,
|
|
4014
|
+
api_key_id=session.key.api_key_id,
|
|
4015
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4016
|
+
owner_epoch=recovery_attempt_owner_epoch,
|
|
4017
|
+
request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
|
|
4018
|
+
response_id=response_id,
|
|
4019
|
+
release_origin_lease=(
|
|
4020
|
+
recovery_attempt_session_id != session.durable_session_id
|
|
4021
|
+
and event_type
|
|
4022
|
+
in {"response.completed", "response.failed", "response.incomplete", "error"}
|
|
4023
|
+
),
|
|
4024
|
+
)
|
|
4025
|
+
except Exception:
|
|
4026
|
+
if settlement_attempt == 2:
|
|
4027
|
+
logger.warning("Failed to settle HTTP bridge recovery attempt", exc_info=True)
|
|
4028
|
+
_schedule_http_bridge_recovery_settlement_retry(
|
|
4029
|
+
self,
|
|
4030
|
+
session,
|
|
4031
|
+
session_id=recovery_attempt_session_id,
|
|
4032
|
+
api_key_id=session.key.api_key_id,
|
|
4033
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4034
|
+
owner_epoch=recovery_attempt_owner_epoch,
|
|
4035
|
+
request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
|
|
4036
|
+
response_id=response_id,
|
|
4037
|
+
release_origin_lease=(
|
|
4038
|
+
recovery_attempt_session_id != session.durable_session_id
|
|
4039
|
+
and event_type
|
|
4040
|
+
in {"response.completed", "response.failed", "response.incomplete", "error"}
|
|
4041
|
+
),
|
|
4042
|
+
)
|
|
4043
|
+
else:
|
|
4044
|
+
await scheduler.sleep(0.05 * (settlement_attempt + 1))
|
|
4045
|
+
if (
|
|
4046
|
+
settlement_marked
|
|
4047
|
+
and event_type in {"response.completed", "response.failed", "response.incomplete", "error"}
|
|
4048
|
+
and recovery_attempt_session_id != session.durable_session_id
|
|
4049
|
+
):
|
|
4050
|
+
try:
|
|
4051
|
+
await self._durable_bridge.release_live_session(
|
|
4052
|
+
session_id=recovery_attempt_session_id,
|
|
4053
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4054
|
+
owner_epoch=recovery_attempt_owner_epoch,
|
|
4055
|
+
draining=False,
|
|
4056
|
+
)
|
|
4057
|
+
except Exception:
|
|
4058
|
+
logger.debug("Failed to release HTTP bridge recovery origin lease", exc_info=True)
|
|
4059
|
+
matched_request_state.recovery_attempt_event_observed = True
|
|
4060
|
+
|
|
4061
|
+
if event_type == "response.completed" and terminal_request_state is not None and not completed_empty_prewarm:
|
|
4062
|
+
# Record the completed response id regardless of input shape so
|
|
4063
|
+
# subsequent turns (including ones that never populated
|
|
4064
|
+
# input_item_count, e.g. string inputs) can still reuse this
|
|
4065
|
+
# anchor for continuity lookups.
|
|
4066
|
+
if response_id is not None:
|
|
4067
|
+
session.last_completed_response_id = response_id
|
|
4068
|
+
# This response was completed on the session's current account, so
|
|
4069
|
+
# that account owns the anchor. Record it so the anchor is only
|
|
4070
|
+
# replayed on the same account (never after a cross-account failover).
|
|
4071
|
+
session.last_completed_response_account_id = session.account.id
|
|
4072
|
+
# Remember which tool-call items the completed response left
|
|
4073
|
+
# pending so an anchored follow-up that omits their outputs
|
|
4074
|
+
# (interrupted turn) can receive synthetic interrupted
|
|
4075
|
+
# outputs instead of an upstream missing-tool-output 400.
|
|
4076
|
+
session.last_pending_tool_calls = dict(terminal_request_state.pending_tool_call_types)
|
|
4077
|
+
# Prefix trimming is only meaningful for list-shaped inputs, so
|
|
4078
|
+
# keep the input-count / fingerprint update scoped to that path.
|
|
4079
|
+
if terminal_request_state.input_item_count > 0:
|
|
4080
|
+
session.last_completed_input_count = terminal_request_state.input_item_count
|
|
4081
|
+
session.last_completed_input_prefix_fingerprint = terminal_request_state.input_full_fingerprint
|
|
4082
|
+
|
|
4083
|
+
normalize_error_event = (
|
|
4084
|
+
terminal_request_state is None or terminal_request_state.enforce_openai_sdk_contract
|
|
4085
|
+
) and (matched_request_state is None or matched_request_state.enforce_openai_sdk_contract)
|
|
4086
|
+
settlement_payload = payload
|
|
4087
|
+
settlement_event = event
|
|
4088
|
+
settlement_event_type = event_type
|
|
4089
|
+
if event_type == "error" and normalize_error_event:
|
|
4090
|
+
http_status = _http_error_status_from_payload(payload)
|
|
4091
|
+
if status_request_state is not None and status_request_state.error_http_status_override is None:
|
|
4092
|
+
status_request_state.error_http_status_override = http_status
|
|
4093
|
+
(
|
|
4094
|
+
event_block,
|
|
4095
|
+
payload,
|
|
4096
|
+
event,
|
|
4097
|
+
event_type,
|
|
4098
|
+
) = _normalize_http_bridge_error_event(
|
|
4099
|
+
event=event,
|
|
4100
|
+
payload=payload,
|
|
4101
|
+
request_state=terminal_request_state or matched_request_state,
|
|
4102
|
+
)
|
|
4103
|
+
settlement_payload = payload
|
|
4104
|
+
settlement_event = event
|
|
4105
|
+
settlement_event_type = event_type
|
|
4106
|
+
elif event_type == "error":
|
|
4107
|
+
http_status = _http_error_status_from_payload(payload)
|
|
4108
|
+
if status_request_state is not None and status_request_state.error_http_status_override is None:
|
|
4109
|
+
status_request_state.error_http_status_override = http_status
|
|
4110
|
+
(
|
|
4111
|
+
_settlement_event_block,
|
|
4112
|
+
settlement_payload,
|
|
4113
|
+
settlement_event,
|
|
4114
|
+
settlement_event_type,
|
|
4115
|
+
) = _normalize_http_bridge_error_event(
|
|
4116
|
+
event=event,
|
|
4117
|
+
payload=payload,
|
|
4118
|
+
request_state=terminal_request_state or matched_request_state,
|
|
4119
|
+
)
|
|
4120
|
+
|
|
4121
|
+
if (
|
|
4122
|
+
settlement_event_type in {"response.failed", "error"}
|
|
4123
|
+
and matched_request_state is not None
|
|
4124
|
+
and matched_request_state.recovery_attempt_fingerprint is not None
|
|
4125
|
+
and recovery_attempt_session_id is not None
|
|
4126
|
+
and recovery_attempt_owner_epoch is not None
|
|
4127
|
+
and not matched_request_state.recovery_attempt_event_observed
|
|
4128
|
+
):
|
|
4129
|
+
# An explicit deterministic failure is terminal evidence for the
|
|
4130
|
+
# journaled request, not an ambiguous transport outcome. Consume
|
|
4131
|
+
# the UNKNOWN row after normalizing top-level errors so a later
|
|
4132
|
+
# identical retry cannot turn it into an account-neutral replay.
|
|
4133
|
+
deterministic_settlement_marked = False
|
|
4134
|
+
for settlement_attempt in range(3):
|
|
4135
|
+
try:
|
|
4136
|
+
marked = await self._durable_bridge.mark_recovery_attempt_replayed(
|
|
4137
|
+
session_id=recovery_attempt_session_id,
|
|
4138
|
+
api_key_id=session.key.api_key_id,
|
|
4139
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4140
|
+
owner_epoch=recovery_attempt_owner_epoch,
|
|
4141
|
+
request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
|
|
4142
|
+
response_id=response_id,
|
|
4143
|
+
)
|
|
4144
|
+
if marked:
|
|
4145
|
+
deterministic_settlement_marked = True
|
|
4146
|
+
break
|
|
4147
|
+
if settlement_attempt == 2:
|
|
4148
|
+
_schedule_http_bridge_recovery_settlement_retry(
|
|
4149
|
+
self,
|
|
4150
|
+
session,
|
|
4151
|
+
session_id=recovery_attempt_session_id,
|
|
4152
|
+
api_key_id=session.key.api_key_id,
|
|
4153
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4154
|
+
owner_epoch=recovery_attempt_owner_epoch,
|
|
4155
|
+
request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
|
|
4156
|
+
response_id=response_id,
|
|
4157
|
+
release_origin_lease=recovery_attempt_session_id != session.durable_session_id,
|
|
4158
|
+
)
|
|
4159
|
+
except Exception:
|
|
4160
|
+
if settlement_attempt == 2:
|
|
4161
|
+
logger.warning("Failed to settle deterministic HTTP bridge recovery attempt", exc_info=True)
|
|
4162
|
+
_schedule_http_bridge_recovery_settlement_retry(
|
|
4163
|
+
self,
|
|
4164
|
+
session,
|
|
4165
|
+
session_id=recovery_attempt_session_id,
|
|
4166
|
+
api_key_id=session.key.api_key_id,
|
|
4167
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4168
|
+
owner_epoch=recovery_attempt_owner_epoch,
|
|
4169
|
+
request_fingerprint=matched_request_state.recovery_attempt_fingerprint,
|
|
4170
|
+
response_id=response_id,
|
|
4171
|
+
release_origin_lease=recovery_attempt_session_id != session.durable_session_id,
|
|
4172
|
+
)
|
|
4173
|
+
else:
|
|
4174
|
+
await scheduler.sleep(0.05 * (settlement_attempt + 1))
|
|
4175
|
+
if deterministic_settlement_marked and recovery_attempt_session_id != session.durable_session_id:
|
|
4176
|
+
try:
|
|
4177
|
+
await self._durable_bridge.release_live_session(
|
|
4178
|
+
session_id=recovery_attempt_session_id,
|
|
4179
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
4180
|
+
owner_epoch=recovery_attempt_owner_epoch,
|
|
4181
|
+
draining=False,
|
|
4182
|
+
)
|
|
4183
|
+
except Exception:
|
|
4184
|
+
logger.debug("Failed to release HTTP bridge recovery origin lease", exc_info=True)
|
|
4185
|
+
|
|
4186
|
+
if event_type == "response.created" and release_create_gate and created_request_state is not None:
|
|
4187
|
+
await _release_websocket_response_create_gate(
|
|
4188
|
+
created_request_state, session.response_create_gate, scheduler=scheduler
|
|
4189
|
+
)
|
|
4190
|
+
|
|
4191
|
+
if terminal_request_state is not None and settlement_event_type in {"response.failed", "error"}:
|
|
4192
|
+
if settlement_event_type == "error":
|
|
4193
|
+
error = settlement_event.error if settlement_event else None
|
|
4194
|
+
else:
|
|
4195
|
+
error = settlement_event.response.error if settlement_event and settlement_event.response else None
|
|
4196
|
+
terminal_error_code = _normalize_error_code(
|
|
4197
|
+
error.code if error else None,
|
|
4198
|
+
error.type if error else None,
|
|
4199
|
+
)
|
|
4200
|
+
terminal_error_message = error.message if error else None
|
|
4201
|
+
if _is_security_work_authorization_required_error(terminal_error_code, terminal_error_message):
|
|
4202
|
+
can_retry_security_work = (
|
|
4203
|
+
not is_http_bridge_account_neutral_replay(
|
|
4204
|
+
kind=session.key.affinity_kind,
|
|
4205
|
+
key=session.key.affinity_key,
|
|
4206
|
+
)
|
|
4207
|
+
and not session.account.security_work_authorized
|
|
4208
|
+
and not has_other_pending_requests
|
|
4209
|
+
and terminal_request_state.response_id is None
|
|
4210
|
+
and terminal_request_state.replay_count < 1
|
|
4211
|
+
and bool(terminal_request_state.request_text)
|
|
4212
|
+
and terminal_request_state.preferred_account_id != session.account.id
|
|
4213
|
+
and _websocket_auth_request_can_switch_account(terminal_request_state)
|
|
4214
|
+
and _websocket_request_can_replay_before_visible_output(terminal_request_state)
|
|
4215
|
+
)
|
|
4216
|
+
_clear_websocket_deferred_reasoning_downstream_texts(terminal_request_state)
|
|
4217
|
+
if terminal_request_state.event_queue is not None:
|
|
4218
|
+
await terminal_request_state.event_queue.put(
|
|
4219
|
+
format_sse_event(
|
|
4220
|
+
_security_work_advisory_event(
|
|
4221
|
+
code=_SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
|
|
4222
|
+
message=(
|
|
4223
|
+
_SECURITY_WORK_RETRY_MESSAGE
|
|
4224
|
+
if can_retry_security_work
|
|
4225
|
+
else "Upstream flagged this request as possible cybersecurity work. "
|
|
4226
|
+
"codex-lb cannot safely switch accounts after this response has already started, "
|
|
4227
|
+
"so the original upstream error is being forwarded."
|
|
4228
|
+
),
|
|
4229
|
+
request_id=terminal_request_state.request_log_id or terminal_request_state.request_id,
|
|
4230
|
+
action=(
|
|
4231
|
+
"retry_security_work_authorized"
|
|
4232
|
+
if can_retry_security_work
|
|
4233
|
+
else "forward_original_security_work_error"
|
|
4234
|
+
),
|
|
4235
|
+
account_id=session.account.id,
|
|
4236
|
+
)
|
|
4237
|
+
)
|
|
4238
|
+
)
|
|
4239
|
+
if can_retry_security_work:
|
|
4240
|
+
retried = await self._retry_http_bridge_security_work_request(session, terminal_request_state)
|
|
4241
|
+
if retried:
|
|
4242
|
+
return
|
|
4243
|
+
|
|
4244
|
+
terminal_strike_failures: int | None = None
|
|
4245
|
+
terminal_poison_detail: str | None = None
|
|
4246
|
+
|
|
4247
|
+
async def _finalize_terminal_settlement(settled_request_state: _WebSocketRequestState) -> None:
|
|
4248
|
+
try:
|
|
4249
|
+
await self._finalize_websocket_request_state(
|
|
4250
|
+
settled_request_state,
|
|
4251
|
+
account=session.account,
|
|
4252
|
+
account_id_value=session.account.id,
|
|
4253
|
+
event=settlement_event,
|
|
4254
|
+
event_type=settlement_event_type,
|
|
4255
|
+
payload=settlement_payload,
|
|
4256
|
+
api_key=settled_request_state.api_key,
|
|
4257
|
+
upstream_control=session.upstream_control,
|
|
4258
|
+
response_create_gate=session.response_create_gate,
|
|
4259
|
+
)
|
|
4260
|
+
finally:
|
|
4261
|
+
await self._maybe_release_idle_http_bridge_session_lease(session)
|
|
4262
|
+
|
|
4263
|
+
if settlement_event_type in {"response.failed", "response.incomplete", "error"}:
|
|
4264
|
+
error_code = None
|
|
4265
|
+
if settlement_event_type == "error":
|
|
4266
|
+
error = settlement_event.error if settlement_event else None
|
|
4267
|
+
error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
|
|
4268
|
+
elif settlement_event and settlement_event.response:
|
|
4269
|
+
error = settlement_event.response.error
|
|
4270
|
+
error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
|
|
4271
|
+
_log_http_bridge_event(
|
|
4272
|
+
"terminal_error",
|
|
4273
|
+
session.key,
|
|
4274
|
+
account_id=session.account.id,
|
|
4275
|
+
model=session.request_model,
|
|
4276
|
+
detail=error_code,
|
|
4277
|
+
pending_count=await self._http_bridge_pending_count(session),
|
|
4278
|
+
cache_key_family=session.key.affinity_kind,
|
|
4279
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
4280
|
+
)
|
|
4281
|
+
if (
|
|
4282
|
+
error_code is not None
|
|
4283
|
+
and terminal_request_state is not None
|
|
4284
|
+
and terminal_request_state.response_event_count == 0
|
|
4285
|
+
# Internal warmup probes carry no anchor and prove nothing
|
|
4286
|
+
# about the key's continuity; charging them would open and
|
|
4287
|
+
# quarantine the hard key before any real turn, exactly the
|
|
4288
|
+
# states the completion settle already excludes.
|
|
4289
|
+
and terminal_request_state.request_kind != "prewarm"
|
|
4290
|
+
and not terminal_request_state.skip_request_log
|
|
4291
|
+
# Only a held safe replay excludes the strike: an unanchored
|
|
4292
|
+
# request with no replay is stranded like any other, and the
|
|
4293
|
+
# delta spec's no-response/no-safe-replay rule applies to it
|
|
4294
|
+
# the same way the retirement funnels apply it.
|
|
4295
|
+
and not _http_bridge_request_state_holds_safe_replay(terminal_request_state)
|
|
4296
|
+
):
|
|
4297
|
+
# An upstream terminal frame that fails the request before any
|
|
4298
|
+
# response event is the same pre-response failure the circuit
|
|
4299
|
+
# measures on eventless retirements; it reaches this settlement
|
|
4300
|
+
# path instead of the retirement funnel, so it would otherwise
|
|
4301
|
+
# never count. Attempt-scoped recording keeps a later
|
|
4302
|
+
# retirement of the same lifecycle from double-counting, and
|
|
4303
|
+
# the recorder itself drops non-circuit details and soft keys.
|
|
4304
|
+
#
|
|
4305
|
+
# This runs before the terminal frame and its queue sentinel
|
|
4306
|
+
# reach the client: once completion is observable the client can
|
|
4307
|
+
# resend the same anchor immediately, and the cooldown and
|
|
4308
|
+
# quarantine have to already be visible to that resend rather
|
|
4309
|
+
# than still awaiting durable I/O. Recovery paths that retry
|
|
4310
|
+
# this request in place have all declined by here.
|
|
4311
|
+
terminal_strike_failures = await self._record_http_bridge_retry_circuit_failure(
|
|
4312
|
+
session,
|
|
4313
|
+
detail=error_code,
|
|
4314
|
+
attempt=terminal_request_state.response_create_attempt,
|
|
4315
|
+
terminal_pre_response_frame=True,
|
|
4316
|
+
)
|
|
4317
|
+
terminal_poison_detail = await self._http_bridge_effective_anchor_poison_detail(session, error_code)
|
|
4318
|
+
|
|
4319
|
+
matched_event_queue = (
|
|
4320
|
+
completed_event_queue
|
|
4321
|
+
if completed_event_queue_claimed and matched_request_state is terminal_request_state
|
|
4322
|
+
else matched_request_state.event_queue
|
|
4323
|
+
if matched_request_state is not None
|
|
4324
|
+
else None
|
|
4325
|
+
)
|
|
4326
|
+
matched_deferred_texts = (
|
|
4327
|
+
_pop_websocket_deferred_reasoning_downstream_texts(matched_request_state)
|
|
4328
|
+
if matched_request_state is not None and not suppress_downstream_event
|
|
4329
|
+
else []
|
|
4330
|
+
)
|
|
4331
|
+
matched_terminal_state = _http_bridge_operation_state_for_event(event_type)
|
|
4332
|
+
if continuity_persistence_failed_after_ack and matched_request_state is not None:
|
|
4333
|
+
# The upstream response was already accepted. The downstream
|
|
4334
|
+
# failure only reports that its durable alias could not be
|
|
4335
|
+
# persisted, so keep the operation fenced as acknowledged while
|
|
4336
|
+
# retaining the failure SSE for the client.
|
|
4337
|
+
matched_terminal_state = "acknowledged"
|
|
4338
|
+
|
|
4339
|
+
async def _publish_matched_and_terminal_frames() -> None:
|
|
4340
|
+
nonlocal matched_terminal_enqueued
|
|
4341
|
+
if matched_request_state is not None and not suppress_downstream_event:
|
|
4342
|
+
for deferred_text in matched_deferred_texts:
|
|
4343
|
+
await _persist_http_bridge_operation_event(
|
|
4344
|
+
self,
|
|
4345
|
+
session,
|
|
4346
|
+
matched_request_state,
|
|
4347
|
+
deferred_text,
|
|
4348
|
+
terminal=False,
|
|
4349
|
+
)
|
|
4350
|
+
if matched_request_state is not None and matched_event_queue is not None and not suppress_downstream_event:
|
|
4351
|
+
for deferred_text in matched_deferred_texts:
|
|
4352
|
+
await matched_event_queue.put(deferred_text)
|
|
4353
|
+
if matched_request_state is not None and not suppress_downstream_event:
|
|
4354
|
+
matched_terminal_enqueued = await _persist_http_bridge_operation_event(
|
|
4355
|
+
self,
|
|
4356
|
+
session,
|
|
4357
|
+
matched_request_state,
|
|
4358
|
+
event_block,
|
|
4359
|
+
terminal=event_type in {"response.completed", "response.failed", "response.incomplete", "error"},
|
|
4360
|
+
terminal_state=matched_terminal_state,
|
|
4361
|
+
terminal_event_queue=matched_event_queue,
|
|
4362
|
+
terminal_delivery_scope=(completed_delivery_scope if completed_event_queue_claimed else None),
|
|
4363
|
+
)
|
|
4364
|
+
if (
|
|
4365
|
+
matched_request_state is not None
|
|
4366
|
+
and matched_event_queue is not None
|
|
4367
|
+
and not suppress_downstream_event
|
|
4368
|
+
and matched_terminal_enqueued is not True
|
|
4369
|
+
):
|
|
4370
|
+
await matched_event_queue.put(event_block)
|
|
4371
|
+
|
|
4372
|
+
if terminal_request_state is None:
|
|
4373
|
+
return
|
|
4374
|
+
|
|
4375
|
+
terminal_event_queue = (
|
|
4376
|
+
completed_event_queue if completed_event_queue_claimed else terminal_request_state.event_queue
|
|
4377
|
+
)
|
|
4378
|
+
terminal_enqueued = matched_terminal_enqueued if terminal_request_state is matched_request_state else False
|
|
4379
|
+
if terminal_request_state is not matched_request_state:
|
|
4380
|
+
deferred_texts = _pop_websocket_deferred_reasoning_downstream_texts(terminal_request_state)
|
|
4381
|
+
for deferred_text in deferred_texts:
|
|
4382
|
+
if not suppress_downstream_event:
|
|
4383
|
+
await _persist_http_bridge_operation_event(
|
|
4384
|
+
self,
|
|
4385
|
+
session,
|
|
4386
|
+
terminal_request_state,
|
|
4387
|
+
deferred_text,
|
|
4388
|
+
terminal=False,
|
|
4389
|
+
)
|
|
4390
|
+
if terminal_event_queue is not None:
|
|
4391
|
+
await terminal_event_queue.put(deferred_text)
|
|
4392
|
+
if not suppress_downstream_event:
|
|
4393
|
+
terminal_enqueued = await _persist_http_bridge_operation_event(
|
|
4394
|
+
self,
|
|
4395
|
+
session,
|
|
4396
|
+
terminal_request_state,
|
|
4397
|
+
event_block,
|
|
4398
|
+
terminal=True,
|
|
4399
|
+
terminal_state=(
|
|
4400
|
+
"acknowledged"
|
|
4401
|
+
if continuity_persistence_failed_after_ack
|
|
4402
|
+
and terminal_request_state is matched_request_state
|
|
4403
|
+
else _http_bridge_operation_state_for_event(event_type)
|
|
4404
|
+
),
|
|
4405
|
+
terminal_event_queue=terminal_event_queue,
|
|
4406
|
+
terminal_delivery_scope=(completed_delivery_scope if completed_event_queue_claimed else None),
|
|
4407
|
+
)
|
|
4408
|
+
if terminal_event_queue is not None and terminal_enqueued is not True:
|
|
4409
|
+
await terminal_event_queue.put(event_block)
|
|
4410
|
+
if terminal_event_queue is not None:
|
|
4411
|
+
if terminal_enqueued is not True:
|
|
4412
|
+
await terminal_event_queue.put(None)
|
|
4413
|
+
if completed_event_queue_claimed and completed_delivery_scope is not None:
|
|
4414
|
+
async with session.pending_lock:
|
|
4415
|
+
# Keep the completed claim authoritative after its producer
|
|
4416
|
+
# returns. A concurrent timeout may still be finishing
|
|
4417
|
+
# awaited recovery work before it rechecks this scope.
|
|
4418
|
+
completed_delivery_scope.terminal_enqueued = True
|
|
4419
|
+
|
|
4420
|
+
matched_terminal_enqueued = False
|
|
4421
|
+
terminal_publication_cancellation: asyncio.CancelledError | None = None
|
|
4422
|
+
if terminal_poison_detail is not None and terminal_strike_failures is not None:
|
|
4423
|
+
# Publication and settlement share one deferral for a poison
|
|
4424
|
+
# terminal: a cancellation landing inside the operation
|
|
4425
|
+
# persistence, between the queued frame and its sentinel, or on
|
|
4426
|
+
# any other publication await used to escape before the owned
|
|
4427
|
+
# settlement task even existed — the outer abort finalized the
|
|
4428
|
+
# popped request and the consult, abandonment, and marker never
|
|
4429
|
+
# started, leaving the poisoned durable anchor reusable once the
|
|
4430
|
+
# process-local quarantine expired.
|
|
4431
|
+
terminal_publication_task = scheduler.create_task(
|
|
4432
|
+
_publish_matched_and_terminal_frames(),
|
|
4433
|
+
name=f"http-bridge-terminal-publication-{session.durable_session_id}",
|
|
4434
|
+
)
|
|
4435
|
+
_terminal_publication_result, terminal_publication_cancellation = await _await_task_deferring_cancellation(
|
|
4436
|
+
terminal_publication_task
|
|
4437
|
+
)
|
|
4438
|
+
else:
|
|
4439
|
+
await _publish_matched_and_terminal_frames()
|
|
4440
|
+
|
|
4441
|
+
if terminal_request_state is None:
|
|
4442
|
+
return
|
|
4443
|
+
|
|
4444
|
+
if terminal_poison_detail is not None and terminal_strike_failures is not None:
|
|
4445
|
+
# The strike above opens the circuit, but the durable anchor that
|
|
4446
|
+
# failed is still stored. Only the retirement and close funnels ever
|
|
4447
|
+
# reached the poison clear, and a terminal frame settles through
|
|
4448
|
+
# neither, so the dead anchor survived every cooldown and re-poisoned
|
|
4449
|
+
# the key on the next reattach. Quarantine suppresses the injection
|
|
4450
|
+
# in the meantime, but it is process-local and expires; the durable
|
|
4451
|
+
# row has to be cleared for the recovery to hold.
|
|
4452
|
+
#
|
|
4453
|
+
# This gates on the circuit's own threshold rather than the
|
|
4454
|
+
# configurable anchor-poison threshold, and fires with the same
|
|
4455
|
+
# evidence that already quarantines the key: one decision, the
|
|
4456
|
+
# in-memory half suppressing the next injection and the durable half
|
|
4457
|
+
# clearing the stored anchor. The configurable threshold governs the
|
|
4458
|
+
# retirement and close funnels, where no circuit gates first. Here it
|
|
4459
|
+
# is unreachable by construction, which is issue #1830/#1852 itself:
|
|
4460
|
+
# the circuit opens at two failures and then refuses the key for
|
|
4461
|
+
# 60-600s per strike, so its default of seven is tens of minutes of
|
|
4462
|
+
# dead conversation away.
|
|
4463
|
+
#
|
|
4464
|
+
# Unlike the strike, this runs after the terminal frame is published.
|
|
4465
|
+
# A resend arriving in between is already covered by the quarantine
|
|
4466
|
+
# armed with the strike, so there is no reason to put a fenced
|
|
4467
|
+
# durable write in front of the client's own failure.
|
|
4468
|
+
#
|
|
4469
|
+
# The terminal frame is already on its way to the client, so
|
|
4470
|
+
# these awaits must not be the last thing that runs. If the
|
|
4471
|
+
# reader is cancelled while the episode consult's durable lookup
|
|
4472
|
+
# or the fenced write is in contention, the cancellation would
|
|
4473
|
+
# otherwise escape before the finalization below is entered and
|
|
4474
|
+
# the request would never be finalized despite having been
|
|
4475
|
+
# answered. The consult reads the currently registered episode,
|
|
4476
|
+
# not the detached count captured with the strike: a multiplexed
|
|
4477
|
+
# sibling that completed during publication has settled the
|
|
4478
|
+
# circuit and persisted a fresh anchor, and this clear must not
|
|
4479
|
+
# delete it.
|
|
4480
|
+
terminal_settlement_cancellation: asyncio.CancelledError | None = None
|
|
4481
|
+
try:
|
|
4482
|
+
terminal_settlement_detail = terminal_poison_detail
|
|
4483
|
+
|
|
4484
|
+
async def _terminal_consult_and_clear() -> None:
|
|
4485
|
+
consult_episode, consult_expected_anchor = await self._http_bridge_poison_anchor_clear_owed(
|
|
4486
|
+
session,
|
|
4487
|
+
consecutive_failures=terminal_strike_failures,
|
|
4488
|
+
)
|
|
4489
|
+
if consult_episode is not None:
|
|
4490
|
+
# A multiplexed survivor holding a verified safe
|
|
4491
|
+
# replay claims the source circuit generation right
|
|
4492
|
+
# before dispatch; settling under it would strip
|
|
4493
|
+
# that fence, so survivors count alongside the
|
|
4494
|
+
# terminal request, snapshotted at decision time.
|
|
4495
|
+
async with session.pending_lock:
|
|
4496
|
+
terminal_surviving_states = list(session.pending_requests)
|
|
4497
|
+
durable_cleared = await _abandon_durable_http_bridge_continuity(
|
|
4498
|
+
self,
|
|
4499
|
+
session,
|
|
4500
|
+
detail=terminal_settlement_detail,
|
|
4501
|
+
settle_circuit=_http_bridge_abandonment_may_settle_circuit(
|
|
4502
|
+
[terminal_request_state, *terminal_surviving_states]
|
|
4503
|
+
),
|
|
4504
|
+
expected_continuity=consult_expected_anchor,
|
|
4505
|
+
authorized_episode=consult_episode,
|
|
4506
|
+
)
|
|
4507
|
+
if durable_cleared:
|
|
4508
|
+
# The abandonment succeeded even when its circuit
|
|
4509
|
+
# settlement remains outstanding: without the
|
|
4510
|
+
# marker, the next strike finds empty continuity,
|
|
4511
|
+
# refuses another abandonment, and the restored
|
|
4512
|
+
# cooldown backs off an anchor that is already
|
|
4513
|
+
# gone.
|
|
4514
|
+
await self._http_bridge_mark_poison_anchor_cleared(session, episode=consult_episode)
|
|
4515
|
+
|
|
4516
|
+
# The terminal frame is already published and the request is
|
|
4517
|
+
# about to be finalized and removed: a cancellation escaping
|
|
4518
|
+
# the consult or the fenced rebind would leave the owed clear
|
|
4519
|
+
# with no lifecycle left to retry it, and once the
|
|
4520
|
+
# process-local quarantine expires the stored dead anchor is
|
|
4521
|
+
# reusable. Defer cancellation across the settlement like the
|
|
4522
|
+
# grouped and reader funnels, finalize regardless, then
|
|
4523
|
+
# re-raise.
|
|
4524
|
+
terminal_settlement_task = scheduler.create_task(
|
|
4525
|
+
_terminal_consult_and_clear(),
|
|
4526
|
+
name=f"http-bridge-terminal-poison-settlement-{session.durable_session_id}",
|
|
4527
|
+
)
|
|
4528
|
+
_terminal_result, terminal_settlement_cancellation = await _await_task_deferring_cancellation(
|
|
4529
|
+
terminal_settlement_task
|
|
4530
|
+
)
|
|
4531
|
+
finally:
|
|
4532
|
+
await _finalize_terminal_settlement(terminal_request_state)
|
|
4533
|
+
if terminal_settlement_cancellation is None:
|
|
4534
|
+
terminal_settlement_cancellation = terminal_publication_cancellation
|
|
4535
|
+
if terminal_settlement_cancellation is not None:
|
|
4536
|
+
raise terminal_settlement_cancellation
|
|
4537
|
+
else:
|
|
4538
|
+
await _finalize_terminal_settlement(terminal_request_state)
|